DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1489 @@
1
+ import math
2
+ from pathlib import Path
3
+ from typing import Iterable, cast
4
+
5
+ import matplotlib.pyplot as plt
6
+ import numpy as np
7
+ import torch
8
+ from dredge.motion_util import MotionEstimate
9
+ from matplotlib.lines import Line2D
10
+ from scipy.sparse.csgraph import connected_components
11
+ from tqdm.auto import tqdm
12
+
13
+ from ..clustering.gmm.mixture import (
14
+ Scores,
15
+ StreamingSpikeData,
16
+ TruncatedMixtureModel,
17
+ TruncatedSpikeData,
18
+ NeighborhoodLUT,
19
+ instantiate_and_bootstrap_tmm,
20
+ labels_from_scores,
21
+ run_split,
22
+ try_kmeans,
23
+ )
24
+ from ..transform import TemporalPCA
25
+ from ..util import spiketorch
26
+ from ..util.data_util import DARTsortSorting, get_tpca, resolve_path
27
+ from ..util.internal_config import (
28
+ ComputationConfig,
29
+ InterpolationParams,
30
+ RefinementConfig,
31
+ default_refinement_cfg,
32
+ )
33
+ from ..util.interpolation_util import (
34
+ NeighborhoodFiller,
35
+ StableFeaturesInterpolator,
36
+ pad_geom,
37
+ )
38
+ from ..util.job_util import ensure_computation_config
39
+ from ..util.multiprocessing_util import CloudpicklePoolExecutor, get_pool
40
+ from ..util.py_util import databag
41
+ from .analysis_plots import distance_matrix_dendro
42
+ from .colors import glasbey1024
43
+ from .layout import BasePlot, flow_layout
44
+ from .waveforms import geomplot
45
+
46
+
47
+ @databag
48
+ class MixtureVisData:
49
+ """Collection passed around to the plots in this file, with helper methods for vis."""
50
+
51
+ tmm: TruncatedMixtureModel
52
+ train_data: TruncatedSpikeData
53
+ val_data: TruncatedSpikeData | None
54
+ full_data: StreamingSpikeData
55
+ sorting: DARTsortSorting
56
+ motion_est: MotionEstimate | None
57
+ train_scores: Scores
58
+ full_scores: Scores
59
+ eval_scores: Scores
60
+ train_times: np.ndarray
61
+ train_labels: np.ndarray
62
+ train_ixs: np.ndarray
63
+ full_labels: np.ndarray
64
+ tpca: TemporalPCA
65
+ inf_diag_unit_distance_matrix: torch.Tensor
66
+ prgeom: np.ndarray
67
+
68
+ @property
69
+ def times_seconds(self):
70
+ return self.sorting.times_seconds # type: ignore
71
+
72
+ def to_sorting(self) -> DARTsortSorting:
73
+ return self.sorting.ephemeral_replace(labels=self.full_labels)
74
+
75
+ def inu_and_times_full(self, unit_id: int):
76
+ inu_full = np.flatnonzero(self.full_labels == unit_id)
77
+ times = self.times_seconds[inu_full]
78
+ return inu_full, times
79
+
80
+ def train_inds_and_chans(
81
+ self, unit_id: int, count=None
82
+ ) -> tuple[np.ndarray, np.ndarray]:
83
+ inu_train = np.flatnonzero(self.train_labels == unit_id)
84
+ if count and inu_train.size > count:
85
+ rg = np.random.default_rng(0)
86
+ inu_train = rg.choice(inu_train, size=count, replace=False)
87
+ inu_train.sort()
88
+
89
+ neighb_ids = self.train_data.neighborhood_ids[inu_train].cpu()
90
+ chans = self.tmm.neighb_cov.obs_ix.cpu()[neighb_ids]
91
+ return inu_train, chans.numpy(force=True)
92
+
93
+ def chans_in_radius(self, unit_id: int, radius: float):
94
+ mean = self.tmm.b.means[unit_id].view(self.tmm.neighb_cov.feat_rank, -1)
95
+ my_chan = mean.square().sum(0).argmax()
96
+ my_xy = self.tmm.neighb_cov.prgeom[my_chan]
97
+ dxy = self.tmm.neighb_cov.prgeom - my_xy
98
+ inf_dist = dxy.abs_().amax(dim=1)
99
+ (close,) = (inf_dist < radius).cpu().nonzero(as_tuple=True)
100
+ return close
101
+
102
+ def friends(self, unit_id: int, count: int, me_last=True):
103
+ dists, neighbors = self.inf_diag_unit_distance_matrix[unit_id].sort()
104
+ count = min(count, dists.numel() - 1)
105
+ assert count >= 0
106
+ if count == 0:
107
+ return np.zeros(1), np.array([unit_id])
108
+ dists = dists[:count]
109
+ neighbors = neighbors[:count]
110
+ assert not (neighbors == unit_id).any()
111
+ assert dists.isfinite().all()
112
+ neighbors = np.concatenate([[unit_id], neighbors.numpy(force=True)])
113
+ dists = np.concatenate([[0.0], dists.numpy(force=True)])
114
+ if me_last:
115
+ dists = np.ascontiguousarray(dists[::-1])
116
+ neighbors = np.ascontiguousarray(neighbors[::-1])
117
+ return dists, neighbors
118
+
119
+ def reconstruct_flat(self, features: torch.Tensor) -> np.ndarray:
120
+ frank = self.tmm.neighb_cov.feat_rank
121
+ single = features.ndim == 1
122
+ if single:
123
+ features = features[None]
124
+ assert features.ndim == 2
125
+ n = features.shape[0]
126
+ features = features.view(n, frank, -1)
127
+ features = features.to(self.tpca.b.components.device)
128
+ recon = self.tpca.force_reconstruct(features)
129
+ if single:
130
+ recon = recon[0]
131
+ return recon.numpy(force=True)
132
+
133
+ def random_train_waveforms(
134
+ self, unit_id: int, count=128
135
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
136
+ inu_train, chans = self.train_inds_and_chans(unit_id, count=count)
137
+ features = self.train_data.x[inu_train]
138
+ waveforms = self.reconstruct_flat(features)
139
+ features = features.numpy(force=True)
140
+ return inu_train, chans, features, waveforms
141
+
142
+ def full_lut_scores(self, show_progress=2, n_candidates=None):
143
+ lut0 = self.tmm.lut
144
+ lut1 = torch.ones_like(lut0.lut)
145
+ unit_ids, neighb_ids = lut1.nonzero(as_tuple=True)
146
+ lut1[unit_ids, neighb_ids] = torch.arange(unit_ids.shape[0]).to(lut1)
147
+ new_lut = NeighborhoodLUT(unit_ids=unit_ids, neighb_ids=neighb_ids, lut=lut1)
148
+ self.tmm.update_lut(new_lut, no_parameter_changes=True)
149
+ # old_nc = self.full_data.n_candidates
150
+ sc = self.tmm.soft_assign(
151
+ data=self.full_data,
152
+ needs_bootstrap=False,
153
+ full_proposal_view=True,
154
+ show_progress=show_progress,
155
+ )
156
+ return sc
157
+
158
+
159
+ class MixtureComponentPlot(BasePlot):
160
+ width = 1
161
+ height = 1
162
+ kind = "mixture"
163
+
164
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
165
+ raise NotImplementedError
166
+
167
+
168
+ class TextInfo(MixtureComponentPlot):
169
+ kind = "small"
170
+ height = 0.5
171
+
172
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
173
+ axis = panel.subplots()
174
+ axis.axis("off")
175
+
176
+ ntot = (mix_data.full_labels == unit_id).sum()
177
+ ntrain = (mix_data.train_labels == unit_id).sum()
178
+
179
+ msg = f"$\\mathbf{{unit\\ {unit_id}}}$\n"
180
+ msg += f"n total: {ntot}\n"
181
+ msg += f"n train: {ntrain}\n"
182
+
183
+ axis.text(
184
+ 0.025,
185
+ 0.5,
186
+ msg,
187
+ fontsize="small",
188
+ ha="left",
189
+ va="center",
190
+ transform=axis.transAxes,
191
+ )
192
+
193
+
194
+ class ISIHistogram(MixtureComponentPlot):
195
+ kind = "small"
196
+ height = 1.0
197
+
198
+ def __init__(self, bin_ms=0.1, max_ms=5):
199
+ self.bin_ms = bin_ms
200
+ self.max_ms = max_ms
201
+
202
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
203
+ axis = panel.subplots()
204
+ inu_full, times_s = mix_data.inu_and_times_full(unit_id)
205
+ dt_ms = np.diff(times_s) * 1000
206
+ bin_edges = np.arange(
207
+ -0.5 * self.bin_ms, self.max_ms + self.bin_ms + 1e-5, self.bin_ms
208
+ )
209
+ counts, _ = np.histogram(dt_ms, bin_edges)
210
+ axis.stairs(counts, bin_edges, color=glasbey1024[unit_id], fill=True)
211
+ axis.set_xlabel(f"isi (ms, {dt_ms.size + 1} tot. sp.)")
212
+ axis.set_ylabel("count")
213
+
214
+
215
+ class ChansHeatmap(MixtureComponentPlot):
216
+ kind = "small"
217
+ height = 1.5
218
+
219
+ def __init__(self, cmap="magma", snr_cmap="viridis"):
220
+ self.cmap = plt.get_cmap(cmap)
221
+ self.snr_cmap = plt.get_cmap(snr_cmap)
222
+
223
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
224
+ inu_train, chans = mix_data.train_inds_and_chans(unit_id)
225
+ chans = chans[chans < mix_data.tmm.neighb_cov.n_channels]
226
+ uchans, counts = np.unique(chans, return_counts=True)
227
+
228
+ mean = mix_data.tmm.b.means[unit_id].view(
229
+ -1, mix_data.tmm.neighb_cov.n_channels
230
+ )
231
+ ptp = spiketorch.ptp(mean, dim=0).cpu()
232
+ (support,) = ptp[uchans].nonzero(as_tuple=True)
233
+ support = uchans[support]
234
+ ptp = ptp[support]
235
+
236
+ ax = panel.subplots()
237
+ xy = mix_data.tmm.neighb_cov.prgeom.numpy(force=True)
238
+ ax.scatter(*xy[support].T, c=ptp, lw=2, s=5)
239
+ s = ax.scatter(*xy[uchans].T, c=counts, lw=0, cmap=self.cmap, s=5)
240
+ plt.colorbar(s, ax=ax, shrink=0.3, label="chan count", pad=0.01)
241
+ ax.scatter(*xy[support[ptp.argmax()]].T, color="gold", lw=1, fc="none", s=5)
242
+ ax.set_title(
243
+ f"chans:{chans.min().item()}-{chans.max().item()}", fontsize="small"
244
+ )
245
+
246
+
247
+ class LikelihoodsView(MixtureComponentPlot):
248
+ def __init__(self, viol_ms=1.0, layout="vert"):
249
+ self.viol_ms = viol_ms
250
+ self.layout = layout
251
+ if layout == "vert":
252
+ self.kind = "small"
253
+ self.width = 1
254
+ self.height = 4
255
+ self.ncols = 1
256
+ self.nrows = 3
257
+ self.width_ratios = [1]
258
+ elif layout == "horz":
259
+ self.kind = "block"
260
+ self.width = 2
261
+ self.height = 1
262
+ self.ncols = 3
263
+ self.nrows = 1
264
+ self.width_ratios = [3, 2, 1]
265
+ else:
266
+ assert False
267
+
268
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
269
+ inu, t = mix_data.inu_and_times_full(unit_id)
270
+ my_ll = mix_data.full_scores.log_liks[:, 0][inu]
271
+ noise_ll = mix_data.full_scores.log_liks[:, -1][inu]
272
+ dt_ms = np.diff(t) * 1000
273
+ viol = dt_ms <= self.viol_ms
274
+ viol = np.logical_or(
275
+ np.pad(viol, (1, 0), constant_values=False),
276
+ np.pad(viol, (0, 1), constant_values=False),
277
+ )
278
+ viol = np.flatnonzero(viol)
279
+ my_min = my_ll.min()
280
+ my_max = my_ll.max()
281
+ noise_min = noise_ll.min()
282
+ noise_max = noise_ll.max()
283
+ mn = min(my_min, noise_min)
284
+ mx = max(my_max, noise_max)
285
+ assert math.isfinite(mn)
286
+ return my_ll, noise_ll, t, viol, mn, mx
287
+
288
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
289
+ my_ll, noise_ll, t, viol, mn, mx = self.compute(mix_data, unit_id)
290
+
291
+ ax_time, ax_noise, ax_hist = panel.subplots(
292
+ ncols=self.ncols,
293
+ nrows=self.nrows,
294
+ width_ratios=self.width_ratios,
295
+ sharey=True,
296
+ )
297
+ ax_time.set_ylabel("log likelihood")
298
+ ax_time.set_xlabel("time (s)")
299
+ ax_noise.set_xlabel("noise loglik")
300
+ ax_hist.set_xlabel("count")
301
+ c = glasbey1024[unit_id]
302
+
303
+ ax_time.scatter(t, noise_ll, s=3, lw=0, color="darkgray")
304
+ ax_time.scatter(t, my_ll, s=3, lw=0, color=c)
305
+ ax_time.scatter(t[viol], my_ll[viol], s=3, lw=0, color="k")
306
+
307
+ ax_noise.scatter(noise_ll, my_ll, s=3, lw=0, color=c, zorder=1)
308
+ ax_noise.plot([mn, mx], [mn, mx], color="k", lw=0.8, zorder=11)
309
+
310
+ histk = dict(histtype="step", orientation="horizontal")
311
+ _, bins, _ = ax_hist.hist(my_ll, color=c, label="unit", bins=64, **histk)
312
+ ax_hist.hist(noise_ll, color="darkgray", label="noise", bins=bins, **histk)
313
+ ax_hist.legend(
314
+ loc="lower right",
315
+ frameon=False,
316
+ borderpad=0.1,
317
+ borderaxespad=0.1,
318
+ handletextpad=0.3,
319
+ handlelength=1.0,
320
+ )
321
+
322
+
323
+ class NeighborMeans(MixtureComponentPlot):
324
+ kind = "block"
325
+ width = 2
326
+ height = 2
327
+
328
+ def __init__(self, count=5, vis_radius=50.0):
329
+ self.count = count
330
+ self.vis_radius = vis_radius
331
+
332
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
333
+ dists, neighbors = mix_data.friends(unit_id, count=self.count)
334
+ k = len(neighbors)
335
+ chans = mix_data.chans_in_radius(unit_id, self.vis_radius)
336
+ means = mix_data.tmm.b.means[neighbors].view(
337
+ k, -1, mix_data.tmm.neighb_cov.n_channels
338
+ )
339
+ means = means[:, :, chans].view(k, -1)
340
+ means = mix_data.reconstruct_flat(means)
341
+ return chans, neighbors, means
342
+
343
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
344
+ chans, neighbors, means = self.compute(mix_data, unit_id)
345
+ colors = [glasbey1024[j] for j in neighbors]
346
+ ax = panel.subplots()
347
+ chans = chans[None].broadcast_to(len(means), *chans.shape)
348
+ geomplot(
349
+ waveforms=means,
350
+ channels=chans.numpy(force=True),
351
+ geom=mix_data.prgeom,
352
+ show_zero=False,
353
+ colors=colors,
354
+ zlim=None,
355
+ ax=ax,
356
+ subar=True,
357
+ annotate_z=True,
358
+ )
359
+ panel.legend(
360
+ handles=[Line2D([0, 1], [0, 0], color=c) for c in colors],
361
+ labels=list(neighbors),
362
+ loc="outside upper center",
363
+ frameon=False,
364
+ ncols=min(5, len(neighbors)),
365
+ fontsize="small",
366
+ borderpad=0,
367
+ labelspacing=0.25,
368
+ handlelength=1.0,
369
+ handletextpad=0.4,
370
+ borderaxespad=0.0,
371
+ columnspacing=1.0,
372
+ )
373
+ ax.axis("off")
374
+
375
+
376
+ class NeighborDistances(MixtureComponentPlot):
377
+ kind = "block"
378
+ width = 2
379
+ height = 2
380
+
381
+ def __init__(self, count=5, cmap="plasma"):
382
+ self.count = count
383
+ self.cmap = plt.get_cmap(cmap)
384
+
385
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
386
+ dists, neighbors = mix_data.friends(unit_id, count=self.count)
387
+ d = mix_data.inf_diag_unit_distance_matrix[neighbors][:, neighbors]
388
+ d = d.numpy(force=True)
389
+ np.fill_diagonal(d, 0.0)
390
+ return d, neighbors
391
+
392
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
393
+ d, neighbors = self.compute(mix_data, unit_id)
394
+ distance_matrix_dendro(
395
+ panel,
396
+ d,
397
+ unit_ids=neighbors,
398
+ dendrogram_linkage=None,
399
+ show_unit_labels=True,
400
+ vmax=mix_data.tmm.max_distance,
401
+ image_cmap=self.cmap,
402
+ show_values=True,
403
+ )
404
+ panel.suptitle(f"{mix_data.tmm.distance_kind} dists", fontsize="small")
405
+
406
+
407
+ class MergeView(MixtureComponentPlot):
408
+ kind = "block"
409
+ width = 2
410
+ height = 2.5
411
+
412
+ def __init__(self):
413
+ pass
414
+
415
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
416
+ # get pair mask
417
+ gsize = mix_data.tmm.max_group_size - 1
418
+ dists, neighbors = mix_data.friends(unit_id, count=gsize)
419
+ d = mix_data.inf_diag_unit_distance_matrix[neighbors][:, neighbors]
420
+ d.diagonal().fill_(0.0)
421
+ pair_mask = d < mix_data.tmm.merge_max_distance
422
+
423
+ # determine connected components and re-order (stably)
424
+ n_comps, labels = connected_components(pair_mask.numpy(force=True))
425
+ if n_comps > 1:
426
+ reorder = [np.flatnonzero(labels == l) for l in range(n_comps)]
427
+ reorder = np.concatenate(reorder)
428
+ d = d[reorder][:, reorder]
429
+ pair_mask = pair_mask[reorder][:, reorder]
430
+ neighbors = neighbors[reorder]
431
+
432
+ # run it
433
+ group_res = mix_data.tmm.try_merge_group(
434
+ group=torch.asarray(neighbors),
435
+ train_data=mix_data.train_data,
436
+ eval_data=mix_data.val_data,
437
+ scores=mix_data.eval_scores,
438
+ pair_mask=pair_mask,
439
+ apply_adj_mask=True,
440
+ debug=True,
441
+ )
442
+
443
+ return neighbors, d, pair_mask, group_res
444
+
445
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
446
+ neighbors, dist, pair_mask, group_res = self.compute(mix_data, unit_id)
447
+ panel_mask, panel_info = panel.subfigures(nrows=2, height_ratios=[2, 0.5])
448
+ distance_matrix_dendro(
449
+ panel_mask,
450
+ pair_mask,
451
+ show_values_from=dist,
452
+ unit_ids=neighbors,
453
+ dendrogram_linkage=None,
454
+ show_unit_labels=True,
455
+ image_cmap="RdGy_r",
456
+ show_values=True,
457
+ with_colorbar=False,
458
+ value_color="w",
459
+ )
460
+ ax_info = panel_info.subplots()
461
+ ax_info.axis("off")
462
+ if group_res is None:
463
+ msg = "bail"
464
+ else:
465
+ propstr = ",".join(f"{p:0.2f}" for p in group_res.sub_proportions.cpu())
466
+ msg = (
467
+ f"part: {group_res.grouping.group_ids.tolist()}\n"
468
+ f"imp: {group_res.improvement:.3f}\n"
469
+ f"props: {propstr}"
470
+ )
471
+ ax_info.text(
472
+ 0.2, 0.5, msg, fontsize="small", va="center", transform=ax_info.transAxes
473
+ )
474
+ panel.suptitle("merge pair mask", fontsize="small")
475
+
476
+
477
+ class MeanView(MixtureComponentPlot):
478
+ def __init__(
479
+ self,
480
+ n_waveforms_show=128,
481
+ alpha=0.25,
482
+ show_zero=True,
483
+ mini=False,
484
+ mini_rad=41.0,
485
+ ):
486
+ self.n_waveforms_show = n_waveforms_show
487
+ self.alpha = alpha
488
+ self.show_zero = show_zero
489
+ self.mini = mini
490
+ self.mini_rad = mini_rad
491
+ if mini:
492
+ self.kind = "tall"
493
+ self.width = 2.5
494
+ self.height = 2
495
+ else:
496
+ self.kind = "bigblock"
497
+ self.width = 3
498
+ self.height = 4
499
+
500
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
501
+ mean = mix_data.tmm.b.means[unit_id]
502
+ mean_recon = mix_data.reconstruct_flat(mean)
503
+ mean_recon = mean_recon.reshape(1, -1, mix_data.tmm.neighb_cov.n_channels)
504
+ inu_train, wchans, features, waveforms = mix_data.random_train_waveforms(
505
+ unit_id=unit_id, count=self.n_waveforms_show
506
+ )
507
+ if self.mini:
508
+ targchans = mix_data.chans_in_radius(unit_id, radius=self.mini_rad)
509
+ wchans = torch.asarray(wchans).to(targchans)
510
+ vchans = torch.isin(wchans, targchans)
511
+ vii, vcc = vchans.cpu().nonzero(as_tuple=True)
512
+ wchans = wchans[vii, vcc].cpu()
513
+ waveforms = waveforms[vii, :, vcc]
514
+ wchans = wchans[:, None]
515
+ assert wchans.ndim == 2
516
+ waveforms = waveforms[:, :, None]
517
+ assert waveforms.ndim == 3
518
+ maa = np.nanmax(np.abs(waveforms)).item()
519
+ return waveforms, wchans, mean_recon, maa
520
+
521
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
522
+ waveforms, wchans, mean_recon, maa = self.compute(mix_data, unit_id)
523
+
524
+ ax = panel.subplots()
525
+ ax.axis("off")
526
+ lines, pchans = geomplot( # type: ignore
527
+ waveforms=waveforms,
528
+ channels=wchans,
529
+ color="k",
530
+ alpha=self.alpha,
531
+ ax=ax,
532
+ max_abs_amp=maa,
533
+ geom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
534
+ show_zero=self.show_zero,
535
+ linewidth=0.5,
536
+ return_chans=True,
537
+ )
538
+ pchans = np.array(list(pchans))
539
+ geomplot(
540
+ waveforms=mean_recon[:, :, pchans],
541
+ channels=pchans[None],
542
+ max_abs_amp=maa,
543
+ geom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
544
+ color=glasbey1024[unit_id],
545
+ ax=ax,
546
+ show_zero=False,
547
+ annotate_z=True,
548
+ )
549
+
550
+
551
+ class CovarianceView(MixtureComponentPlot):
552
+ kind = "bigblock"
553
+ width = 3
554
+ height = 3
555
+
556
+ def __init__(
557
+ self, n_waveforms_show=128, neigs=16, cov_vert=False, reference_cov="interp"
558
+ ):
559
+ self.n_waveforms_show = n_waveforms_show
560
+ self.colors = dict(emp="k", interp="gray", noise="r", signal="g", model="b")
561
+ self.neigs = neigs
562
+ self.cov_vert = cov_vert
563
+ self.reference_cov = reference_cov
564
+
565
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
566
+ # load data
567
+ inu_train, wchans, features, waveforms = mix_data.random_train_waveforms(
568
+ unit_id=unit_id, count=self.n_waveforms_show
569
+ )
570
+
571
+ # pick channels and put data there
572
+ chan_set = torch.asarray(wchans).unique().to(device=mix_data.tmm.b.means.device)
573
+ chan_set = chan_set[chan_set < mix_data.tmm.neighb_cov.n_channels]
574
+ features = features.reshape(
575
+ features.shape[0], -1, mix_data.tmm.neighb_cov.max_nc_obs
576
+ )
577
+ features = torch.asarray(features)
578
+ # -- with nans
579
+ feat_nan = features.new_full(
580
+ (*features.shape[:2], mix_data.tmm.neighb_cov.n_channels + 1), torch.nan
581
+ )
582
+ ix = torch.asarray(wchans)
583
+ ix = ix[:, None, :].broadcast_to(features.shape)
584
+ feat_nan.scatter_(
585
+ dim=2,
586
+ index=ix,
587
+ src=features,
588
+ )
589
+ feat_nan = feat_nan[:, :, chan_set.cpu()]
590
+ feat_nan = feat_nan.view(feat_nan.shape[0], -1)
591
+ # -- with interp
592
+ feat_interp = mix_data.tmm.erp.interp_to_chans(
593
+ waveforms=features.to(mix_data.tmm.b.means),
594
+ neighborhood_ids=mix_data.train_data.neighborhood_ids[inu_train],
595
+ target_channels=chan_set,
596
+ )
597
+ feat_interp = feat_interp.view(feat_interp.shape[0], -1)
598
+
599
+ # empirical covariance
600
+ mean_nan = torch.nanmean(feat_nan, dim=0)
601
+ feat_nan -= mean_nan
602
+ cov_nan = cast(torch.Tensor, spiketorch.nancov(feat_nan, correction=0))
603
+
604
+ # interp covariance
605
+ cov_interp = torch.cov(feat_interp.T)
606
+
607
+ # noise covariance
608
+ cov_noise = mix_data.tmm.noise.marginal_covariance(channels=chan_set).to_dense()
609
+ assert cov_noise.shape == cov_interp.shape
610
+
611
+ # signal covariance
612
+ if mix_data.tmm.signal_rank:
613
+ basis = mix_data.tmm.b.bases[unit_id]
614
+ basis = basis.view(
615
+ mix_data.tmm.signal_rank, -1, mix_data.tmm.neighb_cov.n_channels
616
+ )
617
+ basis = basis[:, :, chan_set].view(mix_data.tmm.signal_rank, -1)
618
+ basis = basis * mix_data.tmm.latent_prior_std
619
+ cov_signal = basis.T @ basis
620
+ assert cov_signal.shape == cov_noise.shape
621
+ else:
622
+ cov_signal = torch.zeros_like(cov_noise)
623
+
624
+ # model covariance
625
+ cov_model = cov_noise + cov_signal
626
+
627
+ # np-ify
628
+ cov_nan = cov_nan.numpy(force=True)
629
+ cov_interp = cov_interp.numpy(force=True)
630
+ cov_noise = cov_noise.numpy(force=True)
631
+ cov_signal = cov_signal.numpy(force=True)
632
+ cov_model = cov_model.numpy(force=True)
633
+
634
+ # spectra: empirical, noise, sginal, model
635
+ ev_nan = np.linalg.eigvalsh(cov_nan)[::-1]
636
+ ev_interp = np.linalg.eigvalsh(cov_interp)[::-1]
637
+ ev_noise = np.linalg.eigvalsh(cov_noise)[::-1]
638
+ ev_signal = np.linalg.eigvalsh(cov_signal)[::-1]
639
+ ev_model = np.linalg.eigvalsh(cov_model)[::-1]
640
+
641
+ return {
642
+ "emp": (cov_nan, ev_nan),
643
+ "interp": (cov_interp, ev_interp),
644
+ "noise": (cov_noise, ev_noise),
645
+ "signal": (cov_signal, ev_signal),
646
+ "model": (cov_model, ev_model),
647
+ }
648
+
649
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
650
+ stats = self.compute(mix_data, unit_id)
651
+ cov_emp = stats[self.reference_cov][0]
652
+ vm = np.percentile(np.abs(cov_emp), 98)
653
+
654
+ toph = 2 + int(self.cov_vert)
655
+ top, bottom = panel.subfigures(nrows=2, height_ratios=[toph, 1])
656
+ if self.cov_vert:
657
+ axes_top = top.subplots(ncols=2, nrows=len(stats))
658
+ else:
659
+ axes_top = top.subplots(ncols=len(stats), nrows=2)
660
+ axes_top = axes_top.T
661
+ ax_bottom = bottom.subplots()
662
+
663
+ cov_kw = dict(
664
+ aspect=1.0, interpolation="none", vmin=-vm, vmax=vm, cmap="seismic"
665
+ )
666
+ res_kw = cov_kw | dict(vmin=-vm, vmax=vm)
667
+
668
+ for row_top, (name, (cov, ev)) in zip(axes_top, stats.items()):
669
+ c = self.colors[name]
670
+ if self.cov_vert:
671
+ row_top[0].set_ylabel(name, color=c)
672
+ else:
673
+ row_top[0].set_title(name, color=c, fontsize=6)
674
+ ima = row_top[0].imshow(cov, **cov_kw)
675
+ imb = row_top[1].imshow(cov_emp - cov, **res_kw)
676
+ ax_bottom.plot(ev[: self.neigs], color=c, label=name)
677
+ if not self.cov_vert:
678
+ plt.colorbar(ima, ax=row_top[0], shrink=0.3) # type: ignore
679
+ plt.colorbar(imb, ax=row_top[1], shrink=0.3) # type: ignore
680
+
681
+ if self.cov_vert:
682
+ axes_top[0, 0].set_title("cov", fontsize="small")
683
+ axes_top[0, 1].set_title(
684
+ f"{self.reference_cov} - cov",
685
+ fontsize="small",
686
+ color=self.colors[self.reference_cov],
687
+ )
688
+ else:
689
+ axes_top[0, 0].set_ylabel("cov", fontsize="small")
690
+ axes_top[0, 1].set_ylabel(
691
+ f"{self.reference_cov} - cov",
692
+ fontsize="small",
693
+ color=self.colors[self.reference_cov],
694
+ )
695
+ for ax in axes_top.flat:
696
+ ax.set_xticks([])
697
+ ax.set_yticks([])
698
+ ax_bottom.grid(which="both")
699
+ ax_bottom.legend(
700
+ borderpad=0.1,
701
+ borderaxespad=0.1,
702
+ handletextpad=0.3,
703
+ handlelength=1.0,
704
+ loc="upper right",
705
+ ncols=2,
706
+ )
707
+
708
+
709
+ class SplitView(MixtureComponentPlot):
710
+ kind = "tall"
711
+ width = 2.5
712
+ height = 5
713
+
714
+ def __init__(
715
+ self,
716
+ colors=["r", "g", "b", "darkorange", "darkviolet", "mediumturquoise"],
717
+ bail_color="k",
718
+ vis_radius=50.0,
719
+ dist_cmap="plasma",
720
+ ):
721
+ self.colors = np.array(colors)
722
+ self.bail_color = bail_color
723
+ self.vis_radius = vis_radius
724
+ self.dist_cmap = plt.get_cmap(dist_cmap)
725
+
726
+ def compute(self, mix_data: MixtureVisData, unit_id: int):
727
+ split_res, debug_info = mix_data.tmm.split_unit(
728
+ unit_id=unit_id,
729
+ train_data=mix_data.train_data,
730
+ eval_data=mix_data.val_data,
731
+ scores=mix_data.eval_scores,
732
+ debug=True,
733
+ )
734
+ assert debug_info is not None
735
+
736
+ if debug_info.split_data is not None:
737
+ n_spikes = debug_info.split_data.x.shape[0]
738
+ else:
739
+ n_spikes = 0
740
+
741
+ # compute colors by kmeans label
742
+ if debug_info.kmeans_responsibilities is not None:
743
+ assert debug_info.kmeans_responsibilities.shape[0] == n_spikes
744
+ kmeans_labels = debug_info.kmeans_responsibilities.argmax(dim=1)
745
+ kmeans_labels = kmeans_labels.cpu()
746
+ colors = self.colors[kmeans_labels]
747
+ else:
748
+ kmeans_labels = torch.zeros(n_spikes, dtype=torch.long)
749
+ colors = np.broadcast_to(np.array([self.bail_color]), (n_spikes,))
750
+
751
+ # compute channels by kmeans label
752
+ if debug_info.kmeans_responsibilities is not None:
753
+ assert debug_info.split_data is not None
754
+ chans_by_km = {}
755
+ for l in kmeans_labels.unique():
756
+ neighbs_l = debug_info.split_data.neighborhood_ids[kmeans_labels == l]
757
+ chans_l = mix_data.tmm.neighb_cov.obs_ix[neighbs_l]
758
+ chans_l = chans_l[chans_l < mix_data.tmm.neighb_cov.n_channels]
759
+ chans_by_km[l.item()] = chans_l.numpy(force=True)
760
+ else:
761
+ chans_by_km = None
762
+
763
+ # compute amplitudes
764
+ if debug_info.split_data is not None:
765
+ x = debug_info.split_data.x.view(
766
+ n_spikes, -1, mix_data.tmm.neighb_cov.max_nc_obs
767
+ )
768
+ t = mix_data.train_times[debug_info.split_data.indices.cpu()]
769
+ amps = x.square().sum(dim=1).amax(dim=1).sqrt_().numpy(force=True)
770
+ else:
771
+ t = amps = None
772
+
773
+ # compute pca embedding of kmeans_x
774
+ # TODO: also embed split_model means and maybe final model?
775
+ if debug_info.kmeans_x is not None:
776
+ assert debug_info.kmeans_x.shape[0] == n_spikes
777
+ assert debug_info.kmeans_chans is not None
778
+ kmean = debug_info.kmeans_x.mean(0)
779
+ loadings, components, *_ = spiketorch.svd_lowrank_helper(
780
+ x=debug_info.kmeans_x,
781
+ rank=2,
782
+ with_loadings=True,
783
+ M=kmean[None].broadcast_to(debug_info.kmeans_x.shape),
784
+ )
785
+ assert loadings is not None
786
+ loadings = loadings.numpy(force=True)
787
+ else:
788
+ loadings = None
789
+
790
+ # distances
791
+ if debug_info.split_model is not None:
792
+ dists = debug_info.split_model.unit_distance_matrix()
793
+ dists = dists.numpy(force=True)
794
+ else:
795
+ dists = None
796
+
797
+ # mean reconstructions near my main channel
798
+ mean_chans = mix_data.chans_in_radius(unit_id, radius=self.vis_radius)
799
+ if debug_info.split_model is not None:
800
+ means = debug_info.split_model.b.means
801
+ means = means.view(means.shape[0], -1, mix_data.tmm.neighb_cov.n_channels)
802
+ means = means[:, :, mean_chans]
803
+ means = means.view(means.shape[0], -1)
804
+ means = mix_data.reconstruct_flat(means)
805
+ else:
806
+ means = None
807
+
808
+ # compute text info
809
+ txt = ""
810
+ if debug_info.bailed:
811
+ txt += f"bail: {debug_info.bail_reason}\n"
812
+ if debug_info.merge_res is not None:
813
+ txt += (
814
+ f"part: {debug_info.merge_res.grouping.group_ids.tolist()}\n"
815
+ f"imp: {debug_info.merge_res.improvement:.3f}\n"
816
+ )
817
+ else:
818
+ txt += f"no allowed partitions\n"
819
+ if debug_info.split_model is not None:
820
+ sprops = debug_info.split_model.b.log_proportions.cpu()
821
+ sprops = sprops - mix_data.tmm.b.log_proportions[unit_id].cpu()
822
+ propstr = "+".join(f"{p:0.3f}" for p in sprops.exp())
823
+ txt += f"norm kmeans model props:\n {propstr}={sprops.exp().sum():.2f}\n"
824
+ if debug_info.merge_res is not None:
825
+ mresprops = debug_info.merge_res.sub_proportions
826
+ propstr = "+".join(f"{p:0.3f}" for p in mresprops)
827
+ txt += f"mres props: {propstr}={mresprops.sum():.2f}\n"
828
+ if debug_info.kmeans_responsibilities is not None:
829
+ kcountstr = ",".join(
830
+ str(p.item()) for p in kmeans_labels.unique(return_counts=True)[1]
831
+ )
832
+ txt += f"kmeans counts: {kcountstr}\n"
833
+ if split_res is not None:
834
+ propstr = "+".join(f"{p:0.3f}" for p in split_res.sub_proportions.cpu())
835
+ countstr = ",".join(
836
+ str(p.item())
837
+ for p in split_res.train_assignments.unique(return_counts=True)[1]
838
+ )
839
+ txt += f"props: {propstr}={split_res.sub_proportions.cpu().sum():.2f}\n"
840
+ txt += f"counts: {countstr}\n"
841
+ txt = txt.rstrip()
842
+
843
+ return (
844
+ txt,
845
+ colors,
846
+ t,
847
+ amps,
848
+ loadings,
849
+ dists,
850
+ means,
851
+ mean_chans,
852
+ chans_by_km,
853
+ )
854
+
855
+ def draw(self, panel, mix_data: MixtureVisData, unit_id: int):
856
+ c = self.compute(mix_data, unit_id)
857
+ txt, colors, t, amps, loadings, dists, means, mean_chans, chans_by_km = c
858
+
859
+ # layout
860
+ amp_info_row, mean_row, dist_pc_row = panel.subfigures(
861
+ nrows=3, height_ratios=[1, 1, 1.5]
862
+ )
863
+ ax_amp, ax_info = amp_info_row.subplots(ncols=2)
864
+ ax_mean = mean_row.subplots()
865
+ fig_dist_chans, fig_pc = dist_pc_row.subfigures(ncols=2)
866
+ ax_pc = fig_pc.subplots()
867
+ fig_dist, fig_chans = fig_dist_chans.subfigures(nrows=2, height_ratios=[3, 2])
868
+ ax_chans = fig_chans.subplots()
869
+
870
+ # unit dist matrix / pair mask
871
+ if dists is not None:
872
+ dists_inf = dists.copy()
873
+ dists_inf[dists > mix_data.tmm.max_distance] = np.inf
874
+ distance_matrix_dendro(
875
+ panel=fig_dist,
876
+ distances=dists_inf,
877
+ show_values_from=dists,
878
+ show_unit_labels=True,
879
+ vmax=mix_data.tmm.max_distance,
880
+ image_cmap=self.dist_cmap,
881
+ show_values=True,
882
+ label_colors=self.colors,
883
+ with_colorbar=False,
884
+ )
885
+ else:
886
+ ax_dist = fig_dist.subplots()
887
+ ax_dist.axis("off")
888
+ ax_dist.text(0, 0, "no dists", fontsize=5)
889
+
890
+ if chans_by_km is not None:
891
+ low = min(c.min().item() for c in chans_by_km.values())
892
+ hi = max(c.max().item() for c in chans_by_km.values())
893
+ bins = np.arange(low - 0.5, hi + 0.6, step=1.0)
894
+ ax_chans.hist(
895
+ list(chans_by_km.values()),
896
+ stacked=True,
897
+ color=self.colors[list(chans_by_km.keys())],
898
+ bins=bins,
899
+ )
900
+
901
+ # merge res text info
902
+ ax_info.axis("off")
903
+ ax_info.text(
904
+ 0.2,
905
+ 0.5,
906
+ txt,
907
+ fontsize="small",
908
+ va="center",
909
+ transform=ax_info.transAxes,
910
+ )
911
+
912
+ # pc scatter
913
+ if loadings is not None:
914
+ ax_pc.scatter(*loadings.T, c=colors, s=5, lw=0, alpha=0.5)
915
+ ax_pc.grid()
916
+ else:
917
+ ax_pc.axis("off")
918
+ ax_pc.text(0, 0, "no kmeans pca", fontsize=5)
919
+
920
+ # recon mean view
921
+ ax_mean.axis("off")
922
+ if means is not None:
923
+ chans = mean_chans[None].broadcast_to(len(means), *mean_chans.shape)
924
+ geomplot(
925
+ waveforms=means,
926
+ channels=chans.numpy(force=True),
927
+ geom=mix_data.prgeom,
928
+ show_zero=False,
929
+ colors=self.colors[: len(means)],
930
+ zlim=None,
931
+ ax=ax_mean,
932
+ subar=True,
933
+ annotate_z=True,
934
+ )
935
+ else:
936
+ ax_mean.text(0, 0, "no subunit means", fontsize=5)
937
+
938
+ # amp view
939
+ if amps is not None:
940
+ ax_amp.scatter(t, amps, c=colors, s=5, lw=0, alpha=0.5)
941
+ ax_amp.grid()
942
+ ax_amp.set_xlabel("t (s)")
943
+ ax_amp.set_ylabel("feat maxchan norm")
944
+ else:
945
+ ax_amp.axis("off")
946
+ ax_amp.text(0, 0, "no amps", fontsize=5)
947
+ panel.suptitle("split result vis")
948
+
949
+
950
+ def default_mixture_plots():
951
+ return (
952
+ TextInfo(),
953
+ ISIHistogram(),
954
+ ChansHeatmap(),
955
+ LikelihoodsView(),
956
+ NeighborMeans(),
957
+ NeighborDistances(),
958
+ MergeView(),
959
+ MeanView(),
960
+ CovarianceView(),
961
+ MeanView(mini=True),
962
+ SplitView(),
963
+ )
964
+
965
+
966
+ def fit_mixture_and_visualize_all_components(
967
+ *,
968
+ sorting: DARTsortSorting,
969
+ motion_est,
970
+ refinement_cfg: RefinementConfig = default_refinement_cfg,
971
+ computation_cfg: ComputationConfig | None = None,
972
+ save_folder: str | Path,
973
+ plots=default_mixture_plots,
974
+ max_height=9,
975
+ figsize=(17, 10),
976
+ hspace=0.01,
977
+ dpi=200,
978
+ image_ext="png",
979
+ n_jobs=0,
980
+ show_progress=True,
981
+ overwrite=False,
982
+ unit_ids=None,
983
+ n_units=None,
984
+ seed=0,
985
+ **other_global_params,
986
+ ):
987
+ computation_cfg = ensure_computation_config(computation_cfg)
988
+ save_folder = resolve_path(save_folder)
989
+ if unit_ids is None:
990
+ unit_ids = sorting.unit_ids
991
+ if n_units is not None and n_units < len(unit_ids):
992
+ rg = np.random.default_rng(seed)
993
+ unit_ids = rg.choice(unit_ids, size=n_units, replace=False)
994
+ if not overwrite and all_summaries_done(unit_ids, save_folder, ext=image_ext):
995
+ return
996
+ mix_data = fit_mixture_for_vis(
997
+ sorting=sorting,
998
+ motion_est=motion_est,
999
+ refinement_cfg=refinement_cfg,
1000
+ computation_cfg=computation_cfg,
1001
+ )
1002
+ return make_mixture_summaries(
1003
+ mix_data=mix_data,
1004
+ save_folder=save_folder,
1005
+ plots=plots,
1006
+ max_height=max_height,
1007
+ figsize=figsize,
1008
+ hspace=hspace,
1009
+ dpi=dpi,
1010
+ image_ext=image_ext,
1011
+ n_jobs=n_jobs,
1012
+ show_progress=show_progress,
1013
+ overwrite=overwrite,
1014
+ unit_ids=unit_ids,
1015
+ n_units=n_units,
1016
+ seed=seed,
1017
+ **other_global_params,
1018
+ )
1019
+
1020
+
1021
+ def fit_mixture_for_vis(
1022
+ *,
1023
+ sorting: DARTsortSorting,
1024
+ motion_est,
1025
+ refinement_cfg: RefinementConfig = default_refinement_cfg,
1026
+ computation_cfg: ComputationConfig | None = None,
1027
+ em: bool = True,
1028
+ split: bool = False,
1029
+ ) -> MixtureVisData:
1030
+ # run model to convergence and soft assign train/full sets
1031
+ mix_data = instantiate_and_bootstrap_tmm(
1032
+ sorting=sorting,
1033
+ motion_est=motion_est,
1034
+ refinement_cfg=refinement_cfg,
1035
+ computation_cfg=computation_cfg,
1036
+ )
1037
+ if em:
1038
+ mix_data.tmm.em(mix_data.train_data)
1039
+ if split:
1040
+ run_split(mix_data.tmm, mix_data.train_data, mix_data.val_data, prog_level=1)
1041
+ mix_data.tmm.em(mix_data.train_data)
1042
+
1043
+ train_scores = mix_data.tmm.soft_assign(
1044
+ data=mix_data.train_data,
1045
+ needs_bootstrap=False,
1046
+ full_proposal_view=True,
1047
+ )
1048
+ train_labels = labels_from_scores(train_scores)
1049
+ full_scores = mix_data.tmm.soft_assign(
1050
+ data=mix_data.full_data,
1051
+ needs_bootstrap=False,
1052
+ full_proposal_view=True,
1053
+ )
1054
+ full_labels = labels_from_scores(full_scores)
1055
+ if mix_data.val_data is None:
1056
+ eval_scores = train_scores
1057
+ else:
1058
+ eval_scores = mix_data.tmm.soft_assign(
1059
+ data=mix_data.val_data,
1060
+ needs_bootstrap=False,
1061
+ full_proposal_view=True,
1062
+ )
1063
+
1064
+ dists = mix_data.tmm.unit_distance_matrix().cpu().clone()
1065
+ dists.diagonal().fill_(torch.inf)
1066
+
1067
+ tpca = get_tpca(sorting)
1068
+ assert tpca is not None
1069
+ assert isinstance(tpca, TemporalPCA)
1070
+ if isinstance(mix_data.train_ixs, slice):
1071
+ assert mix_data.train_ixs == slice(None)
1072
+ assert train_labels.shape == full_labels.shape
1073
+ train_ixs = np.arange(train_labels.shape[0])
1074
+ else:
1075
+ train_ixs = mix_data.train_ixs.numpy(force=True)
1076
+
1077
+ return MixtureVisData(
1078
+ tmm=mix_data.tmm,
1079
+ train_data=mix_data.train_data,
1080
+ val_data=mix_data.val_data,
1081
+ full_data=mix_data.full_data,
1082
+ sorting=sorting,
1083
+ motion_est=motion_est,
1084
+ train_scores=train_scores,
1085
+ full_scores=full_scores,
1086
+ eval_scores=eval_scores,
1087
+ train_times=sorting.times_seconds[mix_data.train_ixs], # type: ignore
1088
+ train_ixs=train_ixs,
1089
+ train_labels=train_labels,
1090
+ full_labels=full_labels,
1091
+ tpca=tpca,
1092
+ inf_diag_unit_distance_matrix=dists,
1093
+ prgeom=mix_data.tmm.neighb_cov.prgeom.numpy(force=True),
1094
+ )
1095
+
1096
+
1097
+ def make_mixture_component_summary(
1098
+ mix_data: MixtureVisData,
1099
+ unit_id: int,
1100
+ plots=default_mixture_plots,
1101
+ max_height=9,
1102
+ figsize=(17, 10),
1103
+ card_hspace=0.01,
1104
+ figure=None,
1105
+ **other_global_params,
1106
+ ):
1107
+ if callable(plots):
1108
+ plots = plots()
1109
+ plots = cast(Iterable[MixtureComponentPlot], plots)
1110
+ for p in plots:
1111
+ p.notify_global_params(**other_global_params)
1112
+ figure = flow_layout(
1113
+ plots,
1114
+ max_height=max_height,
1115
+ figsize=figsize,
1116
+ card_hspace=card_hspace,
1117
+ figure=figure,
1118
+ mix_data=mix_data,
1119
+ unit_id=unit_id,
1120
+ )
1121
+ return figure
1122
+
1123
+
1124
+ def make_mixture_summaries(
1125
+ mix_data: MixtureVisData,
1126
+ save_folder: str | Path,
1127
+ plots=default_mixture_plots,
1128
+ max_height=9,
1129
+ figsize=(17, 10),
1130
+ card_hspace=0.01,
1131
+ dpi=200,
1132
+ image_ext="png",
1133
+ n_jobs=0,
1134
+ show_progress=True,
1135
+ overwrite=False,
1136
+ unit_ids=None,
1137
+ n_units=None,
1138
+ seed=0,
1139
+ **other_global_params,
1140
+ ):
1141
+ save_folder = resolve_path(save_folder)
1142
+ if unit_ids is None:
1143
+ unit_ids = mix_data.tmm.unit_ids.numpy(force=True).tolist()
1144
+ if n_units is not None and n_units < len(unit_ids):
1145
+ rg = np.random.default_rng(seed)
1146
+ unit_ids = rg.choice(unit_ids, size=n_units, replace=False)
1147
+ if not overwrite and all_summaries_done(unit_ids, save_folder, ext=image_ext):
1148
+ return
1149
+
1150
+ save_folder.mkdir(exist_ok=True, parents=True)
1151
+ global_params = dict(**other_global_params)
1152
+ n_jobs, Executor, context = get_pool(n_jobs, cls=CloudpicklePoolExecutor) # type: ignore
1153
+
1154
+ initargs = (
1155
+ mix_data,
1156
+ plots,
1157
+ max_height,
1158
+ figsize,
1159
+ card_hspace,
1160
+ dpi,
1161
+ save_folder,
1162
+ image_ext,
1163
+ overwrite,
1164
+ global_params,
1165
+ plt.rcParams,
1166
+ )
1167
+ # if ispar and not use_threads:
1168
+ # initargs = (cloudpickle.dumps(initargs),)
1169
+ with Executor(
1170
+ max_workers=n_jobs,
1171
+ mp_context=context,
1172
+ initializer=_summary_init,
1173
+ initargs=initargs,
1174
+ ) as pool:
1175
+ results = pool.map(_summary_job, unit_ids)
1176
+ if show_progress:
1177
+ results = tqdm(
1178
+ results,
1179
+ desc="GMM summaries",
1180
+ smoothing=0,
1181
+ total=len(unit_ids),
1182
+ )
1183
+ for res in results:
1184
+ pass
1185
+
1186
+
1187
+ def all_summaries_done(unit_ids, save_folder, ext="png"):
1188
+ return save_folder.exists() and all(
1189
+ (save_folder / f"unit{unit_id:04d}.{ext}").exists() for unit_id in unit_ids
1190
+ )
1191
+
1192
+
1193
+ class SummaryJobContext:
1194
+ def __init__(
1195
+ self,
1196
+ mix_data,
1197
+ plots,
1198
+ max_height,
1199
+ figsize,
1200
+ card_hspace,
1201
+ dpi,
1202
+ save_folder,
1203
+ image_ext,
1204
+ overwrite,
1205
+ global_params,
1206
+ rc,
1207
+ ):
1208
+ self.mix_data = mix_data
1209
+ self.plots = plots
1210
+ self.max_height = max_height
1211
+ self.figsize = figsize
1212
+ self.card_hspace = card_hspace
1213
+ self.dpi = dpi
1214
+ self.save_folder = save_folder
1215
+ self.image_ext = image_ext
1216
+ self.overwrite = overwrite
1217
+ self.global_params = global_params
1218
+
1219
+
1220
+ _summary_job_context = None
1221
+
1222
+
1223
+ def _summary_init(*args):
1224
+ global _summary_job_context
1225
+ _summary_job_context = SummaryJobContext(*args)
1226
+
1227
+
1228
+ def _summary_job(unit_id):
1229
+ global _summary_job_context
1230
+ assert _summary_job_context is not None
1231
+ tmp_out = None
1232
+ try:
1233
+ ext = _summary_job_context.image_ext
1234
+ tmp_out = _summary_job_context.save_folder / f"tmp_unit{unit_id:04d}.{ext}"
1235
+ final_out = _summary_job_context.save_folder / f"unit{unit_id:04d}.{ext}"
1236
+ if tmp_out.exists():
1237
+ tmp_out.unlink()
1238
+ if not _summary_job_context.overwrite and final_out.exists():
1239
+ return
1240
+ if _summary_job_context.overwrite and final_out.exists():
1241
+ final_out.unlink()
1242
+
1243
+ fig = plt.figure(
1244
+ figsize=_summary_job_context.figsize,
1245
+ layout="constrained",
1246
+ dpi=_summary_job_context.dpi,
1247
+ )
1248
+ make_mixture_component_summary(
1249
+ _summary_job_context.mix_data,
1250
+ unit_id,
1251
+ card_hspace=_summary_job_context.card_hspace,
1252
+ plots=_summary_job_context.plots,
1253
+ max_height=_summary_job_context.max_height,
1254
+ figsize=_summary_job_context.figsize,
1255
+ figure=fig,
1256
+ **_summary_job_context.global_params,
1257
+ )
1258
+
1259
+ # the save is done sort of atomically to help with the resuming and avoid
1260
+ # half-baked image files
1261
+ fig.savefig(tmp_out, dpi=_summary_job_context.dpi)
1262
+ tmp_out.rename(final_out)
1263
+ plt.close(fig)
1264
+ except Exception:
1265
+ import traceback
1266
+
1267
+ print("// error in unit", unit_id)
1268
+ print(traceback.format_exc())
1269
+ raise
1270
+ finally:
1271
+ if tmp_out is not None and tmp_out.exists():
1272
+ tmp_out.unlink()
1273
+
1274
+
1275
+ # -- one-offs
1276
+
1277
+
1278
+ def vis_split_interpolation(
1279
+ mix_data: MixtureVisData,
1280
+ unit_id: int,
1281
+ erp: NeighborhoodFiller | None = None,
1282
+ whiten: bool = True,
1283
+ figscale=5,
1284
+ n_per_group=8,
1285
+ seed=0,
1286
+ layout="horz",
1287
+ overlay=True,
1288
+ ):
1289
+ if erp is None:
1290
+ erp = mix_data.tmm.erp
1291
+
1292
+ split_data = mix_data.train_data.dense_slice_by_unit(
1293
+ unit_id, gen=mix_data.tmm.rg, min_count=2 * mix_data.tmm.min_count
1294
+ )
1295
+ assert split_data is not None
1296
+
1297
+ kmeans_responsibilities, kmeans_x, kmeans_chans = try_kmeans(
1298
+ data=split_data,
1299
+ k=mix_data.tmm.split_k,
1300
+ erp=erp,
1301
+ gen=mix_data.tmm.rg,
1302
+ feature_rank=mix_data.tmm.noise.rank,
1303
+ min_count=mix_data.tmm.min_count,
1304
+ min_channel_count=mix_data.tmm.min_channel_count,
1305
+ debug=True,
1306
+ whiten=whiten,
1307
+ )
1308
+
1309
+ n_spikes = split_data.x.shape[0]
1310
+ assert kmeans_x is not None
1311
+ assert kmeans_x.shape[0] == n_spikes
1312
+ assert kmeans_chans is not None
1313
+ if kmeans_responsibilities is not None:
1314
+ assert kmeans_responsibilities.shape[0] == n_spikes
1315
+ kmeans_labels = kmeans_responsibilities.argmax(dim=1)
1316
+ kmeans_labels = kmeans_labels.cpu()
1317
+ else:
1318
+ kmeans_labels = torch.zeros(n_spikes, dtype=torch.long)
1319
+
1320
+ vis_ix = []
1321
+ rg = np.random.default_rng(seed)
1322
+ ulabels = kmeans_labels.unique()
1323
+ for l in ulabels:
1324
+ (in_l,) = (kmeans_labels == l).nonzero(as_tuple=True)
1325
+ in_l = in_l.numpy(force=True)
1326
+ if in_l.size > n_per_group:
1327
+ in_l = rg.choice(in_l, size=n_per_group, replace=False)
1328
+ vis_ix.append(in_l)
1329
+
1330
+ # interpolated waveforms
1331
+ interp_wfs = []
1332
+ for vix in vis_ix:
1333
+ f = kmeans_x[vix].reshape(len(vix), -1)
1334
+ f = mix_data.reconstruct_flat(f)
1335
+ interp_wfs.append(f)
1336
+
1337
+ # original observed waveforms
1338
+ orig_wfs = []
1339
+ orig_chans = []
1340
+ for vix in vis_ix:
1341
+ f = (split_data.whitenedx if whiten else split_data.x)[vix]
1342
+ f = mix_data.reconstruct_flat(f)
1343
+ orig_wfs.append(f)
1344
+ orig_chans.append(
1345
+ split_data.neighborhoods.b.neighborhoods[split_data.neighborhood_ids[vix]]
1346
+ )
1347
+
1348
+ # -- draw
1349
+
1350
+ if layout == "horz":
1351
+ fig, axes = plt.subplots(
1352
+ nrows=2,
1353
+ ncols=ulabels.numel(),
1354
+ figsize=(figscale * ulabels.numel(), figscale * 2),
1355
+ layout="constrained",
1356
+ )
1357
+ axes = axes.T
1358
+ else:
1359
+ fig, axes = plt.subplots(
1360
+ ncols=2,
1361
+ nrows=ulabels.numel(),
1362
+ figsize=(figscale * 2, figscale * ulabels.numel()),
1363
+ layout="constrained",
1364
+ )
1365
+ maa = max([np.abs(w).max() for w in orig_wfs])
1366
+
1367
+ for col, iwf, owf, ochans, c in zip(axes, interp_wfs, orig_wfs, orig_chans, "rgb"):
1368
+ print(f"{np.abs(iwf).max()=} {np.abs(owf).max()=}")
1369
+ geomplot(
1370
+ waveforms=iwf,
1371
+ channels=kmeans_chans[None].broadcast_to(iwf.shape[0], *kmeans_chans.shape),
1372
+ geom=mix_data.prgeom,
1373
+ show_zero=False,
1374
+ subar=False,
1375
+ ax=col[1],
1376
+ max_abs_amp=maa,
1377
+ linewidth=1,
1378
+ zlim=None,
1379
+ )
1380
+ for ax in col[: 1 + overlay]:
1381
+ geomplot(
1382
+ waveforms=owf,
1383
+ channels=ochans,
1384
+ geom=mix_data.prgeom,
1385
+ show_zero=True,
1386
+ subar=True,
1387
+ ax=ax,
1388
+ max_abs_amp=maa,
1389
+ color="k",
1390
+ linewidth=1,
1391
+ zlim=None,
1392
+ )
1393
+ for ax in axes.flat:
1394
+ ax.axis("off")
1395
+
1396
+ return fig
1397
+
1398
+
1399
+ def vis_obs_interpolation(
1400
+ mix_data: MixtureVisData,
1401
+ unit_id: int,
1402
+ count: int = 256,
1403
+ erp_params: dict[str, InterpolationParams] | None = None,
1404
+ figscale=5.0,
1405
+ disp_colormap="viridis",
1406
+ time_colormap="plasma",
1407
+ ):
1408
+ (trix,) = (mix_data.train_labels == unit_id).nonzero()
1409
+ if trix.size > count:
1410
+ trix = np.random.default_rng(0).choice(trix, size=count, replace=False)
1411
+ trix.sort()
1412
+ fix = mix_data.train_ixs[trix]
1413
+ n = len(trix)
1414
+
1415
+ t_s = mix_data.train_times[trix]
1416
+ z = mix_data.sorting.slice_feature_by_name("point_source_localizations", fix)[:, 2]
1417
+ if mix_data.motion_est is None:
1418
+ disp = np.zeros_like(z)
1419
+ else:
1420
+ disp = mix_data.motion_est.disp_at_s(t_s, z)
1421
+
1422
+ dev = mix_data.tmm.b.means.device
1423
+ sgeom = pad_geom(mix_data.sorting.geom, device=dev) # type: ignore
1424
+ tgeom = torch.asarray(mix_data.prgeom).to(sgeom)
1425
+ channel_index = torch.asarray(mix_data.sorting.channel_index, device=dev) # type: ignore
1426
+ erps = {}
1427
+ for k, ip in (erp_params or {}).items():
1428
+ erps[k] = StableFeaturesInterpolator(
1429
+ source_geom=sgeom,
1430
+ target_geom=tgeom,
1431
+ channel_index=channel_index,
1432
+ params=ip,
1433
+ )
1434
+
1435
+ features = {"actual": mix_data.train_data.x[trix]}
1436
+ origf = mix_data.sorting.slice_feature_by_name(
1437
+ "collisioncleaned_tpca_features", fix
1438
+ )
1439
+ origf = torch.asarray(origf).to(sgeom)
1440
+ chans = torch.asarray(mix_data.sorting.channels[fix]).to(channel_index)
1441
+ shifts = torch.asarray(-disp).to(sgeom)
1442
+ tchans = mix_data.tmm.neighb_cov.obs_ix[mix_data.train_data.neighborhood_ids[trix]]
1443
+ for name, erp in (erps or {}).items():
1444
+ features[name] = erp.interp(
1445
+ features=origf,
1446
+ source_main_channels=chans,
1447
+ target_channels=tchans,
1448
+ source_shifts=shifts,
1449
+ )
1450
+
1451
+ waveforms = {
1452
+ k: mix_data.reconstruct_flat(f.view(n, -1)) for k, f in features.items()
1453
+ }
1454
+
1455
+ maa = origf.abs().nan_to_num_().max().cpu().item()
1456
+ dcolors = plt.get_cmap(disp_colormap)(spiketorch.minmax(disp))
1457
+ tcolors = plt.get_cmap(time_colormap)(spiketorch.minmax(t_s))
1458
+
1459
+ ncols = 2
1460
+ nrows = len(features)
1461
+ fig = plt.figure(
1462
+ figsize=(ncols * figscale, (nrows + 0.5) * figscale),
1463
+ layout="constrained",
1464
+ )
1465
+ panels = fig.subfigures(
1466
+ nrows=nrows + 1,
1467
+ height_ratios=[1] + ([2] * nrows),
1468
+ )
1469
+
1470
+ ax_top = panels[0].subplots()
1471
+ ax_top.scatter(t_s, disp, c=dcolors, s=3, lw=0)
1472
+
1473
+ for row, (name, wf) in zip(panels[1:], waveforms.items()):
1474
+ row_axs = row.subplots(ncols=2)
1475
+ row.suptitle(name, fontsize="small")
1476
+ for ax, cs in zip(row_axs, (dcolors, tcolors)):
1477
+ geomplot(
1478
+ waveforms=wf,
1479
+ channels=tchans.numpy(force=True),
1480
+ geom=mix_data.prgeom,
1481
+ colors=cs,
1482
+ subar=True,
1483
+ ax=ax,
1484
+ max_abs_amp=maa,
1485
+ lw=0.5,
1486
+ )
1487
+ ax.axis("off")
1488
+
1489
+ return fig