DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/vis/unit.py ADDED
@@ -0,0 +1,917 @@
1
+ """Toolkit for extensible single unit summary plots
2
+
3
+ The goal is to make it easy to add and remove plots without the plotting
4
+ code turning into a web of if statements, for loops, and bizarre subplot
5
+ and subfigure mazes.
6
+
7
+ Relies on the DARTsortAnalysis object of utils/analysis.py to do most of
8
+ the data work so that this file can focus on plotting (sort of MVC).
9
+ """
10
+
11
+ from dataclasses import replace
12
+ from pathlib import Path
13
+
14
+ from tqdm.auto import tqdm
15
+
16
+ import matplotlib.pyplot as plt
17
+ import numpy as np
18
+ from matplotlib.legend_handler import HandlerTuple
19
+
20
+ from ..clustering import split
21
+ from ..util.internal_config import raw_template_cfg
22
+ from ..util.job_util import get_global_computation_config
23
+ from ..evaluate.analysis import DARTsortAnalysis, WaveformsBag
24
+ from ..util.multiprocessing_util import CloudpicklePoolExecutor, get_pool, cloudpickle
25
+ from . import layout
26
+ from .analysis_plots import isi_hist, correlogram, plot_correlogram, bar
27
+ from .colors import glasbey1024
28
+ from .waveforms import geomplot, geomplot_templates
29
+
30
+ # -- main class. see fn make_unit_summary below to make lots of UnitPlots.
31
+
32
+
33
+ class UnitPlot(layout.BasePlot):
34
+ can_sharey = True
35
+
36
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
37
+ raise NotImplementedError
38
+
39
+
40
+ class UnitMultiPlot(layout.BaseMultiPlot):
41
+ def plots(self, sorting_analysis: DARTsortAnalysis, unit_id: int):
42
+ # return [UnitPlot()]
43
+ raise NotImplementedError
44
+
45
+
46
+ class UnitTextInfo(UnitPlot):
47
+ kind = "text"
48
+ height = 0.5
49
+
50
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
51
+ axis = panel.subplots()
52
+ axis.axis("off")
53
+ msg = f"unit {unit_id}\n"
54
+
55
+ h5_path = sorting_analysis.sorting.parent_h5_path
56
+ if h5_path:
57
+ msg += f"feature source: {h5_path.name}\n"
58
+
59
+ nspikes = (sorting_analysis.sorting.labels == unit_id).sum()
60
+ msg += f"n spikes: {nspikes}\n"
61
+
62
+ assert sorting_analysis.template_data is not None
63
+ temps = sorting_analysis.template_data.unit_templates(unit_id)
64
+ if not temps.size:
65
+ msg += "no template (too few spikes)"
66
+ elif temps.shape[0] == 1:
67
+ ptp = np.ptp(temps, 1).max(1)[0]
68
+ msg += f"maxptp: {ptp:0.2f} su\n"
69
+ snr = ptp * np.sqrt(nspikes)
70
+ msg += f"template snr: {snr:.1f}"
71
+ else:
72
+ ptp = np.ptp(temps, 1).max(1).mean()
73
+ msg += f"mean superres maxptp: {ptp:0.1f}su\n"
74
+ in_unit = sorting_analysis.template_data.unit_mask(unit_id)
75
+ counts = sorting_analysis.template_data.spike_counts[in_unit]
76
+ snrs = np.ptp(temps, 1).max(1) * np.sqrt(counts)
77
+ msg += "template snrs:\n " + ", ".join(f"{s:0.1f}" for s in snrs)
78
+
79
+ axis.text(0, 0, msg, fontsize=6.5)
80
+
81
+
82
+ # -- small summary plots
83
+
84
+
85
+ class ACG(UnitPlot):
86
+ kind = "histogram"
87
+ height = 0.75
88
+
89
+ def __init__(self, max_lag=50):
90
+ super().__init__()
91
+ self.max_lag = max_lag
92
+
93
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
94
+ axis = panel.subplots()
95
+ which = sorting_analysis.in_unit(unit_id)
96
+ times_samples = sorting_analysis.sorting.times_samples[which]
97
+ plot_correlogram(axis, times_samples, max_lag=self.max_lag)
98
+ axis.set_ylabel("acg")
99
+
100
+
101
+ class ISIHistogram(UnitPlot):
102
+ kind = "histogram"
103
+ height = 0.75
104
+
105
+ def __init__(self, bin_ms=0.1, max_ms=5):
106
+ super().__init__()
107
+ self.bin_ms = bin_ms
108
+ self.max_ms = max_ms
109
+
110
+ def draw(
111
+ self,
112
+ panel,
113
+ sorting_analysis: DARTsortAnalysis,
114
+ unit_id,
115
+ axis=None,
116
+ color="k",
117
+ label=None,
118
+ ):
119
+ if axis is None:
120
+ axis = panel.subplots()
121
+ which = sorting_analysis.in_unit(unit_id)
122
+ assert sorting_analysis.times_seconds is not None
123
+ times_s = sorting_analysis.times_seconds[which]
124
+ isi_hist(
125
+ times_s,
126
+ axis,
127
+ bin_ms=self.bin_ms,
128
+ max_ms=self.max_ms,
129
+ color=color,
130
+ label=label,
131
+ )
132
+
133
+
134
+ class XZScatter(UnitPlot):
135
+ kind = "scatter"
136
+
137
+ def __init__(
138
+ self,
139
+ registered=True,
140
+ amplitude_color_cutoff=15,
141
+ probe_margin_um=100,
142
+ colorbar=False,
143
+ ):
144
+ super().__init__()
145
+ self.registered = registered
146
+ self.amplitude_color_cutoff = amplitude_color_cutoff
147
+ self.probe_margin_um = probe_margin_um
148
+ self.colorbar = colorbar
149
+
150
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id, axis=None):
151
+ if axis is None:
152
+ axis = panel.subplots()
153
+
154
+ in_unit = sorting_analysis.in_unit(unit_id)
155
+ assert sorting_analysis.x is not None
156
+ assert sorting_analysis.amplitudes is not None
157
+ x = sorting_analysis.x[in_unit]
158
+ if self.registered:
159
+ assert sorting_analysis.registered_z is not None
160
+ z = sorting_analysis.registered_z[in_unit]
161
+ else:
162
+ assert sorting_analysis.z is not None
163
+ z = sorting_analysis.z[in_unit]
164
+ geomx, geomz = sorting_analysis.geom.T
165
+ pad = self.probe_margin_um
166
+ valid = x == np.clip(x, geomx.min() - pad, geomx.max() + pad)
167
+ valid &= z == np.clip(z, geomz.min() - pad, geomz.max() + pad)
168
+ amps = sorting_analysis.amplitudes[in_unit]
169
+ c = dict(c=np.minimum(amps, self.amplitude_color_cutoff))
170
+ s = axis.scatter(
171
+ x[valid],
172
+ z[valid],
173
+ lw=0,
174
+ s=3,
175
+ **c,
176
+ rasterized=True,
177
+ )
178
+ axis.set_xlabel("x (um)")
179
+ reg_str = "reg " * self.registered
180
+ axis.set_ylabel(reg_str + "z (um)")
181
+ if self.colorbar:
182
+ plt.colorbar(s, ax=axis, shrink=0.5, label="amp (su)")
183
+
184
+
185
+ class PCAScatter(UnitPlot):
186
+ kind = "scatter"
187
+
188
+ def __init__(
189
+ self,
190
+ amplitude_color_cutoff=15,
191
+ pca_radius_um=75.0,
192
+ colorbar=False,
193
+ ):
194
+ super().__init__()
195
+ self.amplitude_color_cutoff = amplitude_color_cutoff
196
+ self.colorbar = colorbar
197
+ self.pca_radius_um = pca_radius_um
198
+
199
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id, axis=None):
200
+ if axis is None:
201
+ axis = panel.subplots()
202
+
203
+ which, loadings = sorting_analysis.unit_pca_features(unit_id=unit_id)
204
+ if which is None:
205
+ return
206
+ assert loadings is not None
207
+ assert sorting_analysis.amplitudes is not None
208
+ amps = sorting_analysis.amplitudes[which]
209
+ c = dict(c=np.minimum(amps, self.amplitude_color_cutoff))
210
+ s = axis.scatter(
211
+ *loadings.T,
212
+ lw=0,
213
+ s=3,
214
+ rasterized=True,
215
+ **c,
216
+ )
217
+ axis.set_xlabel("PC1")
218
+ axis.set_ylabel("PC2")
219
+ if self.colorbar:
220
+ plt.colorbar(s, ax=axis, shrink=0.5, label="amp (su)")
221
+
222
+ return axis
223
+
224
+
225
+ # -- wide scatter plots
226
+
227
+
228
+ class TimeFeatScatter(UnitPlot):
229
+ kind = "widescatter"
230
+ width = 2
231
+
232
+ def __init__(
233
+ self,
234
+ feat_name,
235
+ color_by_amplitude=True,
236
+ amplitude_color_cutoff=15,
237
+ alpha=1.0,
238
+ label=None,
239
+ ):
240
+ super().__init__()
241
+ self.feat_name = feat_name
242
+ self.amplitude_color_cutoff = amplitude_color_cutoff
243
+ self.color_by_amplitude = color_by_amplitude
244
+ self.alpha = alpha
245
+ self.label = label or feat_name
246
+
247
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
248
+ axis = panel.subplots()
249
+ assert sorting_analysis.times_seconds is not None
250
+ assert sorting_analysis.amplitudes is not None
251
+ in_unit = sorting_analysis.in_unit(unit_id)
252
+ t = sorting_analysis.times_seconds[in_unit]
253
+ feat = sorting_analysis.named_feature(self.feat_name, which=in_unit)
254
+ c = None
255
+ if self.color_by_amplitude:
256
+ amps = sorting_analysis.amplitudes[in_unit]
257
+ c = np.minimum(amps, self.amplitude_color_cutoff)
258
+ s = axis.scatter(t, feat, c=c, lw=0, s=3, alpha=self.alpha, rasterized=True)
259
+ axis.set_xlabel("time (s)")
260
+ axis.set_ylabel(self.label)
261
+ if self.color_by_amplitude:
262
+ plt.colorbar(s, ax=axis, shrink=0.5, label="amp (su)")
263
+
264
+
265
+ class TimeZScatter(TimeFeatScatter):
266
+ def __init__(self, **kwargs):
267
+ super().__init__(feat_name="z", label="z (um)", **kwargs)
268
+
269
+
270
+ class TimeRegZScatter(TimeFeatScatter):
271
+ def __init__(self, **kwargs):
272
+ super().__init__(feat_name="registered_z", label="reg. z (um)", **kwargs)
273
+
274
+
275
+ class TimeAmpScatter(TimeFeatScatter):
276
+ def __init__(self, **kwargs):
277
+ super().__init__(feat_name="amplitudes", label="amp (su)", **kwargs)
278
+
279
+
280
+ # -- waveform plots
281
+
282
+
283
+ class WaveformPlot(UnitPlot):
284
+ kind = "waveform"
285
+ width = 3
286
+ height = 2
287
+ can_sharey = False
288
+
289
+ # for my subclasses
290
+ wfs_kind = ""
291
+
292
+ def __init__(
293
+ self,
294
+ count=100,
295
+ color="k",
296
+ alpha=0.1,
297
+ show_template=True,
298
+ template_color="orange",
299
+ max_abs_template_scale=1.5,
300
+ legend=True,
301
+ title=None,
302
+ ):
303
+ super().__init__()
304
+ self.count = count
305
+ self.color = color
306
+ self.alpha = alpha
307
+ self.show_template = show_template
308
+ self.template_color = template_color
309
+ self.legend = legend
310
+ self.max_abs_template_scale = max_abs_template_scale
311
+ self.title = title
312
+
313
+ def get_waveforms(
314
+ self, sorting_analysis: DARTsortAnalysis, unit_id: int
315
+ ) -> WaveformsBag | None:
316
+ raise NotImplementedError
317
+
318
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id, axis=None):
319
+ if axis is None:
320
+ axis = panel.subplots()
321
+
322
+ waves = self.get_waveforms(sorting_analysis, unit_id)
323
+ tslice = None if waves is None else waves.temporal_slice
324
+ trough_offset_samples = sorting_analysis.trough_offset_samples
325
+ spike_length_samples = sorting_analysis.spike_length_samples
326
+ if tslice is not None and tslice.start is not None:
327
+ trough_offset_samples = (
328
+ sorting_analysis.trough_offset_samples - tslice.start
329
+ )
330
+ spike_length_samples = sorting_analysis.spike_length_samples - tslice.start
331
+ if tslice is not None and tslice.stop is not None:
332
+ spike_length_samples = tslice.stop - tslice.start
333
+
334
+ max_abs_amp = None
335
+ show_template = self.show_template
336
+ template_color = self.template_color
337
+ assert sorting_analysis.coarse_template_data is not None
338
+ if show_template:
339
+ templates = sorting_analysis.coarse_template_data.unit_templates(unit_id)
340
+ show_template = bool(templates.size)
341
+ else:
342
+ templates = None
343
+
344
+ if show_template:
345
+ assert templates is not None
346
+ templates = trim_waveforms(
347
+ templates,
348
+ old_offset=sorting_analysis.coarse_template_data.trough_offset_samples,
349
+ new_offset=trough_offset_samples,
350
+ new_length=spike_length_samples,
351
+ )
352
+ max_abs_amp = self.max_abs_template_scale * np.nanmax(np.abs(templates))
353
+
354
+ handles = {}
355
+ if waves is not None:
356
+ if np.isfinite(waves.waveforms[:, 0, :]).any():
357
+ max_abs_amp = self.max_abs_template_scale * np.nanpercentile(
358
+ np.abs(waves.waveforms), 99
359
+ )
360
+ ls = geomplot(
361
+ waves.waveforms,
362
+ max_channels=np.full(len(waves.waveforms), waves.main_channel),
363
+ channel_index=waves.channel_index,
364
+ geom=waves.geom,
365
+ ax=axis,
366
+ show_zero=False,
367
+ subar=True,
368
+ msbar=False,
369
+ zlim="tight",
370
+ color=self.color,
371
+ alpha=self.alpha,
372
+ max_abs_amp=max_abs_amp,
373
+ trough_offset=trough_offset_samples,
374
+ lw=1,
375
+ )
376
+ handles["waveforms"] = ls
377
+
378
+ if show_template:
379
+ assert templates is not None
380
+ if waves is None:
381
+ showchans = sorting_analysis.vis_channel_index[
382
+ sorting_analysis.unit_max_channel(unit_id)
383
+ ]
384
+ geom = sorting_analysis.registered_geom
385
+ else:
386
+ showchans = waves.channel_index[waves.main_channel]
387
+ geom = waves.geom
388
+ showchans = showchans[showchans < len(geom)]
389
+ ls = geomplot(
390
+ templates[:, :, showchans],
391
+ geom=geom[showchans],
392
+ ax=axis,
393
+ show_zero=False,
394
+ zlim="tight",
395
+ color=template_color,
396
+ alpha=1,
397
+ max_abs_amp=max_abs_amp,
398
+ lw=1,
399
+ )
400
+ handles["mean"] = ls
401
+
402
+ shift_str = "shifted " * sorting_analysis.shifting
403
+ if self.title is None:
404
+ axis.set_title(shift_str + self.wfs_kind)
405
+ else:
406
+ axis.set_title(self.title)
407
+ axis.set_xticks([])
408
+ axis.set_yticks([])
409
+
410
+ if self.legend:
411
+ axis.legend(
412
+ handles.values(),
413
+ handles.keys(),
414
+ handler_map={tuple: HandlerTuple(ndivide=None)},
415
+ fancybox=False,
416
+ loc="upper left",
417
+ )
418
+
419
+
420
+ class RawWaveformPlot(WaveformPlot):
421
+ wfs_kind = "raw wfs"
422
+
423
+ def get_waveforms(
424
+ self, sorting_analysis: DARTsortAnalysis, unit_id: int
425
+ ) -> WaveformsBag | None:
426
+ return sorting_analysis.unit_raw_waveforms(unit_id, max_count=self.count)
427
+
428
+
429
+ class TPCAWaveformPlot(WaveformPlot):
430
+ wfs_kind = "coll.-cl. tpca wfs"
431
+
432
+ def get_waveforms(
433
+ self, sorting_analysis: DARTsortAnalysis, unit_id: int
434
+ ) -> WaveformsBag | None:
435
+ return sorting_analysis.unit_tpca_waveforms(unit_id, max_count=self.count)
436
+
437
+
438
+ # -- merge-focused plots
439
+
440
+
441
+ class NearbyCoarseTemplatesPlot(UnitPlot):
442
+ title = "nearby coarse templates"
443
+ kind = "neighbors"
444
+ width = 3
445
+ height = 2
446
+ can_sharey = False
447
+
448
+ def __init__(self, n_neighbors=5, legend=True):
449
+ super().__init__()
450
+ self.n_neighbors = n_neighbors
451
+ self.legend = legend
452
+
453
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id, axis=None):
454
+ if axis is None:
455
+ axis = panel.subplots()
456
+ if np.asarray(unit_id).size > 1:
457
+ unit_id = unit_id[0]
458
+ (
459
+ neighbor_ixs,
460
+ neighbor_ids,
461
+ neighbor_dists,
462
+ neighbor_coarse_templates,
463
+ ) = sorting_analysis.nearby_coarse_templates(
464
+ unit_id, n_neighbors=self.n_neighbors
465
+ )
466
+ if not np.equal(neighbor_ids, unit_id).any():
467
+ axis.axis("off")
468
+ return
469
+ assert neighbor_ids[0] == unit_id
470
+ geomplot_templates(
471
+ axis,
472
+ neighbor_ids,
473
+ neighbor_coarse_templates,
474
+ sorting_analysis.vis_channel_index,
475
+ sorting_analysis.registered_geom,
476
+ main_channel=sorting_analysis.unit_max_channel(unit_id),
477
+ title="",
478
+ )
479
+
480
+
481
+ class CoarseTemplateDistancePlot(UnitPlot):
482
+ title = "coarse template distance"
483
+ kind = "neighbors"
484
+ width = 3
485
+ height = 1.25
486
+
487
+ def __init__(
488
+ self,
489
+ channel_show_radius_um=50,
490
+ n_neighbors=5,
491
+ dist_vmax=1.0,
492
+ show_values=True,
493
+ ):
494
+ super().__init__()
495
+ self.channel_show_radius_um = channel_show_radius_um
496
+ self.n_neighbors = n_neighbors
497
+ self.dist_vmax = dist_vmax
498
+ self.show_values = show_values
499
+
500
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id, axis=None):
501
+ if np.asarray(unit_id).size > 1:
502
+ unit_id = unit_id[0]
503
+ if axis is None:
504
+ axis = panel.subplots()
505
+ (
506
+ neighbor_ixs,
507
+ neighbor_ids,
508
+ neighbor_dists,
509
+ neighbor_coarse_templates,
510
+ ) = sorting_analysis.nearby_coarse_templates(
511
+ unit_id, n_neighbors=self.n_neighbors
512
+ )
513
+ colors = np.array(glasbey1024)[neighbor_ids % len(glasbey1024)]
514
+ if not np.equal(neighbor_ids, unit_id).any():
515
+ axis.axis("off")
516
+ return
517
+ assert neighbor_ids[0] == unit_id
518
+
519
+ im = axis.imshow(
520
+ neighbor_dists,
521
+ vmin=0,
522
+ vmax=self.dist_vmax,
523
+ cmap="RdGy",
524
+ origin="lower",
525
+ interpolation="none",
526
+ )
527
+ if self.show_values:
528
+ for (j, i), label in np.ndenumerate(neighbor_dists):
529
+ axis.text(i, j, f"{label:.2f}", ha="center", va="center")
530
+ plt.colorbar(im, ax=axis, shrink=0.3)
531
+ axis.set_xticks(range(len(neighbor_ids)), neighbor_ids)
532
+ axis.set_yticks(range(len(neighbor_ids)), neighbor_ids)
533
+ for i, (tx, ty) in enumerate(
534
+ zip(axis.xaxis.get_ticklabels(), axis.yaxis.get_ticklabels())
535
+ ):
536
+ tx.set_color(colors[i])
537
+ ty.set_color(colors[i])
538
+ axis.set_title(self.title)
539
+
540
+
541
+ class NeighborCCGPlot(UnitPlot):
542
+ kind = "neighbors"
543
+ width = 3
544
+ height = 0.75
545
+
546
+ def __init__(self, n_neighbors=3, max_lag=50):
547
+ super().__init__()
548
+ self.n_neighbors = n_neighbors
549
+ self.max_lag = max_lag
550
+
551
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
552
+ (
553
+ neighbor_ixs,
554
+ neighbor_ids,
555
+ neighbor_dists,
556
+ neighbor_coarse_templates,
557
+ ) = sorting_analysis.nearby_coarse_templates(
558
+ unit_id, n_neighbors=self.n_neighbors + 1
559
+ )
560
+ # assert neighbor_ids[0] == unit_id
561
+ neighbor_ids = neighbor_ids[1:]
562
+ colors = np.array(glasbey1024)[neighbor_ids % len(glasbey1024)]
563
+
564
+ my_st = sorting_analysis.sorting.times_samples[
565
+ sorting_analysis.in_unit(unit_id)
566
+ ]
567
+ neighb_sts = [
568
+ sorting_analysis.sorting.times_samples[sorting_analysis.in_unit(nid)]
569
+ for nid in neighbor_ids
570
+ ]
571
+
572
+ axes = panel.subplots(
573
+ nrows=2, sharey="row", sharex=True, squeeze=False, ncols=len(neighb_sts)
574
+ )
575
+ for j in range(len(neighb_sts)):
576
+ clags, ccg = correlogram(my_st, neighb_sts[j], max_lag=self.max_lag)
577
+ merged_st = np.concatenate((my_st, neighb_sts[j]))
578
+ merged_st.sort()
579
+ alags, acg = correlogram(merged_st, max_lag=self.max_lag)
580
+
581
+ bar(axes[0, j], clags, ccg, fill=True, fc=colors[j]) # , ec="k", lw=1)
582
+ bar(axes[1, j], alags, acg, fill=True, fc=colors[j]) # , ec="k", lw=1)
583
+ axes[0, j].set_title(f"unit {neighbor_ids[j]}")
584
+ axes[0, 0].set_ylabel("ccg")
585
+ axes[1, 0].set_ylabel("merged acg")
586
+ axes[1, len(neighb_sts) // 2].set_xlabel("lag (samples)")
587
+
588
+
589
+ class NeighborQDAPlot(UnitPlot):
590
+ kind = "neighbors"
591
+ width = 3
592
+ height = 2
593
+
594
+ def __init__(self, n_neighbors=6, log=True):
595
+ super().__init__()
596
+ self.n_neighbors = n_neighbors
597
+ self.log = log
598
+
599
+ def draw(self, panel, sorting_analysis: DARTsortAnalysis, unit_id: int):
600
+ (
601
+ neighbor_ixs,
602
+ neighbor_ids,
603
+ neighbor_dists,
604
+ neighbor_coarse_templates,
605
+ ) = sorting_analysis.nearby_coarse_templates(
606
+ unit_id, n_neighbors=self.n_neighbors + 1
607
+ )
608
+ assert sorting_analysis.sorting.labels is not None
609
+ # assert neighbor_ids[0] == unit_id
610
+ neighbor_ids = neighbor_ids[1:]
611
+ colors = np.array(glasbey1024)[neighbor_ids % len(glasbey1024)]
612
+
613
+ axes = panel.subplots(
614
+ nrows=2,
615
+ sharey="row",
616
+ sharex=False,
617
+ squeeze=False,
618
+ ncols=len(neighbor_ids) // 2,
619
+ )
620
+ for ax, nid, color in zip(axes.flat, neighbor_ids, colors):
621
+ in_pair = np.flatnonzero(
622
+ np.isin(sorting_analysis.sorting.labels, [unit_id, nid])
623
+ )
624
+ cand = sorting_analysis.sorting.candidates[in_pair] # type: ignore
625
+ ll = sorting_analysis.sorting.log_liks[in_pair] # type: ignore
626
+
627
+ my_mask = cand == unit_id
628
+ nid_mask = cand == nid
629
+
630
+ overlap = np.logical_and(nid_mask.any(1), my_mask.any(1))
631
+ count = overlap.sum()
632
+ iou = count / in_pair.shape[0]
633
+ ax.set_title(f"{nid}, SAiou: {iou.item():.2f}", fontsize="small")
634
+ if not count:
635
+ continue
636
+
637
+ _, my_ix = np.nonzero(my_mask[overlap])
638
+ my_ll = np.take_along_axis(ll[overlap], my_ix[:, None], axis=1)[:, 0]
639
+
640
+ _, their_ix = np.nonzero(nid_mask[overlap])
641
+ their_ll = np.take_along_axis(ll[overlap], their_ix[:, None], axis=1)[:, 0]
642
+
643
+ ax.hist(
644
+ my_ll - their_ll,
645
+ bins=32,
646
+ color=color,
647
+ histtype="step",
648
+ log=self.log,
649
+ density=True,
650
+ )
651
+ ax.axvline(0, color="k", lw=0.8)
652
+ ax.grid()
653
+
654
+ axes[0, 0].set_ylabel("density")
655
+ axes[1, 0].set_ylabel("density")
656
+ axes[1, axes.shape[1] // 2].set_xlabel("my ll - their ll")
657
+
658
+
659
+ # -- main routines
660
+
661
+ default_plots = (
662
+ UnitTextInfo(),
663
+ ACG(),
664
+ ISIHistogram(),
665
+ XZScatter(),
666
+ PCAScatter(),
667
+ TimeZScatter(),
668
+ TimeRegZScatter(),
669
+ TimeAmpScatter(),
670
+ RawWaveformPlot(),
671
+ TPCAWaveformPlot(),
672
+ NearbyCoarseTemplatesPlot(),
673
+ CoarseTemplateDistancePlot(),
674
+ NeighborCCGPlot(),
675
+ )
676
+
677
+
678
+ template_assignment_plots = (
679
+ UnitTextInfo(),
680
+ RawWaveformPlot(),
681
+ )
682
+
683
+
684
+ def make_unit_summary(
685
+ sorting_analysis: DARTsortAnalysis,
686
+ unit_id,
687
+ amplitude_color_cutoff=15.0,
688
+ pca_radius_um=75.0,
689
+ plots=default_plots,
690
+ max_height=4,
691
+ figsize=(16, 8.5),
692
+ figure=None,
693
+ gizmo_name="sorting_analysis",
694
+ **other_global_params,
695
+ ):
696
+ # notify plots of global params
697
+ for p in plots:
698
+ p.notify_global_params(
699
+ amplitude_color_cutoff=amplitude_color_cutoff,
700
+ pca_radius_um=pca_radius_um,
701
+ **other_global_params,
702
+ )
703
+
704
+ figure = layout.flow_layout(
705
+ plots,
706
+ max_height=max_height,
707
+ figsize=figsize,
708
+ figure=figure,
709
+ unit_id=unit_id,
710
+ **{gizmo_name: sorting_analysis}, # type: ignore
711
+ )
712
+
713
+ return figure
714
+
715
+
716
+ def make_all_summaries(
717
+ sorting_analysis: DARTsortAnalysis,
718
+ save_folder,
719
+ plots=default_plots,
720
+ amplitude_color_cutoff=15.0,
721
+ pca_radius_um=75.0,
722
+ max_height=4,
723
+ figsize=(16, 8.5),
724
+ dpi=200,
725
+ image_ext="png",
726
+ n_jobs=None,
727
+ show_progress=True,
728
+ namebyamp=False,
729
+ overwrite=False,
730
+ unit_ids=None,
731
+ gizmo_name="sorting_analysis",
732
+ n_units=None,
733
+ seed=0,
734
+ taskname="summaries",
735
+ **other_global_params,
736
+ ):
737
+ save_folder = Path(save_folder)
738
+ if unit_ids is None:
739
+ unit_ids = sorting_analysis.sorting.unit_ids
740
+ if n_units is not None and n_units < len(unit_ids):
741
+ rg = np.random.default_rng(seed)
742
+ unit_ids = rg.choice(unit_ids, size=n_units, replace=False)
743
+ unit_ids.sort()
744
+ if not overwrite and all_summaries_done(
745
+ unit_ids,
746
+ save_folder,
747
+ sorting_analysis=sorting_analysis,
748
+ namebyamp=namebyamp,
749
+ ext=image_ext,
750
+ ):
751
+ return
752
+
753
+ save_folder.mkdir(exist_ok=True, parents=True)
754
+
755
+ global_params = dict(
756
+ amplitude_color_cutoff=amplitude_color_cutoff,
757
+ pca_radius_um=pca_radius_um,
758
+ **other_global_params,
759
+ )
760
+ initargs = (
761
+ sorting_analysis,
762
+ plots,
763
+ max_height,
764
+ figsize,
765
+ dpi,
766
+ save_folder,
767
+ image_ext,
768
+ overwrite,
769
+ global_params,
770
+ gizmo_name,
771
+ namebyamp,
772
+ )
773
+ if n_jobs is None:
774
+ n_jobs = get_global_computation_config().n_jobs_cpu
775
+ if n_jobs:
776
+ initargs = (cloudpickle.dumps(initargs),)
777
+ n_jobs, Executor, context = get_pool(n_jobs, cls=CloudpicklePoolExecutor) # type: ignore
778
+ with Executor( # type: ignore
779
+ max_workers=n_jobs,
780
+ mp_context=context,
781
+ initializer=_summary_init,
782
+ initargs=initargs,
783
+ ) as pool:
784
+ results = pool.map(_summary_job, unit_ids)
785
+ if show_progress:
786
+ results = tqdm(
787
+ results,
788
+ desc=f"Unit {taskname}",
789
+ smoothing=0,
790
+ total=len(unit_ids),
791
+ )
792
+ for res in results:
793
+ pass
794
+
795
+
796
+ # -- utilities
797
+
798
+
799
+ def trim_waveforms(waveforms, old_offset=42, new_offset=42, new_length=121):
800
+ if waveforms.shape[1] == new_length and old_offset == new_offset:
801
+ return waveforms
802
+
803
+ start = old_offset - new_offset
804
+ end = start + new_length
805
+ return waveforms[:, start:end]
806
+
807
+
808
+ def pngname(unit_id, sorting_analysis=None, namebyamp=False, ext="png"):
809
+ if not namebyamp:
810
+ return f"unit{unit_id:04d}.{ext}"
811
+ if sorting_analysis is None:
812
+ raise ValueError(f"Need a sorting_analysis if namebyamp.")
813
+ amp = float(sorting_analysis.unit_amplitudes(unit_id).item())
814
+ amp = f"{amp:07.2f}"
815
+ return f"amp{amp}_unit{unit_id:04d}.{ext}"
816
+
817
+
818
+ def all_summaries_done(
819
+ unit_ids, save_folder, sorting_analysis=None, namebyamp=False, ext="png"
820
+ ):
821
+ if not save_folder.exists():
822
+ return False
823
+ return all(
824
+ (
825
+ save_folder
826
+ / pngname(
827
+ unit_id, sorting_analysis=sorting_analysis, namebyamp=namebyamp, ext=ext
828
+ )
829
+ ).exists()
830
+ for unit_id in unit_ids
831
+ )
832
+
833
+
834
+ # -- parallelism helpers
835
+
836
+
837
+ class SummaryJobContext:
838
+ def __init__(
839
+ self,
840
+ sorting_analysis: DARTsortAnalysis,
841
+ plots,
842
+ max_height,
843
+ figsize,
844
+ dpi,
845
+ save_folder,
846
+ image_ext,
847
+ overwrite,
848
+ global_params,
849
+ gizmo_name,
850
+ namebyamp,
851
+ ):
852
+ self.sorting_analysis = sorting_analysis
853
+ self.plots = plots
854
+ self.max_height = max_height
855
+ self.figsize = figsize
856
+ self.dpi = dpi
857
+ self.save_folder = save_folder
858
+ self.image_ext = image_ext
859
+ self.overwrite = overwrite
860
+ self.global_params = global_params
861
+ self.gizmo_name = gizmo_name
862
+ self.namebyamp = namebyamp
863
+
864
+
865
+ _summary_job_context = None
866
+
867
+
868
+ def _summary_init(*args):
869
+ global _summary_job_context
870
+ if len(args) == 1:
871
+ args = cloudpickle.loads(args[0])
872
+ _summary_job_context = SummaryJobContext(*args)
873
+
874
+
875
+ def _summary_job(unit_id: int):
876
+ # handle resuming/overwriting
877
+ assert _summary_job_context is not None
878
+ ext = _summary_job_context.image_ext
879
+ tmp_out = _summary_job_context.save_folder / f"tmp_unit{unit_id:04d}.{ext}"
880
+ imfn = pngname(
881
+ unit_id,
882
+ sorting_analysis=_summary_job_context.sorting_analysis,
883
+ namebyamp=_summary_job_context.namebyamp,
884
+ ext=_summary_job_context.image_ext,
885
+ )
886
+ final_out = _summary_job_context.save_folder / imfn
887
+ if tmp_out.exists():
888
+ tmp_out.unlink()
889
+ if not _summary_job_context.overwrite and final_out.exists():
890
+ return
891
+ if _summary_job_context.overwrite and final_out.exists():
892
+ final_out.unlink()
893
+
894
+ fig = plt.figure(
895
+ figsize=_summary_job_context.figsize,
896
+ layout="constrained",
897
+ # dpi=_summary_job_context.dpi,
898
+ )
899
+ try:
900
+ make_unit_summary(
901
+ _summary_job_context.sorting_analysis,
902
+ unit_id,
903
+ plots=_summary_job_context.plots,
904
+ max_height=_summary_job_context.max_height,
905
+ figsize=_summary_job_context.figsize,
906
+ figure=fig,
907
+ gizmo_name=_summary_job_context.gizmo_name,
908
+ **_summary_job_context.global_params,
909
+ )
910
+ except Exception as e:
911
+ raise ValueError(f"Error making plots for {unit_id=}.") from e
912
+
913
+ # the save is done sort of atomically to help with the resuming and avoid
914
+ # half-baked image files
915
+ fig.savefig(tmp_out, dpi=_summary_job_context.dpi)
916
+ tmp_out.rename(final_out)
917
+ plt.close(fig)