DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/vis/versus.py ADDED
@@ -0,0 +1,300 @@
1
+ from typing import Sequence
2
+
3
+ import matplotlib.pyplot as plt
4
+ from matplotlib.lines import Line2D
5
+ import numpy as np
6
+ import seaborn as sns
7
+
8
+ from ..evaluate.comparison import DARTsortGTVersus
9
+ from .colors import glasbey1024
10
+ from .layout import BasePlot, flow_layout
11
+
12
+
13
+ _cm = {
14
+ "gt_ptp_amplitude": "viridis",
15
+ "gt_firing_rate": "cividis",
16
+ "min_temp_dist": [k for k in ("managua", "magma") if k in plt.colormaps][0],
17
+ }
18
+ _c = {
19
+ "accuracy": "green",
20
+ "precision": "b",
21
+ "recall": "r",
22
+ "unsorted_recall": "blueviolet",
23
+ "min_temp_dist": "darkorange",
24
+ "n_units": "k",
25
+ "gt_matched_collidedness": "teal",
26
+ "gt_dt_rms": "palevioletred",
27
+ }
28
+ _o = {
29
+ "accuracy": 1,
30
+ "precision": 1,
31
+ "recall": 1,
32
+ "unsorted_recall": 1,
33
+ "min_temp_dist": -1,
34
+ "gt_matched_collidedness": -1,
35
+ "gt_dt_rms": -1,
36
+ }
37
+
38
+ _legkw = dict(
39
+ borderpad=0.2,
40
+ labelspacing=0.2,
41
+ handlelength=1.0,
42
+ columnspacing=1.0,
43
+ )
44
+ _box_kw = dict(showmeans=True, meanprops=dict(markeredgecolor="k", marker="+", markersize=15))
45
+ _scatter_kw = dict(linewidths=0, s=5)
46
+ _reg_common = dict(ci=None, scatter_kws=_scatter_kw)
47
+ _logistic = dict(logistic=True, **_reg_common)
48
+ _none = dict(fit_reg=False, **_reg_common)
49
+ _lowess = dict(lowess=True, **_reg_common)
50
+ _linear = dict(**_reg_common)
51
+ _regkind = {
52
+ "accuracy": _logistic,
53
+ "precision": _logistic,
54
+ "recall": _logistic,
55
+ "unsorted_recall": _logistic,
56
+ "min_temp_dist": _lowess,
57
+ "n_units": _none,
58
+ "gt_matched_collidedness": _lowess,
59
+ "gt_dt_rms": _lowess,
60
+ }
61
+ default_metrics = tuple([k for k in _c.keys() if k != "n_units"])
62
+
63
+
64
+ class VersusPlot(BasePlot):
65
+ kind = "vs"
66
+ width = 1
67
+ height = 1
68
+
69
+ # some plots only work for vs of two sorters
70
+ # the main fn below will filter plots depending on if the vs has only two
71
+ two_only = False
72
+
73
+ def draw(self, panel, vs: DARTsortGTVersus):
74
+ raise NotImplementedError
75
+
76
+
77
+ class MetricColumn(VersusPlot):
78
+ kind = "column"
79
+
80
+ def __init__(
81
+ self,
82
+ x: str = "gt_ptp_amplitude",
83
+ metrics: Sequence[str] = default_metrics,
84
+ diff=False,
85
+ hmul=1.0,
86
+ logx=True,
87
+ box=False,
88
+ box_x_cuts=None,
89
+ ):
90
+ self.x = x
91
+ self.nmetrics = len(metrics)
92
+ self.metrics = metrics
93
+ self.height = hmul * self.nmetrics
94
+ self.two_only = self.diff = diff
95
+ self.logx = logx
96
+ self.box = box
97
+ self.box_x_cuts = box_x_cuts
98
+
99
+ def draw(self, panel, vs):
100
+ axes = panel.subplots(nrows=self.nmetrics, sharex=True)
101
+ df = vs.unit_versus_dataframe()
102
+ if self.diff:
103
+ lines = [Line2D([0, 1], [0, 0], color='k')]
104
+ else:
105
+ spal = sns.color_palette(glasbey1024[: vs.n_vs])
106
+ lines = [Line2D([0, 1], [0, 0], color=c) for c in spal]
107
+
108
+ if self.diff:
109
+ df_a = df[df[vs.sorter_var] == vs.a_name]
110
+ df_b = df[df[vs.sorter_var] == vs.b_name]
111
+ df_a = df_a.sort_values(by="gt_unit_id")
112
+ df_b = df_b.sort_values(by="gt_unit_id")
113
+ assert np.array_equal(df_b.gt_unit_id, df_b.gt_unit_id)
114
+ df = df_a.copy()
115
+ for met in self.metrics:
116
+ df[met] -= df_b[met]
117
+
118
+ x = self.x
119
+ if self.box and self.box_x_cuts:
120
+ bins = [0] + self.box_x_cuts + [int(np.ceil(df[x].max().item()))]
121
+ df = df.copy(deep=False)
122
+ edge_strs = [f"{int(b)}" for b in bins]
123
+ bin_strs = np.array([f"{a}-{b}" for a, b in zip(bins, bins[1:])])
124
+ binix = np.searchsorted(bins, df[x].values, side="right") - 1
125
+ assert binix.min() >= 0
126
+ df["binix"] = binix
127
+ df[f"{x} bin"] = bin_strs[binix]
128
+ df = df.sort_values(by="binix")
129
+ x = f"{x} bin"
130
+ elif self.box:
131
+ x = None
132
+
133
+ for ax, met in zip(axes.flat, self.metrics):
134
+ logy = met == "min_temp_dist" and not self.diff
135
+ symlogy = met == "min_temp_dist" and self.diff
136
+
137
+ if self.diff:
138
+ ax.axhline(0, lw=0.8, color="k")
139
+ ckw = dict(color=_c[met])
140
+ else:
141
+ ckw = dict(hue=vs.sorter_var, hue_order=vs.other_names, palette=spal)
142
+
143
+ if not self.box:
144
+ regkw = _lowess if self.diff else _regkind[met]
145
+ if self.diff:
146
+ regkw = regkw | dict(line_kws=dict(color="k"))
147
+ sns.regplot(df, ax=ax, x=x, y=met, **regkw, **ckw)
148
+ else:
149
+ # first, scatter in random order
150
+ sdf = df.sample(frac=1).reset_index()
151
+ sns.scatterplot(
152
+ sdf,
153
+ ax=ax,
154
+ x=x,
155
+ y=met,
156
+ legend=False,
157
+ **ckw,
158
+ **_scatter_kw,
159
+ )
160
+
161
+ # next, regressions.
162
+ for sorter, color in zip(vs.other_names, glasbey1024):
163
+ sdf = df[df[vs.sorter_var] == sorter]
164
+ sns.regplot(
165
+ sdf,
166
+ ax=ax,
167
+ x=x,
168
+ y=met,
169
+ color=color,
170
+ scatter=False,
171
+ **regkw,
172
+ )
173
+ else:
174
+ needcols = [vs.sorter_var, met]
175
+ if x is not None:
176
+ needcols.append(x)
177
+ sns.boxplot(
178
+ df[needcols].dropna().reset_index(), x=x, y=met, legend=False, log_scale=logy, ax=ax, **_box_kw, **ckw
179
+ )
180
+
181
+ if logy and self.logx and not self.box:
182
+ ax.loglog()
183
+ elif logy and not self.box:
184
+ ax.semilogy()
185
+ elif self.logx and not self.box:
186
+ ax.semilogx()
187
+ elif symlogy and not self.box:
188
+ ax.set_yscale("symlog")
189
+ if self.logx:
190
+ ax.set_xscale("log")
191
+ ax.set_ylabel(met, color=_c[met], fontsize="small")
192
+ ax.grid(which="both")
193
+ # sns.despine(ax=ax, bottom=self.diff)
194
+ if self.diff:
195
+ mean = df[met].mean()
196
+ lines = [Line2D([0, 1], [0, 0], color=_c[met])]
197
+ labels = [f"{vs.a_name}-{vs.b_name} ({mean:.2f})"]
198
+ ax.legend(
199
+ handles=lines,
200
+ labels=labels,
201
+ fancybox=False,
202
+ loc=("upper" if _o[met] < 0 else "lower") + " right",
203
+ **_legkw,
204
+ )
205
+ else:
206
+ means = [df[df[vs.sorter_var] == s][met].mean() for s in vs.other_names]
207
+ labels = [f"{n} ({m:.2f})" for n, m in zip(vs.other_names, means)]
208
+ ax.legend(
209
+ handles=lines,
210
+ labels=labels,
211
+ fancybox=False,
212
+ loc=("upper" if _o[met] < 0 else "lower") + " right",
213
+ **_legkw,
214
+ )
215
+ ax.set_xlabel(self.x)
216
+
217
+ if self.diff and self.box and self.box_x_cuts is None:
218
+ axes.flat[0].set_title("perf diff boxplots")
219
+ elif self.diff and self.box:
220
+ axes.flat[0].set_title("amp binned perf diff boxplots")
221
+ elif self.box and self.box_x_cuts is None:
222
+ axes.flat[0].set_title("perf boxplots")
223
+ elif self.box:
224
+ axes.flat[0].set_title("amp binned perf boxplots")
225
+ elif self.diff:
226
+ axes.flat[0].set_title(f"perf diff scatter vs. {self.x}")
227
+ else:
228
+ axes.flat[0].set_title(f"perf scatter vs. {self.x}")
229
+
230
+
231
+ class OrderedPerformance(VersusPlot):
232
+ """Inspired by spikeinterface.benchmark.benchmark_plot_tools.plot_performances_ordered"""
233
+
234
+ def __init__(self, metrics=default_metrics, hmul=1.0):
235
+ self.nmetrics = len(metrics)
236
+ self.metrics = metrics
237
+ self.height = hmul * self.nmetrics
238
+
239
+ def draw(self, panel, vs):
240
+ axes = panel.subplots(nrows=self.nmetrics, sharex=True)
241
+ df = vs.unit_versus_dataframe()
242
+ x = np.arange(vs.n_gt_units)
243
+
244
+ for ax, met in zip(axes.flat, self.metrics):
245
+ for sorter, color in zip(vs.other_names, glasbey1024):
246
+ y = df[df[vs.sorter_var] == sorter][met].values
247
+ y = y[np.isfinite(y)]
248
+ y = y[np.argsort(-_o[met] * y)]
249
+ ax.step(x[:len(y)], y, color=color, lw=1, label=sorter)
250
+ ax.grid(which="both")
251
+ ax.set_ylabel(met, color=_c[met])
252
+ if met == "min_temp_dist":
253
+ ax.semilogy()
254
+ ax.set_xlabel("ordered GT units")
255
+ axes.flat[-1].legend(loc="upper left", fancybox=False, **_legkw)
256
+ axes.flat[0].set_title("sorted performance")
257
+
258
+
259
+ def get_versus_plots(vs) -> list[VersusPlot]:
260
+ plots = [
261
+ MetricColumn(),
262
+ MetricColumn(x="gt_collidedness", metrics=["recall", "min_temp_dist"]),
263
+ MetricColumn(x="gt_matched_collidedness", metrics=["recall", "min_temp_dist"]),
264
+ MetricColumn(x="gt_dt_rms", metrics=["recall", "min_temp_dist"]),
265
+ MetricColumn(diff=True),
266
+ MetricColumn(box=True),
267
+ MetricColumn(box=True, box_x_cuts=[5, 10, 15, 20]),
268
+ MetricColumn(box=True, box_x_cuts=[5, 10, 15, 20], diff=True),
269
+ OrderedPerformance(),
270
+ ]
271
+ if not vs.is_two:
272
+ plots = [p for p in plots if not p.two_only]
273
+ return plots
274
+
275
+
276
+ def make_versus_summary(
277
+ vs: DARTsortGTVersus,
278
+ plots: Sequence[VersusPlot] | None=None,
279
+ max_height=6,
280
+ figsize=(22, 14),
281
+ figure=None,
282
+ suptitle=False,
283
+ ):
284
+ if plots is None:
285
+ plots = get_versus_plots(vs)
286
+
287
+ figure = flow_layout(
288
+ plots,
289
+ max_height=max_height,
290
+ figsize=figsize,
291
+ figure=figure,
292
+ vs=vs,
293
+ )
294
+ if suptitle is True:
295
+ vs_str = " vs. ".join(vs.other_names)
296
+ figure.suptitle(f"{vs.gt_name} compare: {vs_str}")
297
+ elif suptitle:
298
+ figure.suptitle(suptitle)
299
+
300
+ return figure