DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,501 @@
1
+ from tempfile import tempdir
2
+ from typing import Optional
3
+ import warnings
4
+
5
+ import numpy as np
6
+ from scipy.cluster.hierarchy import fcluster, linkage
7
+ from scipy.sparse import coo_array
8
+ from scipy.sparse.csgraph import maximum_bipartite_matching
9
+ from tqdm.auto import tqdm
10
+
11
+
12
+ from ..util.internal_config import (
13
+ TemplateConfig,
14
+ TemplateMergeConfig,
15
+ ComputationConfig,
16
+ )
17
+ from ..util.logging_util import get_logger
18
+ from ..templates import TemplateData, template_util
19
+ from ..peel.matching_util.pairwise_util import (
20
+ construct_shift_indices,
21
+ iterate_compressed_pairwise_convolutions,
22
+ )
23
+ from ..util.data_util import DARTsortSorting, combine_sortings
24
+ from ..util import job_util
25
+
26
+
27
+ logger = get_logger(__name__)
28
+
29
+
30
+ def merge_templates(
31
+ sorting: DARTsortSorting,
32
+ template_data: TemplateData,
33
+ max_shift_samples=40,
34
+ superres_linkage=np.max,
35
+ linkage="complete",
36
+ distance_kind="rms",
37
+ sym_function=np.minimum,
38
+ merge_distance_threshold=0.25,
39
+ temporal_upsampling_factor=1,
40
+ amplitude_scaling_variance=0.001,
41
+ amplitude_scaling_boundary=0.1,
42
+ svd_compression_rank=20,
43
+ min_channel_amplitude=0.0,
44
+ spatial_radius_a=None,
45
+ min_spatial_cosine=0.0,
46
+ conv_batch_size=128,
47
+ units_batch_size=8,
48
+ computation_cfg=None,
49
+ show_progress=True,
50
+ ):
51
+ """Template distance based merge
52
+
53
+ Pass in a sorting, recording and template config to make templates,
54
+ and this will merge them (with superres). Or, if you have templates
55
+ already, pass them into template_data and we can skip the template
56
+ construction.
57
+
58
+ Arguments
59
+ ---------
60
+ max_shift_samples
61
+ Max offset during matching
62
+ superres_linkage
63
+ How to combine distances between two units' superres templates
64
+ By default, it's the max.
65
+ amplitude_scaling_*
66
+ Optionally allow scaling during matching
67
+
68
+ Returns
69
+ -------
70
+ A new DARTsortSorting
71
+ """
72
+ computation_cfg = job_util.ensure_computation_config(computation_cfg)
73
+ dist_matrix_kwargs = dict(
74
+ superres_linkage=superres_linkage,
75
+ sym_function=sym_function,
76
+ max_shift_samples=max_shift_samples,
77
+ temporal_upsampling_factor=temporal_upsampling_factor,
78
+ amplitude_scaling_variance=amplitude_scaling_variance,
79
+ amplitude_scaling_boundary=amplitude_scaling_boundary,
80
+ svd_compression_rank=svd_compression_rank,
81
+ min_channel_amplitude=min_channel_amplitude,
82
+ min_spatial_cosine=min_spatial_cosine,
83
+ conv_batch_size=conv_batch_size,
84
+ units_batch_size=units_batch_size,
85
+ distance_kind=distance_kind,
86
+ spatial_radius_a=spatial_radius_a,
87
+ )
88
+ units, dists, shifts, template_snrs = calculate_merge_distances(
89
+ template_data,
90
+ device=computation_cfg.actual_device(),
91
+ n_jobs=computation_cfg.actual_n_jobs(),
92
+ show_progress=show_progress,
93
+ **dist_matrix_kwargs, # type: ignore
94
+ )
95
+
96
+ # now run hierarchical clustering
97
+ merged_sorting, new_unit_ids = recluster(
98
+ sorting,
99
+ units,
100
+ dists,
101
+ shifts,
102
+ template_snrs,
103
+ merge_distance_threshold=merge_distance_threshold,
104
+ link=linkage,
105
+ )
106
+
107
+ return dict(sorting=merged_sorting, new_unit_ids=new_unit_ids)
108
+
109
+
110
+ def get_merge_distances(
111
+ template_data: TemplateData,
112
+ template_merge_cfg: TemplateMergeConfig,
113
+ computation_cfg: ComputationConfig | None = None,
114
+ cooccurrence_mask=None,
115
+ show_progress=True,
116
+ conv_batch_size=128,
117
+ units_batch_size=8,
118
+ sampling_frequency: float = 30000.0,
119
+ ):
120
+ computation_cfg = job_util.ensure_computation_config(computation_cfg)
121
+ shift_samples = int(template_merge_cfg.max_shift_ms * (sampling_frequency / 1000))
122
+ units, dists, shifts, template_snrs = calculate_merge_distances(
123
+ template_data=template_data,
124
+ max_shift_samples=shift_samples,
125
+ temporal_upsampling_factor=template_merge_cfg.temporal_upsampling_factor,
126
+ amplitude_scaling_variance=template_merge_cfg.amplitude_scaling_variance,
127
+ amplitude_scaling_boundary=template_merge_cfg.amplitude_scaling_boundary,
128
+ svd_compression_rank=template_merge_cfg.svd_compression_rank,
129
+ min_spatial_cosine=template_merge_cfg.min_spatial_cosine,
130
+ cooccurrence_mask=cooccurrence_mask,
131
+ conv_batch_size=conv_batch_size,
132
+ units_batch_size=units_batch_size,
133
+ device=computation_cfg.actual_device(),
134
+ n_jobs=computation_cfg.actual_n_jobs(),
135
+ show_progress=show_progress,
136
+ distance_kind=template_merge_cfg.distance_kind,
137
+ )
138
+ return units, dists, shifts, template_snrs
139
+
140
+
141
+ def calculate_merge_distances(
142
+ template_data,
143
+ superres_linkage=np.max,
144
+ sym_function=np.minimum,
145
+ max_shift_samples=40,
146
+ temporal_upsampling_factor=1,
147
+ amplitude_scaling_variance=0.001,
148
+ amplitude_scaling_boundary=0.1,
149
+ svd_compression_rank=20,
150
+ min_channel_amplitude=0.0,
151
+ min_spatial_cosine=0.0,
152
+ spatial_radius_a=None,
153
+ cooccurrence_mask=None,
154
+ conv_batch_size=128,
155
+ units_batch_size=8,
156
+ device=None,
157
+ n_jobs=0,
158
+ show_progress=True,
159
+ distance_kind="rms",
160
+ ):
161
+ # allocate distance + shift matrices. shifts[i,j] is trough[j]-trough[i].
162
+ n_templates = template_data.templates.shape[0]
163
+ sup_dists = np.full((n_templates, n_templates), np.inf)
164
+ sup_shifts = np.zeros((n_templates, n_templates), dtype=int)
165
+
166
+ # build distance matrix
167
+ dec_res_iter = get_deconv_resid_decrease_iter(
168
+ template_data,
169
+ max_shift_samples=max_shift_samples,
170
+ temporal_upsampling_factor=temporal_upsampling_factor,
171
+ amplitude_scaling_variance=amplitude_scaling_variance,
172
+ amplitude_scaling_boundary=amplitude_scaling_boundary,
173
+ svd_compression_rank=svd_compression_rank,
174
+ min_channel_amplitude=min_channel_amplitude,
175
+ min_spatial_cosine=min_spatial_cosine,
176
+ cooccurrence_mask=cooccurrence_mask,
177
+ conv_batch_size=conv_batch_size,
178
+ units_batch_size=units_batch_size,
179
+ spatial_radius_a=spatial_radius_a,
180
+ device=device,
181
+ n_jobs=n_jobs,
182
+ show_progress=show_progress,
183
+ distance_kind=distance_kind,
184
+ )
185
+ for res in dec_res_iter:
186
+ if res is None:
187
+ # all pairs in chunk were ignored for one reason or another
188
+ continue
189
+
190
+ tixa = res.template_indices_a
191
+ tixb = res.template_indices_b
192
+ sup_dists[tixa, tixb] = res.deconv_resid_decreases / res.template_a_norms # type: ignore
193
+ sup_shifts[tixa, tixb] = res.shifts # type: ignore
194
+
195
+ # apply linkage to reduce across superres templates
196
+ units = np.unique(template_data.unit_ids)
197
+ if units.size < n_templates:
198
+ dists = np.full((units.size, units.size), np.inf)
199
+ shifts = np.zeros((units.size, units.size), dtype=int)
200
+ for ia, ua in enumerate(units):
201
+ in_ua = np.flatnonzero(template_data.unit_ids == ua)
202
+ for ib, ub in enumerate(units):
203
+ in_ub = np.flatnonzero(template_data.unit_ids == ub)
204
+ in_pair = (in_ua[:, None], in_ub[None, :])
205
+ dists[ia, ib] = superres_linkage(sup_dists[in_pair])
206
+ shifts[ia, ib] = np.median(sup_shifts[in_pair])
207
+ coarse_td = template_data.coarsen(with_locs=False)
208
+ template_snrs = np.ptp(coarse_td.templates, 1).max(1) / coarse_td.spike_counts
209
+ else:
210
+ dists = sup_dists
211
+ shifts = sup_shifts
212
+ template_snrs = (
213
+ np.ptp(template_data.templates, 1).max(1) / template_data.spike_counts
214
+ )
215
+
216
+ dists = sym_function(dists, dists.T)
217
+ np.fill_diagonal(dists, 0.0) # sometimes numerical 0 is -1e-6.
218
+ min_dist = dists.min()
219
+ if min_dist < -1e-3:
220
+ warnings.warn(f"Alarmingly negative min distance {min_dist}.")
221
+ dists = np.maximum(dists, 0.0, out=dists)
222
+
223
+ return units, dists, shifts, template_snrs
224
+
225
+
226
+ def cross_match_distance_matrix(
227
+ template_data_a,
228
+ template_data_b,
229
+ superres_linkage=np.max,
230
+ sym_function=np.minimum,
231
+ max_shift_samples=40,
232
+ temporal_upsampling_factor=1,
233
+ amplitude_scaling_variance=0.001,
234
+ amplitude_scaling_boundary=0.1,
235
+ svd_compression_rank=20,
236
+ min_channel_amplitude=0.0,
237
+ min_spatial_cosine=0.0,
238
+ spatial_radius_a=None,
239
+ conv_batch_size=128,
240
+ units_batch_size=8,
241
+ distance_kind="rms",
242
+ device=None,
243
+ n_jobs=0,
244
+ show_progress=True,
245
+ ):
246
+ template_data, cross_mask, ids_a, ids_b = combine_templates(
247
+ template_data_a, template_data_b
248
+ )
249
+ units, dists, shifts, template_snrs = calculate_merge_distances(
250
+ template_data,
251
+ superres_linkage=superres_linkage,
252
+ sym_function=sym_function,
253
+ max_shift_samples=max_shift_samples,
254
+ temporal_upsampling_factor=temporal_upsampling_factor,
255
+ amplitude_scaling_variance=amplitude_scaling_variance,
256
+ amplitude_scaling_boundary=amplitude_scaling_boundary,
257
+ svd_compression_rank=svd_compression_rank,
258
+ min_channel_amplitude=min_channel_amplitude,
259
+ min_spatial_cosine=min_spatial_cosine,
260
+ spatial_radius_a=spatial_radius_a,
261
+ cooccurrence_mask=cross_mask,
262
+ conv_batch_size=conv_batch_size,
263
+ units_batch_size=units_batch_size,
264
+ distance_kind=distance_kind,
265
+ device=device,
266
+ n_jobs=n_jobs,
267
+ show_progress=show_progress,
268
+ )
269
+
270
+ # symmetrize the merge distance matrix, which right now is block off-diagonal
271
+ # (with infs on main diag blocks)
272
+ a_mask = np.flatnonzero(np.isin(units, ids_a))
273
+ b_mask = np.flatnonzero(np.isin(units, ids_b))
274
+ Dab = dists[a_mask[:, None], b_mask[None, :]]
275
+ Dba = dists[b_mask[:, None], a_mask[None, :]]
276
+ Dstack = np.stack((Dab, Dba.T))
277
+ shifts_ab = shifts[a_mask[:, None], b_mask[None, :]]
278
+ shifts_ba = shifts[b_mask[:, None], a_mask[None, :]]
279
+ shifts_stack = np.stack((shifts_ab, -shifts_ba.T))
280
+
281
+ # choices[ia, ib] = 0 if Dab[ia, ib] < Dba[ib, ia], else 1
282
+ # but what does that mean? if 0, it means that the upsampled model
283
+ # of B was better able to match A than the other way around. it's not
284
+ # that important, at the end of the day. still, we keep track of it
285
+ # so that we can pick the shift and move on with our lives.
286
+ choices = np.argmin(Dstack, axis=0)
287
+ dists = Dstack[
288
+ choices, np.arange(choices.shape[0])[:, None], np.arange(choices.shape[1])[None]
289
+ ]
290
+ shifts = shifts_stack[
291
+ choices, np.arange(choices.shape[0])[:, None], np.arange(choices.shape[1])[None]
292
+ ]
293
+
294
+ # handle duplicates and missing
295
+ a_inds = np.searchsorted(units[a_mask], ids_a, side="right") - 1
296
+ a_kept = units[a_mask][a_inds] == ids_a
297
+ b_inds = np.searchsorted(units[b_mask], ids_b, side="right") - 1
298
+ b_kept = units[b_mask][b_inds] == ids_b
299
+ dists = dists[a_inds[a_kept][:, None], b_inds[b_kept][None, :]]
300
+
301
+ snrs_a = template_snrs[a_mask][a_inds[a_kept]]
302
+ snrs_b = template_snrs[b_mask][b_inds[b_kept]]
303
+
304
+ return (
305
+ dists,
306
+ shifts,
307
+ snrs_a,
308
+ snrs_b,
309
+ a_kept,
310
+ b_kept,
311
+ )
312
+
313
+
314
+ def recluster(
315
+ sorting,
316
+ units,
317
+ dists,
318
+ shifts,
319
+ template_snrs,
320
+ merge_distance_threshold=0.25,
321
+ link="complete",
322
+ ):
323
+ # upper triangle not including diagonal, aka condensed distance matrix in scipy
324
+ pdist = dists[np.triu_indices(dists.shape[0], k=1)]
325
+ # scipy hierarchical clustering only supports finite values, so let's just
326
+ # drop in a huge value here
327
+ finite = np.isfinite(pdist)
328
+ if not finite.any():
329
+ return sorting, np.arange(dists.shape[0])
330
+ assert units.shape[0] == dists.shape[0] == dists.shape[1]
331
+
332
+ pdist[~finite] = 1_000_000 + pdist[finite].max()
333
+ Z = linkage(pdist, method=link)
334
+ # extract flat clustering using our max dist threshold
335
+ new_labels = fcluster(Z, merge_distance_threshold, criterion="distance")
336
+ assert new_labels.min() == 1 # start at 1 for some reason
337
+ new_labels -= 1
338
+
339
+ # update labels
340
+ labels_updated = np.full_like(sorting.labels, -1)
341
+ kept = np.flatnonzero(np.isin(sorting.labels, units))
342
+ labels_updated[kept] = new_labels[sorting.labels[kept]]
343
+
344
+ # update times according to shifts
345
+ times_updated = sorting.times_samples.copy()
346
+
347
+ # find original labels in each cluster
348
+ clust_inverse = {i: [] for i in new_labels}
349
+ for orig_label, new_label in enumerate(new_labels):
350
+ clust_inverse[new_label].append(orig_label)
351
+ n_merges = sum(len(v) - 1 for v in clust_inverse.values())
352
+ logger.dartsortdebug(f"Merged {n_merges} templates.")
353
+
354
+ # align to best snr unit
355
+ for new_label, orig_labels in clust_inverse.items():
356
+ # we don't need to realign clusters which didn't change
357
+ if len(orig_labels) <= 1:
358
+ continue
359
+
360
+ orig_snrs = template_snrs[orig_labels]
361
+ best_orig = orig_labels[orig_snrs.argmax()]
362
+ for ogl in np.setdiff1d(orig_labels, [best_orig]):
363
+ in_orig_unit = np.flatnonzero(sorting.labels == ogl)
364
+ # this is like trough[best] - trough[ogl]
365
+ shift_og_best = shifts[ogl, best_orig]
366
+ # if >0, trough of og is behind trough of best.
367
+ # subtracting will move trough of og to the right.
368
+ times_updated[in_orig_unit] -= shift_og_best
369
+
370
+ new_sorting = sorting.ephemeral_replace(
371
+ times_samples=times_updated, labels=labels_updated
372
+ )
373
+ return new_sorting, new_labels
374
+
375
+
376
+ def get_deconv_resid_decrease_iter(
377
+ template_data,
378
+ max_shift_samples=40,
379
+ temporal_upsampling_factor=8,
380
+ amplitude_scaling_variance=0.001,
381
+ amplitude_scaling_boundary=0.1,
382
+ svd_compression_rank=20,
383
+ min_channel_amplitude=0.0,
384
+ min_spatial_cosine=0.0,
385
+ cooccurrence_mask=None,
386
+ conv_batch_size=128,
387
+ units_batch_size=8,
388
+ spatial_radius_a=None,
389
+ ignore_empty_channels=True,
390
+ distance_kind="rms",
391
+ device=None,
392
+ n_jobs=0,
393
+ show_progress=True,
394
+ ):
395
+ # get template aux data
396
+ low_rank_templates_b = template_util.svd_compress_templates(
397
+ template_data,
398
+ min_channel_amplitude=min_channel_amplitude,
399
+ rank=svd_compression_rank,
400
+ )
401
+ compressed_upsampled_temporal = template_util.compressed_upsampled_templates(
402
+ low_rank_templates_b.temporal_components,
403
+ ptps=np.ptp(template_data.templates, 1).max(1),
404
+ max_upsample=temporal_upsampling_factor,
405
+ )
406
+
407
+ # restrict spatial subset of target templates
408
+ template_data_a = template_data_b = template_data
409
+ low_rank_templates_a = low_rank_templates_b
410
+ if spatial_radius_a:
411
+ template_data_a = template_util.spatially_mask_templates(
412
+ template_data, spatial_radius_a
413
+ )
414
+ low_rank_templates_a = template_util.svd_compress_templates(
415
+ template_data_a,
416
+ min_channel_amplitude=min_channel_amplitude,
417
+ rank=svd_compression_rank,
418
+ )
419
+
420
+ # construct helper data and run pairwise convolutions
421
+ (
422
+ template_shift_index_a,
423
+ template_shift_index_b,
424
+ upsampled_shifted_template_index,
425
+ cooccurrence,
426
+ ) = construct_shift_indices(
427
+ None,
428
+ None,
429
+ template_data,
430
+ compressed_upsampled_temporal,
431
+ motion_est=None,
432
+ )
433
+ if cooccurrence_mask is not None:
434
+ cooccurrence = cooccurrence & cooccurrence_mask
435
+ yield from iterate_compressed_pairwise_convolutions(
436
+ template_data_a,
437
+ low_rank_templates_a,
438
+ template_data_b,
439
+ low_rank_templates_b,
440
+ compressed_upsampled_temporal,
441
+ template_shift_index_a,
442
+ template_shift_index_b,
443
+ cooccurrence,
444
+ upsampled_shifted_template_index,
445
+ do_shifting=False,
446
+ reduce_deconv_resid_decrease=True,
447
+ geom=template_data.registered_geom,
448
+ conv_ignore_threshold=0.0,
449
+ min_spatial_cosine=min_spatial_cosine,
450
+ coarse_approx_error_threshold=0.0,
451
+ amplitude_scaling_variance=amplitude_scaling_variance,
452
+ amplitude_scaling_boundary=amplitude_scaling_boundary,
453
+ ignore_empty_channels=ignore_empty_channels,
454
+ distance_kind=distance_kind,
455
+ max_shift=max_shift_samples, # type: ignore
456
+ conv_batch_size=conv_batch_size,
457
+ units_batch_size=units_batch_size,
458
+ device=device,
459
+ n_jobs=n_jobs,
460
+ show_progress=show_progress,
461
+ )
462
+
463
+
464
+ def combine_templates(template_data_a, template_data_b):
465
+ rgeom = template_data_a.registered_geom
466
+ if rgeom is not None:
467
+ if not np.array_equal(rgeom, template_data_b.registered_geom):
468
+ raise ValueError(
469
+ f"Template data had different registered geoms: "
470
+ f"{template_data_a.registered_geom=} {template_data_b.registered_geom=}"
471
+ )
472
+
473
+ ids_a = template_data_a.unit_ids
474
+ ids_b = template_data_b.unit_ids + ids_a.max() + 1
475
+ unit_ids = np.concatenate((ids_a, ids_b))
476
+ templates = np.row_stack((template_data_a.templates, template_data_b.templates))
477
+ spike_counts = np.concatenate(
478
+ (template_data_a.spike_counts, template_data_b.spike_counts)
479
+ )
480
+ spike_counts_by_channel = None
481
+ if template_data_a.spike_counts_by_channel is not None:
482
+ spike_counts_by_channel = np.concatenate(
483
+ (
484
+ template_data_a.spike_counts_by_channel,
485
+ template_data_b.spike_counts_by_channel,
486
+ )
487
+ )
488
+ template_data = TemplateData(
489
+ templates=templates,
490
+ unit_ids=unit_ids,
491
+ spike_counts=spike_counts,
492
+ registered_geom=rgeom,
493
+ spike_counts_by_channel=spike_counts_by_channel,
494
+ )
495
+
496
+ cross_mask = np.logical_and(
497
+ np.isin(unit_ids, ids_a)[:, None], np.isin(unit_ids, ids_b)[None]
498
+ )
499
+ cross_mask = np.logical_or(cross_mask, cross_mask.T)
500
+
501
+ return template_data, cross_mask, ids_a, ids_b
@@ -0,0 +1,214 @@
1
+ import numpy as np
2
+ from scipy.signal import find_peaks
3
+ from scipy.stats import norm
4
+
5
+ from . import density
6
+
7
+ try:
8
+ from isosplit import up_down_isotonic_regression, jisotonic5 # type: ignore[reportMissingImports]
9
+ except ImportError:
10
+ up_down_isotonic_regression = lambda *a, **k: None
11
+ jisotonic5 = None
12
+ pass
13
+
14
+ # todo: replace all isosplit stuff with things based on scipy's isotonic regression.
15
+
16
+
17
+ def fit_unimodal_right(x, f, weights=None, cut=0, hard=False):
18
+ """Unimodal to the right of cut, increasing to the left. Continuity at the border."""
19
+ assert jisotonic5 is not None
20
+ if weights is None:
21
+ weights = np.ones_like(f)
22
+
23
+ # figure out where cut lands and what f(cut) should be
24
+ cuti_left = np.searchsorted(x, cut)
25
+ if cuti_left >= len(x) - 1:
26
+ # everything is to the left... fit increasing!
27
+ return jisotonic5.jisotonic5(f, weights=weights)[0]
28
+ cuti_right = np.searchsorted(x, cut, side="right") - 1
29
+ if cuti_right <= 0:
30
+ # everything is to the right... fit unimodal!
31
+ return up_down_isotonic_regression(f, weights=weights)
32
+ # else, in the middle somewhere...
33
+ assert cuti_left in (cuti_right, cuti_right + 1)
34
+
35
+ out = np.empty_like(f)
36
+ out[cuti_right:] = up_down_isotonic_regression(
37
+ np.ascontiguousarray(f[cuti_right:]),
38
+ weights=np.ascontiguousarray(weights[cuti_right:]),
39
+ )
40
+
41
+ # impose continuity with a large penalty
42
+ f_left = np.concatenate((f[:cuti_left], out[cuti_right : cuti_right + 1]))
43
+ w_left = np.concatenate(
44
+ (weights[:cuti_left], 1e6 + 10 * weights.sum(keepdims=True))
45
+ )
46
+ out[:cuti_left] = jisotonic5.jisotonic5(f_left, w_left)[0][:-1]
47
+
48
+ return out
49
+
50
+
51
+ def fit_truncnorm_right(x, f, weights=None, cut=0, hard=False, n_iter=10):
52
+ """Like above, but fits truncated normal to xs > cut with MoM."""
53
+ if weights is None:
54
+ weights = np.ones_like(f)
55
+
56
+ # figure out where cut lands and what f(cut) should be
57
+ cuti_left = np.searchsorted(x, cut)
58
+ if cuti_left >= len(x) - 2:
59
+ # everything is to the left... return uniform
60
+ return np.full_like(f, 1 / len(f))
61
+ cuti_right = np.searchsorted(x, cut, side="right") - 1
62
+ if cuti_right <= 1:
63
+ # everything is to the right... fit normal!
64
+ mean = np.average(x, weights=weights)
65
+ var = np.average((x - mean) ** 2, weights=weights)
66
+ return norm.pdf(x, loc=mean, scale=np.sqrt(var))
67
+ # else, in the middle somewhere...
68
+ assert cuti_left in (cuti_right, cuti_right + 1)
69
+
70
+ xcut = x[cuti_right:]
71
+ mean = np.average(xcut, weights=weights[cuti_right:])
72
+ var = np.average((xcut - mean) ** 2, weights=weights[cuti_right:])
73
+ mu = mean
74
+ sigma = std = np.sqrt(var)
75
+ for i in range(n_iter):
76
+ alpha = (cut - mu) / sigma
77
+ phi_alpha = norm.cdf(alpha)
78
+ Z = 1.0 - phi_alpha
79
+ # we have: mean ~ mu + sigma*phi(alpha)/Z
80
+ # and var ~ sigma^2(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
81
+ # implies mu = mean - sigma*phi(alpha)/Z
82
+ # sigma^2 = var/(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
83
+ sigma = np.sqrt(var / (1 + alpha * phi_alpha / Z - (phi_alpha / Z) ** 2))
84
+ mu = mean - sigma * phi_alpha / Z
85
+
86
+ return norm.pdf(x, loc=mu, scale=sigma)
87
+
88
+
89
+ def fit_bimodal_at(x, f, weights=None, cut=0):
90
+ from isosplit import up_down_isotonic_regression # type: ignore
91
+
92
+ if weights is None:
93
+ weights = np.ones_like(f)
94
+ which_right = x > cut
95
+ which_left = np.logical_not(which_right)
96
+ out = np.empty_like(f)
97
+ if which_left.any():
98
+ out[which_left] = up_down_isotonic_regression(
99
+ f[which_left], weights=weights[which_left]
100
+ )
101
+ if which_right.any():
102
+ out[which_right] = up_down_isotonic_regression(
103
+ f[which_right], weights=weights[which_right]
104
+ )
105
+ return out
106
+
107
+
108
+ def smoothed_dipscore_at(
109
+ cut,
110
+ samples,
111
+ alternative="smoothed",
112
+ dipscore_only=False,
113
+ score_kind="tv",
114
+ cut_relmax_order=3,
115
+ null="isotoniconesideunnormed",
116
+ debug_info=None,
117
+ ):
118
+ if samples.size < 3:
119
+ return 0.0
120
+
121
+ densities = density.get_smoothed_density(
122
+ samples[:, None],
123
+ sigma=0.5 * 1.06 * samples.std() * (samples.size**-0.2),
124
+ bin_size_ratio=20.0,
125
+ min_bin_size=0.025,
126
+ max_n_bins=512,
127
+ min_n_bins=10,
128
+ )
129
+ spacings = np.diff(samples)
130
+ spacings = np.concatenate(
131
+ (spacings[:1], 0.5 * (spacings[1:] + spacings[:-1]), spacings[-1:])
132
+ )
133
+ with np.errstate(all="raise"):
134
+ densities /= (densities * spacings).sum()
135
+
136
+ if cut is None:
137
+ # closest maxes left + right of 0
138
+ maxers, _ = find_peaks(densities, distance=cut_relmax_order)
139
+ coords = samples[maxers]
140
+ left_cut = right_cut = 0
141
+ if (coords < 0).any():
142
+ left_cut = coords[coords < 0].max()
143
+ if (coords > 0).any():
144
+ right_cut = coords[coords > 0].min()
145
+ candidates = np.logical_and(samples >= left_cut, samples <= right_cut)
146
+ cut = 0
147
+ if candidates.any():
148
+ cut = samples[candidates][np.argmin(densities[candidates])]
149
+
150
+ if alternative == "bimodal":
151
+ densities = fit_bimodal_at(samples, densities, cut=cut)
152
+ densities /= (densities * spacings).sum()
153
+ else:
154
+ assert alternative == "smoothed"
155
+
156
+ score = 0
157
+ best_dens_err = np.inf
158
+ best_uni = None
159
+
160
+ if score_kind == "ks":
161
+ empirical = np.cumsum(densities * spacings)
162
+
163
+ if null in ("isotonic", "gcm"):
164
+ orders = (slice(None),)
165
+ signs = (1,)
166
+ else:
167
+ orders = (slice(None), slice(None, None, -1))
168
+ signs = (1, -1)
169
+
170
+ # change to density which is best overall fit
171
+ for hard in (False, True):
172
+ for order, sign in zip(orders, signs):
173
+ s = np.ascontiguousarray(samples[order])
174
+ d = np.ascontiguousarray(densities[order])
175
+ if null in ("isotoniconeside", "isotoniconesideunnormed"):
176
+ dens = fit_unimodal_right(sign * s, d, cut=sign * cut, hard=hard)
177
+ elif null == "isotonic":
178
+ dens = up_down_isotonic_regression(d)
179
+ elif null == "truncnorm":
180
+ dens = fit_truncnorm_right(sign * s, d, cut=sign * cut, hard=hard)
181
+ else:
182
+ assert False
183
+
184
+ dens = dens[order] # type: ignore
185
+ if null not in ("isotonic", "isotoniconesideunnormed"):
186
+ dens /= (dens * spacings).sum()
187
+
188
+ dens_err = (np.abs(dens - densities) * spacings).sum()
189
+ if score_kind == "ks":
190
+ my_score = np.abs(empirical - np.cumsum(dens * spacings)).max() # type: ignore
191
+ elif score_kind == "tv":
192
+ my_score = 0.5 * dens_err
193
+ else:
194
+ assert False
195
+
196
+ # if my_score > score:
197
+ if dens_err < best_dens_err:
198
+ score = my_score
199
+ best_dens_err = dens_err
200
+ best_uni = dens
201
+
202
+ if debug_info is not None:
203
+ debug_info["domain"] = samples
204
+ debug_info["alternative_density"] = densities
205
+ debug_info["cut"] = cut
206
+ debug_info["score"] = score
207
+ debug_info["score_kind"] = score_kind
208
+ debug_info["uni_density"] = best_uni
209
+ debug_info["samples"] = samples
210
+
211
+ if dipscore_only:
212
+ return score
213
+
214
+ return score, samples, densities, best_uni, cut