DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,555 @@
1
+ from pathlib import Path
2
+ from typing import Any
3
+ import gc
4
+
5
+ import numpy as np
6
+ from sklearn.decomposition import PCA, TruncatedSVD
7
+ from spikeinterface.core import BaseRecording
8
+ from scipy.spatial import KDTree
9
+ from scipy.sparse import coo_array
10
+ import torch
11
+
12
+ from . import TemplateData, realign
13
+ from ..util.data_util import DARTsortSorting
14
+ from ..util.internal_config import (
15
+ ComputationConfig,
16
+ TemplateConfig,
17
+ TemplateMergeConfig,
18
+ TemplateRealignmentConfig,
19
+ WaveformConfig,
20
+ SubtractionConfig,
21
+ default_template_cfg,
22
+ default_waveform_cfg,
23
+ )
24
+ from ..util.job_util import ensure_computation_config
25
+ from ..util.logging_util import get_logger
26
+ from ..util.py_util import resolve_path
27
+ from ..util.spiketorch import ptp
28
+
29
+ logger = get_logger(__name__)
30
+
31
+
32
+ def estimate_template_library(
33
+ recording: BaseRecording,
34
+ sorting: DARTsortSorting,
35
+ motion_est=None,
36
+ min_template_snr: float = 0.0,
37
+ min_template_ptp: float = 0.0,
38
+ min_template_count: int = 0,
39
+ max_cc_flag_rate: float = 1.0,
40
+ cc_flag_entropy_cutoff: float = 0.0,
41
+ waveform_cfg: WaveformConfig = default_waveform_cfg,
42
+ template_cfg: TemplateConfig = default_template_cfg,
43
+ realign_cfg: TemplateRealignmentConfig | None = None,
44
+ template_merge_cfg: TemplateMergeConfig | None = None,
45
+ tsvd: PCA | TruncatedSVD | None = None,
46
+ computation_cfg: ComputationConfig | None = None,
47
+ detection_cfg: Any | None = None,
48
+ depth_order: bool = False,
49
+ template_npz_path=None,
50
+ ) -> tuple[DARTsortSorting, TemplateData]:
51
+ """Postprocess spike train and estimate a TemplateData."""
52
+ if template_npz_path is not None:
53
+ template_npz_path = resolve_path(template_npz_path)
54
+ if template_npz_path.exists():
55
+ return sorting, TemplateData.from_npz(template_npz_path)
56
+
57
+ if (sorting.labels is None) or (sorting.labels < 0).all():
58
+ raise ValueError("No labels in sorting input to template postprocessing.")
59
+ computation_cfg = ensure_computation_config(computation_cfg)
60
+
61
+ # realign sorting and estimate template snr
62
+ sorting, templates0 = realign(
63
+ recording=recording,
64
+ sorting=sorting,
65
+ realign_cfg=realign_cfg,
66
+ waveform_cfg=waveform_cfg,
67
+ computation_cfg=computation_cfg,
68
+ motion_est=motion_est,
69
+ )
70
+
71
+ # filter out low-count/snr units
72
+ if templates0 is None and (min_template_count or min_template_snr):
73
+ templates0 = _quick_mean_templates(
74
+ recording=recording,
75
+ sorting=sorting,
76
+ waveform_cfg=waveform_cfg,
77
+ computation_cfg=computation_cfg,
78
+ motion_est=motion_est,
79
+ )
80
+ if min_template_count or min_template_snr or (max_cc_flag_rate < 1.0):
81
+ assert templates0 is not None
82
+ count_mask = templates0.spike_counts >= min_template_count
83
+ snr_mask = templates0.snrs_by_channel().max(1) >= min_template_snr
84
+ amp_mask = ptp(templates0.templates).max(1) >= min_template_ptp
85
+ mask = count_mask & snr_mask & amp_mask
86
+ flag_mask = cc_flag_criterion(
87
+ sorting,
88
+ detection_cfg,
89
+ max_cc_flag_rate,
90
+ cc_flag_entropy_cutoff,
91
+ amplitudes_dataset_name=template_cfg.amplitudes_dataset_name,
92
+ )
93
+ if flag_mask is not None:
94
+ mask = mask & flag_mask
95
+ sorting = filter_by_unit_mask(sorting, mask, mask_ids=templates0.unit_ids)
96
+ del templates0
97
+
98
+ _check_still_valid(sorting)
99
+ gc.collect()
100
+ torch.cuda.empty_cache()
101
+
102
+ # main task: get denoised templates from aligned spike train
103
+ templates = TemplateData.from_config(
104
+ recording=recording,
105
+ sorting=sorting,
106
+ motion_est=motion_est,
107
+ waveform_cfg=waveform_cfg,
108
+ template_cfg=template_cfg,
109
+ computation_cfg=computation_cfg,
110
+ tsvd=tsvd,
111
+ )
112
+ gc.collect()
113
+ torch.cuda.empty_cache()
114
+
115
+ # merge units by template distance
116
+ sorting, templates = _handle_merge(
117
+ recording=recording,
118
+ sorting=sorting,
119
+ template_data=templates,
120
+ motion_est=motion_est,
121
+ merge_cfg=template_merge_cfg,
122
+ computation_cfg=computation_cfg,
123
+ waveform_cfg=waveform_cfg,
124
+ template_cfg=template_cfg,
125
+ )
126
+ gc.collect()
127
+ torch.cuda.empty_cache()
128
+
129
+ # re-order along probe length
130
+ if depth_order:
131
+ sorting, templates = reorder_by_depth(sorting, templates)
132
+
133
+ return sorting, ensure_save(templates, template_npz_path)
134
+
135
+
136
+ def realign_and_chuck_noisy_template_units(
137
+ recording,
138
+ sorting,
139
+ template_data=None,
140
+ motion_est=None,
141
+ min_n_spikes=50,
142
+ min_template_snr=15.0,
143
+ waveform_cfg=default_waveform_cfg,
144
+ template_cfg=default_template_cfg,
145
+ tsvd=None,
146
+ computation_cfg=None,
147
+ template_save_folder=None,
148
+ template_npz_filename=None,
149
+ ):
150
+ """Get rid of noise units.
151
+
152
+ This will reindex the sorting and template data -- unit labels will
153
+ change, and the number of templates will change.
154
+ """
155
+ if template_save_folder is not None:
156
+ if template_npz_filename is not None:
157
+ npz = Path(template_save_folder) / template_npz_filename
158
+ if npz.exists():
159
+ return sorting, TemplateData.from_npz(npz)
160
+
161
+ if template_data is None:
162
+ template_data = TemplateData.from_config(
163
+ recording,
164
+ sorting,
165
+ template_cfg=template_cfg,
166
+ motion_est=motion_est,
167
+ tsvd=tsvd,
168
+ waveform_cfg=waveform_cfg,
169
+ computation_cfg=computation_cfg,
170
+ save_folder=None,
171
+ save_npz_name=None,
172
+ )
173
+ assert sorting is not None
174
+ assert template_data.spike_counts_by_channel is not None
175
+
176
+ good_templates = snr_mask(
177
+ template_data, min_n_spikes=min_n_spikes, min_template_snr=min_template_snr
178
+ )
179
+ logger.dartsortdebug(
180
+ f"Discard {np.logical_not(good_templates).sum()} low-signal templates."
181
+ )
182
+ good_unit_ids = template_data.unit_ids[good_templates]
183
+ assert np.all(np.diff(good_unit_ids) >= 0)
184
+ unique_good_unit_ids, new_template_unit_ids = np.unique(
185
+ good_unit_ids, return_inverse=True
186
+ )
187
+
188
+ if template_data.properties:
189
+ properties = {k: v[good_templates] for k, v in template_data.properties.items()}
190
+ else:
191
+ properties = None
192
+
193
+ assert sorting.labels is not None
194
+ new_labels = sorting.labels.copy()
195
+ valid = np.isin(new_labels, unique_good_unit_ids)
196
+ new_labels[~valid] = -1
197
+ _, new_labels[valid] = np.unique(new_labels[valid], return_inverse=True)
198
+
199
+ new_sorting = sorting.ephemeral_replace(labels=new_labels)
200
+ new_template_data = TemplateData(
201
+ templates=template_data.templates[good_templates],
202
+ unit_ids=new_template_unit_ids,
203
+ spike_counts=template_data.spike_counts[good_templates],
204
+ spike_counts_by_channel=template_data.spike_counts_by_channel[good_templates],
205
+ registered_geom=template_data.registered_geom,
206
+ trough_offset_samples=template_data.trough_offset_samples,
207
+ properties=properties,
208
+ )
209
+ if template_save_folder is not None:
210
+ if template_npz_filename is not None:
211
+ npz = Path(template_save_folder) / template_npz_filename
212
+ new_template_data.to_npz(npz)
213
+
214
+ return new_sorting, new_template_data
215
+
216
+
217
+ def snr_mask(template_data, min_n_spikes=50, min_template_snr=15.0):
218
+ template_ptps = np.ptp(template_data.templates, 1).max(1)
219
+ template_snrs = template_ptps * np.sqrt(template_data.spike_counts)
220
+ good_templates = np.logical_and(
221
+ template_data.spike_counts >= min_n_spikes,
222
+ template_snrs > min_template_snr,
223
+ )
224
+ return good_templates
225
+
226
+
227
+ def reorder_by_depth(sorting, template_data):
228
+ assert template_data.registered_geom is not None
229
+ w = template_data.snrs_by_channel()
230
+ w /= w.sum(axis=1, keepdims=True)
231
+ meanz = np.sum(template_data.registered_geom[:, 1] * w, axis=1)
232
+
233
+ # new_to_old[i] = old id for new id i
234
+ new_to_old = np.argsort(meanz, kind="stable")
235
+ # old_to_new[i] = new id for old id i
236
+ old_to_new = np.argsort(new_to_old, kind="stable")
237
+
238
+ if template_data.properties:
239
+ properties = {k: v[new_to_old] for k, v in template_data.properties.items()}
240
+ else:
241
+ properties = {}
242
+
243
+ valid = np.flatnonzero(sorting.labels >= 0)
244
+ labels = np.full_like(sorting.labels, -1)
245
+ labels[valid] = old_to_new[sorting.labels[valid]]
246
+ sorting = sorting.ephemeral_replace(labels=labels)
247
+
248
+ uids = np.arange(len(new_to_old))
249
+ scbc = template_data.raw_std_dev
250
+ if scbc is not None:
251
+ scbc = scbc[new_to_old]
252
+ rsd = template_data.raw_std_dev
253
+ if rsd is not None:
254
+ rsd = rsd[new_to_old]
255
+ template_data = TemplateData(
256
+ templates=template_data.templates[new_to_old],
257
+ unit_ids=uids,
258
+ spike_counts=template_data.spike_counts[new_to_old],
259
+ spike_counts_by_channel=scbc,
260
+ raw_std_dev=rsd,
261
+ registered_geom=template_data.registered_geom,
262
+ trough_offset_samples=template_data.trough_offset_samples,
263
+ properties=properties,
264
+ )
265
+ return sorting, template_data
266
+
267
+
268
+ def ensure_save(template_data, template_npz_path):
269
+ if template_npz_path is not None:
270
+ template_npz_path.parent.parent.mkdir(exist_ok=True)
271
+ template_npz_path.parent.mkdir(exist_ok=True)
272
+ template_data.to_npz(template_npz_path)
273
+ return template_data
274
+
275
+
276
+ def _check_still_valid(sorting: DARTsortSorting):
277
+ assert sorting.labels is not None
278
+ if (sorting.labels < 0).all():
279
+ raise ValueError("All units were thrown away during template postprocessing.")
280
+
281
+
282
+ def _handle_merge(
283
+ *,
284
+ recording: BaseRecording,
285
+ sorting: DARTsortSorting,
286
+ motion_est,
287
+ template_data: TemplateData,
288
+ merge_cfg: TemplateMergeConfig | None,
289
+ computation_cfg: ComputationConfig,
290
+ waveform_cfg: WaveformConfig,
291
+ template_cfg: TemplateConfig,
292
+ ) -> tuple[DARTsortSorting, TemplateData]:
293
+ if merge_cfg is None or not merge_cfg.merge_distance_threshold:
294
+ return sorting, template_data
295
+
296
+ from ..clustering.merge import merge_templates
297
+
298
+ merge_shift_samples = waveform_cfg.ms_to_samples(merge_cfg.max_shift_ms)
299
+ merge_res = merge_templates(
300
+ sorting=sorting,
301
+ template_data=template_data,
302
+ max_shift_samples=merge_shift_samples,
303
+ linkage=merge_cfg.linkage,
304
+ merge_distance_threshold=merge_cfg.merge_distance_threshold,
305
+ temporal_upsampling_factor=merge_cfg.temporal_upsampling_factor,
306
+ amplitude_scaling_variance=merge_cfg.amplitude_scaling_variance,
307
+ amplitude_scaling_boundary=merge_cfg.amplitude_scaling_boundary,
308
+ svd_compression_rank=merge_cfg.svd_compression_rank,
309
+ min_spatial_cosine=merge_cfg.min_spatial_cosine,
310
+ computation_cfg=computation_cfg,
311
+ show_progress=True,
312
+ )
313
+ sorting = merge_res["sorting"]
314
+ new_unit_ids = merge_res["new_unit_ids"]
315
+ del merge_res
316
+ assert sorting.labels is not None
317
+
318
+ # determine which units were merged and recompute only those templates
319
+ ul, ui, uc = np.unique(new_unit_ids, return_index=True, return_counts=True)
320
+ n_merged_units = ul.shape[0]
321
+ assert np.array_equal(ul, np.arange(len(ul)))
322
+ needs_recompute = ul[uc > 1]
323
+ if needs_recompute.size:
324
+ recompute_labels = np.where(
325
+ np.isin(sorting.labels, needs_recompute), sorting.labels, -1
326
+ )
327
+ recompute_sorting = sorting.ephemeral_replace(labels=recompute_labels)
328
+ # turn off merge here
329
+ recompute_sorting, recompute_template_data = estimate_template_library(
330
+ recording=recording,
331
+ sorting=recompute_sorting,
332
+ motion_est=motion_est,
333
+ min_template_snr=0.0,
334
+ min_template_count=0,
335
+ waveform_cfg=waveform_cfg,
336
+ template_cfg=template_cfg,
337
+ realign_cfg=None,
338
+ template_merge_cfg=None,
339
+ tsvd=template_data.tsvd,
340
+ computation_cfg=computation_cfg,
341
+ depth_order=False,
342
+ )
343
+ assert len(recompute_template_data.templates) == needs_recompute.size
344
+ assert (
345
+ recompute_template_data.trough_offset_samples
346
+ == template_data.trough_offset_samples
347
+ )
348
+ assert (
349
+ recompute_template_data.spike_length_samples
350
+ == template_data.spike_length_samples
351
+ )
352
+ else:
353
+ recompute_template_data = None
354
+
355
+ # new indices corresponding to kept units
356
+ new_kept_ixs = np.flatnonzero(uc <= 1)
357
+ # original indices corresponding to kept units
358
+ old_kept_ixs = ui[new_kept_ixs]
359
+ # new indices for recomputed units
360
+ new_recompute_ix = np.flatnonzero(uc > 1)
361
+ # original indices corresponding to recomputed units
362
+ old_recompute_ix = ui[new_recompute_ix]
363
+
364
+ logger.dartsortverbose(f"Merged template indices: {new_recompute_ix.tolist()}")
365
+
366
+ # pack up the the templates
367
+ templates = np.empty(
368
+ (n_merged_units, *template_data.templates.shape[1:]),
369
+ dtype=template_data.templates.dtype,
370
+ )
371
+ templates[new_kept_ixs] = template_data.templates[old_kept_ixs]
372
+ if recompute_template_data is not None:
373
+ templates[new_recompute_ix] = recompute_template_data.templates
374
+
375
+ spike_counts = np.empty(
376
+ (n_merged_units, *template_data.spike_counts.shape[1:]),
377
+ dtype=template_data.spike_counts.dtype,
378
+ )
379
+ spike_counts[new_kept_ixs] = template_data.spike_counts[old_kept_ixs]
380
+ if recompute_template_data is not None:
381
+ spike_counts[new_recompute_ix] = recompute_template_data.spike_counts
382
+
383
+ if template_data.spike_counts_by_channel is not None:
384
+ spike_counts_by_channel = np.empty(
385
+ (n_merged_units, *template_data.spike_counts_by_channel.shape[1:]),
386
+ dtype=template_data.spike_counts_by_channel.dtype,
387
+ )
388
+ spike_counts_by_channel[new_kept_ixs] = template_data.spike_counts_by_channel[
389
+ old_kept_ixs
390
+ ]
391
+ if recompute_template_data is not None:
392
+ spike_counts_by_channel[new_recompute_ix] = (
393
+ recompute_template_data.spike_counts_by_channel
394
+ )
395
+ else:
396
+ spike_counts_by_channel = None
397
+
398
+ if template_data.raw_std_dev is not None:
399
+ raw_std_dev = np.empty(
400
+ (n_merged_units, *template_data.raw_std_dev.shape[1:]),
401
+ dtype=template_data.raw_std_dev.dtype,
402
+ )
403
+ raw_std_dev[new_kept_ixs] = template_data.raw_std_dev[old_kept_ixs]
404
+ if recompute_template_data is not None:
405
+ raw_std_dev[new_recompute_ix] = recompute_template_data.raw_std_dev
406
+ else:
407
+ raw_std_dev = None
408
+
409
+ template_data = TemplateData(
410
+ templates,
411
+ unit_ids=np.arange(len(templates)),
412
+ spike_counts=spike_counts,
413
+ spike_counts_by_channel=spike_counts_by_channel,
414
+ raw_std_dev=raw_std_dev,
415
+ registered_geom=template_data.registered_geom,
416
+ trough_offset_samples=template_data.trough_offset_samples,
417
+ )
418
+ return sorting, template_data
419
+
420
+
421
+ def _quick_mean_templates(
422
+ recording, sorting, waveform_cfg, computation_cfg, motion_est
423
+ ):
424
+ return TemplateData.from_config(
425
+ recording=recording,
426
+ sorting=sorting,
427
+ motion_est=motion_est,
428
+ waveform_cfg=waveform_cfg,
429
+ template_cfg=TemplateConfig(denoising_method="none"),
430
+ computation_cfg=computation_cfg,
431
+ )
432
+
433
+
434
+ def filter_by_unit_mask(
435
+ sorting: DARTsortSorting, keep_mask: np.ndarray, mask_ids: np.ndarray | None = None
436
+ ) -> DARTsortSorting:
437
+ assert sorting.labels is not None
438
+
439
+ if mask_ids is not None:
440
+ assert mask_ids.shape == keep_mask.shape
441
+ k_full = mask_ids.max() + 1
442
+ assert k_full >= keep_mask.shape[0]
443
+ mask = np.zeros(k_full, dtype=bool)
444
+ mask[mask_ids[keep_mask]] = True
445
+ keep_mask = mask
446
+
447
+ discard_mask = np.logical_not(keep_mask)
448
+ if not discard_mask.any():
449
+ return sorting
450
+
451
+ valid = np.flatnonzero(sorting.labels >= 0)
452
+ chuck = valid[discard_mask[sorting.labels[valid]]]
453
+ sorting.labels[chuck] = -1
454
+
455
+ return sorting.flatten()
456
+
457
+
458
+ def flag_possible_cc_error_spikes(
459
+ sorting: DARTsortSorting,
460
+ subtraction_cfg,
461
+ amplitudes_dataset_name="denoised_ptp_amplitudes",
462
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
463
+ from ..util.py_util import timer
464
+
465
+ times = sorting.times_samples
466
+ channels = sorting.channels
467
+ amps = getattr(sorting, amplitudes_dataset_name)
468
+ xy = sorting.geom[channels] # type: ignore
469
+ n = len(times)
470
+
471
+ # rescale units so that the max allowed temporal and spatial dists are 10
472
+ times = times / subtraction_cfg.temporal_dedup_radius_samples
473
+ xy_subtract = xy / subtraction_cfg.subtract_radius
474
+ kdt_subtract = KDTree(np.c_[times, xy_subtract])
475
+
476
+ # get all possible neighbors
477
+ with timer("a"):
478
+ sdm = kdt_subtract.sparse_distance_matrix(
479
+ kdt_subtract, max_distance=1.0, p=np.inf, output_type="ndarray"
480
+ )
481
+ # this is more an adjacency matrix. 0s will be discarded later.
482
+ v = np.ones(sdm["i"].shape, dtype=np.float32)
483
+ coo = coo_array((v, (sdm["i"], sdm["j"])), shape=(n, n), dtype=v.dtype)
484
+ print(f"{coo=}")
485
+
486
+ # remove the exclusions by marking their distance as large
487
+ if subtraction_cfg.spatial_dedup_radius:
488
+ xy_dedup = xy / subtraction_cfg.spatial_dedup_radius
489
+ kdt_dedup = KDTree(np.c_[times, xy_dedup])
490
+ with timer("b"):
491
+ sdm_dedup = kdt_dedup.sparse_distance_matrix(
492
+ kdt_dedup, max_distance=1.0, p=np.inf, output_type="ndarray"
493
+ )
494
+ del kdt_dedup, xy_dedup
495
+ v = np.ones(sdm_dedup["i"].shape, dtype=np.float32)
496
+ coo_dedup = coo_array(
497
+ (v, (sdm_dedup["i"], sdm_dedup["j"])), shape=(n, n), dtype=v.dtype
498
+ )
499
+ print(f"{coo_dedup=}")
500
+ coo = coo - coo_dedup
501
+ coo = coo.tocoo()
502
+ print(f"{coo=}")
503
+
504
+ # dedup neighbors are removed
505
+ coo.sum_duplicates()
506
+ coo.eliminate_zeros()
507
+
508
+ # apply higher amplitude criterion
509
+ bigger = np.flatnonzero(amps[coo.coords[0]] > amps[coo.coords[1]])
510
+ coo.data[bigger] = 0.0
511
+ coo.eliminate_zeros()
512
+
513
+ # i am suspicious if i have any neighbors. obviously many spikes do.
514
+ # it's just that a unit consisting entirely of such spikes is suspicious.
515
+ flagged = np.zeros(len(times), dtype=bool)
516
+ ii, jj = coo.coords
517
+ flagged[ii] = True
518
+
519
+ return flagged, ii, jj
520
+
521
+
522
+ def cc_flag_criterion(
523
+ sorting: DARTsortSorting,
524
+ detection_cfg: Any | None = None,
525
+ max_cc_flag_rate=1.0,
526
+ cc_flag_entropy_cutoff=0.0,
527
+ amplitudes_dataset_name="denoised_ptp_amplitudes",
528
+ ) -> np.ndarray | None:
529
+ if max_cc_flag_rate == 1.0:
530
+ return None
531
+ if not isinstance(detection_cfg, SubtractionConfig):
532
+ # this step applied only at subtraction to clean up nn error units
533
+ return None
534
+
535
+ flagged, ii, jj = flag_possible_cc_error_spikes(
536
+ sorting=sorting,
537
+ subtraction_cfg=detection_cfg,
538
+ amplitudes_dataset_name=amplitudes_dataset_name,
539
+ )
540
+ assert sorting.labels is not None
541
+ li = sorting.labels[ii]
542
+ lj = sorting.labels[jj]
543
+
544
+ rate = np.zeros(sorting.unit_ids.shape)
545
+ entropy = np.zeros_like(rate)
546
+ for j, u in enumerate(sorting.unit_ids):
547
+ inu = np.flatnonzero(sorting.labels == u)
548
+ rate[j] = flagged[inu].mean()
549
+ _, friend_count = np.unique(lj[li == u], return_counts=True)
550
+ friend_p = friend_count / friend_count.sum()
551
+ entropy[j] = -(np.log(friend_p) * friend_p).sum()
552
+
553
+ bad = (rate > max_cc_flag_rate) & (entropy < cc_flag_entropy_cutoff)
554
+ logger.dartsortdebug(f"CCG peak criterion flagged {bad.sum().item()} units.")
555
+ return np.logical_not(bad)