DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,765 @@
1
+ import gc
2
+ from typing import cast, Literal, Self
3
+
4
+ import numpy as np
5
+ import sklearn.cluster
6
+ import torch
7
+ from spikeinterface.core import BaseRecording
8
+
9
+ from ..util.data_util import chunk_time_ranges, fit_reweighting, DARTsortSorting
10
+ from ..util.main_util import ds_save_intermediate_labels
11
+ from ..util.internal_config import (
12
+ ClusteringConfig,
13
+ RefinementConfig,
14
+ FitSamplingConfig,
15
+ ComputationConfig,
16
+ )
17
+ from .clustering_features import SimpleMatrixFeatures
18
+ from ..util import job_util
19
+ from . import cluster_util, density, forward_backward, refine_util
20
+ from .gmm import mixture
21
+
22
+
23
+ clustering_strategies: dict[str, "type[Clusterer]"] = {}
24
+ refinement_strategies: dict[str, "type[Refinement]"] = {}
25
+
26
+
27
+ def get_clusterer(
28
+ clustering_cfg: ClusteringConfig | None = None,
29
+ refinement_cfgs: list[RefinementConfig | None] | None = None,
30
+ computation_cfg: ComputationConfig | None = None,
31
+ save_cfg=None,
32
+ save_labels_dir=None,
33
+ initial_name=None,
34
+ refine_labels_fmt=None,
35
+ ):
36
+ if clustering_cfg is not None:
37
+ if clustering_cfg.cluster_strategy not in clustering_strategies:
38
+ raise ValueError(
39
+ f"Unknown cluster_strategy={clustering_cfg.cluster_strategy}. "
40
+ f"Options are: {', '.join(clustering_strategies.keys())}."
41
+ )
42
+ clus_strategy = clustering_cfg.cluster_strategy
43
+ else:
44
+ clus_strategy = "none"
45
+
46
+ saving_labels = save_cfg is not None and save_cfg.save_intermediate_labels
47
+ if saving_labels:
48
+ shared_save_kw = dict(
49
+ save_labels_dir=save_labels_dir,
50
+ save_cfg=save_cfg,
51
+ )
52
+ else:
53
+ shared_save_kw = dict(
54
+ save_labels_dir=None,
55
+ save_cfg=None,
56
+ )
57
+
58
+ C = clustering_strategies[clus_strategy]
59
+ init_fmt = initial_name if (saving_labels and clustering_cfg is not None) else None
60
+ clusterer = C.from_config(
61
+ clustering_cfg,
62
+ labels_fmt=init_fmt,
63
+ computation_cfg=computation_cfg,
64
+ **shared_save_kw,
65
+ )
66
+
67
+ refinement_cfgs = refinement_cfgs or []
68
+ for j, r_cfg in enumerate(refinement_cfgs):
69
+ if r_cfg is None:
70
+ continue
71
+
72
+ strat = r_cfg.refinement_strategy
73
+ if strat not in refinement_strategies:
74
+ raise ValueError(
75
+ f"Unknown refinement_strategy={strat}. "
76
+ f"Options are: {', '.join(refinement_strategies.keys())}."
77
+ )
78
+
79
+ R = refinement_strategies[strat]
80
+ rsave = saving_labels and strat != "none"
81
+ if rsave and refine_labels_fmt is not None:
82
+ stepfmt = f"_{j}_{strat}_{{stepname}}"
83
+ r_fmt = refine_labels_fmt.format(stepname=stepfmt)
84
+ else:
85
+ r_fmt = None
86
+
87
+ clusterer = R(
88
+ clusterer,
89
+ refinement_cfg=r_cfg,
90
+ labels_fmt=r_fmt,
91
+ computation_cfg=computation_cfg,
92
+ **shared_save_kw,
93
+ )
94
+
95
+ return clusterer
96
+
97
+
98
+ class Clusterer:
99
+ def __init__(
100
+ self,
101
+ computation_cfg: ComputationConfig | None = None,
102
+ sampling_cfg: FitSamplingConfig | None = None,
103
+ save_cfg=None,
104
+ save_labels_dir=None,
105
+ labels_fmt=None,
106
+ ):
107
+ self.computation_cfg = computation_cfg
108
+ self.sampling_cfg = sampling_cfg
109
+ if computation_cfg is None:
110
+ self.computation_cfg = job_util.get_global_computation_config()
111
+ self.save_cfg = save_cfg
112
+ self.save_labels_dir = save_labels_dir
113
+ self.labels_fmt = labels_fmt
114
+
115
+ @classmethod
116
+ def from_config(
117
+ cls,
118
+ clustering_cfg: ClusteringConfig | None,
119
+ computation_cfg: ComputationConfig | None = None,
120
+ save_cfg=None,
121
+ save_labels_dir=None,
122
+ labels_fmt=None,
123
+ ) -> Self:
124
+ if clustering_cfg is None:
125
+ sampling_cfg = None
126
+ else:
127
+ sampling_cfg = clustering_cfg.sampling_cfg
128
+ return cls(
129
+ computation_cfg=computation_cfg,
130
+ save_cfg=save_cfg,
131
+ save_labels_dir=save_labels_dir,
132
+ labels_fmt=labels_fmt,
133
+ sampling_cfg=sampling_cfg,
134
+ )
135
+
136
+ def handle_sampling(
137
+ self, features: SimpleMatrixFeatures
138
+ ) -> tuple[Literal[False], slice] | tuple[Literal[True], np.ndarray]:
139
+ if self.sampling_cfg is None:
140
+ return False, slice(None)
141
+ elif features.n <= self.sampling_cfg.n_waveforms_fit:
142
+ return False, slice(None)
143
+
144
+ weights = fit_reweighting(
145
+ voltages=features.signed_amplitudes,
146
+ fit_sampling=self.sampling_cfg.fit_sampling,
147
+ fit_max_reweighting=self.sampling_cfg.fit_max_reweighting,
148
+ )
149
+ rg = np.random.default_rng(self.sampling_cfg.fit_subsampling_random_state)
150
+ ixs = rg.choice(
151
+ features.n,
152
+ size=self.sampling_cfg.n_waveforms_fit,
153
+ p=weights,
154
+ replace=False,
155
+ )
156
+ ixs.sort()
157
+ return True, ixs
158
+
159
+ def cluster(
160
+ self,
161
+ features: SimpleMatrixFeatures,
162
+ sorting: DARTsortSorting,
163
+ recording: BaseRecording,
164
+ motion_est=None,
165
+ ) -> DARTsortSorting:
166
+ if features is None:
167
+ pass
168
+ else:
169
+ labels = self._cluster(features, sorting, recording, motion_est)
170
+ sorting = sorting.ephemeral_replace(labels=labels)
171
+ if self.labels_fmt and self.save_labels_dir is not None:
172
+ assert "{" not in self.labels_fmt
173
+ assert "}" not in self.labels_fmt
174
+ ds_save_intermediate_labels(
175
+ self.labels_fmt, sorting, self.save_labels_dir, self.save_cfg
176
+ )
177
+ return sorting
178
+
179
+ def _cluster(
180
+ self,
181
+ features: SimpleMatrixFeatures,
182
+ sorting: DARTsortSorting,
183
+ recording: BaseRecording,
184
+ motion_est=None,
185
+ ) -> np.ndarray:
186
+ """Unused method but shows API."""
187
+ del features, recording, motion_est
188
+ assert sorting.labels is not None
189
+ return sorting.labels
190
+
191
+
192
+ clustering_strategies["none"] = Clusterer
193
+
194
+
195
+ class ChannelSnapClusterer(Clusterer):
196
+ def _cluster(
197
+ self,
198
+ features: SimpleMatrixFeatures,
199
+ sorting: DARTsortSorting,
200
+ recording: BaseRecording,
201
+ motion_est=None,
202
+ ) -> np.ndarray:
203
+ return cluster_util.closest_registered_channels(
204
+ times_seconds=sorting.times_seconds, # type: ignore
205
+ x=features.x,
206
+ z_abs=features.z,
207
+ z_reg=features.z_reg,
208
+ geom=recording.get_channel_locations(),
209
+ motion_est=motion_est,
210
+ )
211
+
212
+
213
+ clustering_strategies["channel_snap"] = ChannelSnapClusterer
214
+
215
+
216
+ class GridSnapClusterer(Clusterer):
217
+ def __init__(self, grid_dx=15.0, grid_dz=15.0, **kwargs):
218
+ super().__init__(**kwargs)
219
+ self.grid_dx = grid_dx
220
+ self.grid_dz = grid_dz
221
+
222
+ @classmethod
223
+ def from_config(
224
+ cls,
225
+ clustering_cfg: ClusteringConfig | None,
226
+ computation_cfg: ComputationConfig | None = None,
227
+ save_cfg=None,
228
+ save_labels_dir=None,
229
+ labels_fmt=None,
230
+ ) -> Self:
231
+ assert clustering_cfg is not None
232
+ return cls(
233
+ grid_dx=clustering_cfg.grid_dx,
234
+ grid_dz=clustering_cfg.grid_dz,
235
+ computation_cfg=computation_cfg,
236
+ save_cfg=save_cfg,
237
+ save_labels_dir=save_labels_dir,
238
+ labels_fmt=labels_fmt,
239
+ sampling_cfg=clustering_cfg.sampling_cfg,
240
+ )
241
+
242
+ def _cluster(
243
+ self,
244
+ features: SimpleMatrixFeatures,
245
+ sorting: DARTsortSorting,
246
+ recording: BaseRecording,
247
+ motion_est=None,
248
+ ) -> np.ndarray:
249
+ return cluster_util.grid_snap(
250
+ times_seconds=sorting.times_seconds, # type: ignore
251
+ x=features.x,
252
+ z_abs=features.z,
253
+ z_reg=features.z_reg,
254
+ grid_dx=self.grid_dx,
255
+ grid_dz=self.grid_dz,
256
+ geom=recording.get_channel_locations(),
257
+ motion_est=motion_est,
258
+ )
259
+
260
+
261
+ clustering_strategies["grid_snap"] = GridSnapClusterer
262
+
263
+
264
+ class DensityPeaksClusterer(Clusterer):
265
+ def __init__(
266
+ self,
267
+ knn_k=None,
268
+ sigma_local=5.0,
269
+ sigma_regional: float | None = 25.0,
270
+ n_neighbors_search=20,
271
+ radius_search=25.0,
272
+ remove_clusters_smaller_than=50,
273
+ noise_density=0.0,
274
+ outlier_radius=5.0,
275
+ outlier_neighbor_count=5,
276
+ workers=-1,
277
+ uhdversion=False,
278
+ random_seed=0,
279
+ **kwargs,
280
+ ):
281
+ super().__init__(**kwargs)
282
+ self.knn_k = knn_k
283
+ self.uhdversion = uhdversion
284
+ self.sigma_local = sigma_local
285
+ self.sigma_regional = sigma_regional
286
+ self.n_neighbors_search = n_neighbors_search
287
+ self.radius_search = radius_search
288
+ self.remove_clusters_smaller_than = remove_clusters_smaller_than
289
+ self.noise_density = noise_density
290
+ self.outlier_radius = outlier_radius
291
+ self.outlier_neighbor_count = outlier_neighbor_count
292
+ self.workers = workers
293
+ self.uhdversion = uhdversion
294
+ self.random_seed = random_seed
295
+
296
+ @classmethod
297
+ def from_config(
298
+ cls,
299
+ clustering_cfg: ClusteringConfig | None,
300
+ computation_cfg: ComputationConfig | None = None,
301
+ save_cfg=None,
302
+ save_labels_dir=None,
303
+ labels_fmt=None,
304
+ ) -> Self:
305
+ assert clustering_cfg is not None
306
+ uhdversion = clustering_cfg.cluster_strategy == "density_peaks_uhdversion"
307
+ return cls(
308
+ knn_k=clustering_cfg.knn_k,
309
+ sigma_local=clustering_cfg.sigma_local,
310
+ sigma_regional=clustering_cfg.sigma_regional,
311
+ n_neighbors_search=clustering_cfg.n_neighbors_search,
312
+ radius_search=clustering_cfg.radius_search,
313
+ remove_clusters_smaller_than=clustering_cfg.min_cluster_size,
314
+ noise_density=clustering_cfg.noise_density,
315
+ random_seed=clustering_cfg.random_seed,
316
+ outlier_radius=clustering_cfg.outlier_radius,
317
+ outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
318
+ workers=clustering_cfg.workers,
319
+ uhdversion=uhdversion,
320
+ computation_cfg=computation_cfg,
321
+ save_cfg=save_cfg,
322
+ save_labels_dir=save_labels_dir,
323
+ labels_fmt=labels_fmt,
324
+ sampling_cfg=clustering_cfg.sampling_cfg,
325
+ )
326
+
327
+ def _cluster(
328
+ self,
329
+ features: SimpleMatrixFeatures,
330
+ sorting: DARTsortSorting,
331
+ recording: BaseRecording,
332
+ motion_est=None,
333
+ ) -> np.ndarray:
334
+ subsampling, ixs = self.handle_sampling(features)
335
+ X = features.features
336
+ X_fit = X[ixs]
337
+
338
+ if not self.uhdversion:
339
+ res = density.density_peaks(
340
+ X_fit,
341
+ knn_k=self.knn_k,
342
+ sigma_local=self.sigma_local,
343
+ sigma_regional=self.sigma_regional,
344
+ n_neighbors_search=self.n_neighbors_search,
345
+ radius_search=self.radius_search,
346
+ remove_clusters_smaller_than=0,
347
+ noise_density=self.noise_density,
348
+ outlier_radius=self.outlier_radius,
349
+ outlier_neighbor_count=self.outlier_neighbor_count,
350
+ workers=self.workers,
351
+ )
352
+ else:
353
+ res = density.density_peaks_fancy(
354
+ features.xyza,
355
+ features.amplitudes,
356
+ sorting,
357
+ motion_est,
358
+ recording.get_channel_locations(),
359
+ sigma_local=self.sigma_local,
360
+ sigma_regional=self.sigma_regional,
361
+ n_neighbors_search=self.n_neighbors_search,
362
+ radius_search=self.radius_search,
363
+ remove_clusters_smaller_than=self.remove_clusters_smaller_than,
364
+ noise_density=self.noise_density,
365
+ outlier_radius=self.outlier_radius,
366
+ outlier_neighbor_count=self.outlier_neighbor_count,
367
+ workers=self.workers,
368
+ )
369
+
370
+ if subsampling:
371
+ kdtree = res["kdtree"]
372
+ assert isinstance(ixs, np.ndarray)
373
+ rest = np.setdiff1d(np.arange(len(X)), ixs)
374
+ other_labels = density.nearest_neighbor_assign(
375
+ kdtree,
376
+ res["labels"],
377
+ X[rest],
378
+ radius_search=self.radius_search,
379
+ workers=self.workers,
380
+ )
381
+ labels = cluster_util.combine_disjoint(
382
+ ixs, res["labels"], rest, other_labels
383
+ )
384
+ else:
385
+ labels = res["labels"]
386
+
387
+ labels = cluster_util.decrumb(
388
+ labels, min_size=self.remove_clusters_smaller_than, in_place=True
389
+ )
390
+
391
+ return labels
392
+
393
+
394
+ clustering_strategies["dpc"] = DensityPeaksClusterer
395
+ clustering_strategies["density_peaks_uhdversion"] = DensityPeaksClusterer
396
+
397
+
398
+ class GMMDensityPeaksClusterer(Clusterer):
399
+ def __init__(
400
+ self,
401
+ outlier_neighbor_count=10,
402
+ outlier_radius=25.0,
403
+ remove_clusters_smaller_than=50,
404
+ workers=-1,
405
+ n_initializations=5,
406
+ n_iter=50,
407
+ max_components_per_channel=20,
408
+ min_spikes_per_component=10,
409
+ random_state=0,
410
+ use_hellinger=True,
411
+ kmeanspp_min_dist=5.0,
412
+ hellinger_cutoff=0.8,
413
+ hellinger_strong=0.0,
414
+ hellinger_weak=0.0,
415
+ mop=False,
416
+ n_neighbors_search=20,
417
+ max_sigma=5.0,
418
+ max_samples=2_000_000,
419
+ **kwargs,
420
+ ):
421
+ super().__init__(**kwargs)
422
+ self.outlier_neighbor_count = outlier_neighbor_count
423
+ self.outlier_radius = outlier_radius
424
+ self.remove_clusters_smaller_than = remove_clusters_smaller_than
425
+ self.workers = workers
426
+ self.n_initializations = n_initializations
427
+ self.n_iter = n_iter
428
+ self.max_components_per_channel = max_components_per_channel
429
+ self.min_spikes_per_component = min_spikes_per_component
430
+ self.random_state = random_state
431
+ self.use_hellinger = use_hellinger
432
+ self.hellinger_cutoff = hellinger_cutoff
433
+ self.hellinger_strong = hellinger_strong
434
+ self.hellinger_weak = hellinger_weak
435
+ self.mop = mop
436
+ self.max_samples = max_samples
437
+ self.n_neighbors_search = n_neighbors_search
438
+
439
+ # unconfigured params
440
+ self.kmeanspp_min_dist = kmeanspp_min_dist
441
+ self.max_sigma = max_sigma
442
+
443
+ @classmethod
444
+ def from_config(
445
+ cls,
446
+ clustering_cfg: ClusteringConfig | None,
447
+ computation_cfg: ComputationConfig | None = None,
448
+ save_cfg=None,
449
+ save_labels_dir=None,
450
+ labels_fmt=None,
451
+ ) -> Self:
452
+ assert clustering_cfg is not None
453
+ return cls(
454
+ outlier_neighbor_count=clustering_cfg.outlier_neighbor_count,
455
+ outlier_radius=clustering_cfg.outlier_radius,
456
+ remove_clusters_smaller_than=clustering_cfg.min_cluster_size,
457
+ workers=clustering_cfg.workers,
458
+ n_initializations=clustering_cfg.kmeanspp_initializations,
459
+ n_iter=clustering_cfg.kmeans_iter,
460
+ max_components_per_channel=clustering_cfg.components_per_channel,
461
+ random_state=clustering_cfg.random_seed,
462
+ use_hellinger=clustering_cfg.use_hellinger,
463
+ hellinger_cutoff=clustering_cfg.component_overlap,
464
+ hellinger_strong=clustering_cfg.hellinger_strong,
465
+ hellinger_weak=clustering_cfg.hellinger_weak,
466
+ mop=clustering_cfg.mop,
467
+ max_sigma=clustering_cfg.gmmdpc_max_sigma,
468
+ max_samples=clustering_cfg.sampling_cfg.n_waveforms_fit,
469
+ n_neighbors_search=clustering_cfg.n_neighbors_search,
470
+ computation_cfg=computation_cfg,
471
+ save_cfg=save_cfg,
472
+ save_labels_dir=save_labels_dir,
473
+ labels_fmt=labels_fmt,
474
+ sampling_cfg=clustering_cfg.sampling_cfg,
475
+ )
476
+
477
+ def _cluster(
478
+ self,
479
+ features: SimpleMatrixFeatures,
480
+ sorting: DARTsortSorting,
481
+ recording: BaseRecording,
482
+ motion_est=None,
483
+ ) -> np.ndarray:
484
+ res = density.gmm_density_peaks(
485
+ X=features.features,
486
+ channels=sorting.channels,
487
+ outlier_neighbor_count=self.outlier_neighbor_count,
488
+ outlier_radius=self.outlier_radius,
489
+ remove_clusters_smaller_than=self.remove_clusters_smaller_than,
490
+ workers=self.workers,
491
+ n_initializations=self.n_initializations,
492
+ n_iter=self.n_iter,
493
+ max_components_per_channel=self.max_components_per_channel,
494
+ min_spikes_per_component=self.min_spikes_per_component,
495
+ random_state=self.random_state,
496
+ kmeanspp_min_dist=self.kmeanspp_min_dist,
497
+ use_hellinger=self.use_hellinger,
498
+ hellinger_cutoff=self.hellinger_cutoff,
499
+ hellinger_strong=self.hellinger_strong,
500
+ hellinger_weak=self.hellinger_weak,
501
+ n_neighbors_search=self.n_neighbors_search,
502
+ mop=self.mop,
503
+ max_sigma=self.max_sigma,
504
+ max_samples=self.max_samples,
505
+ )
506
+ return res["labels"]
507
+
508
+
509
+ clustering_strategies["gmmdpc"] = GMMDensityPeaksClusterer
510
+
511
+
512
+ class RecursiveHDBSCANClusterer(Clusterer):
513
+ def __init__(
514
+ self,
515
+ min_cluster_size=25,
516
+ min_samples=25,
517
+ cluster_selection_epsilon=1,
518
+ recursive=True,
519
+ **kwargs,
520
+ ):
521
+ super().__init__(**kwargs)
522
+ self.min_cluster_size = min_cluster_size
523
+ self.min_samples = min_samples
524
+ self.cluster_selection_epsilon = cluster_selection_epsilon
525
+ self.recursive = recursive
526
+
527
+ @classmethod
528
+ def from_config(
529
+ cls,
530
+ clustering_cfg: ClusteringConfig | None,
531
+ computation_cfg: ComputationConfig | None = None,
532
+ save_cfg=None,
533
+ save_labels_dir=None,
534
+ labels_fmt=None,
535
+ ) -> Self:
536
+ assert clustering_cfg is not None
537
+ return cls(
538
+ min_cluster_size=clustering_cfg.min_cluster_size,
539
+ min_samples=clustering_cfg.min_samples,
540
+ cluster_selection_epsilon=clustering_cfg.cluster_selection_epsilon,
541
+ recursive=clustering_cfg.recursive,
542
+ computation_cfg=computation_cfg,
543
+ save_cfg=save_cfg,
544
+ save_labels_dir=save_labels_dir,
545
+ labels_fmt=labels_fmt,
546
+ sampling_cfg=clustering_cfg.sampling_cfg,
547
+ )
548
+
549
+ def _cluster(
550
+ self,
551
+ features: SimpleMatrixFeatures,
552
+ sorting: DARTsortSorting,
553
+ recording: BaseRecording,
554
+ motion_est=None,
555
+ ) -> np.ndarray:
556
+ return cluster_util.recursive_hdbscan_clustering(
557
+ features.features,
558
+ min_cluster_size=self.min_cluster_size,
559
+ min_samples=self.min_samples,
560
+ cluster_selection_epsilon=self.cluster_selection_epsilon,
561
+ recursive=self.recursive,
562
+ )
563
+
564
+
565
+ class ScikitLearnClusterer(Clusterer):
566
+ def __init__(self, sklearn_class_name="DBSCAN", sklearn_kwargs=None, **kwargs):
567
+ super().__init__(**kwargs)
568
+ self.sklearn_class_name = sklearn_class_name
569
+ self.sklearn_kwargs = sklearn_kwargs or {}
570
+
571
+ @classmethod
572
+ def from_config(
573
+ cls,
574
+ clustering_cfg: ClusteringConfig | None,
575
+ computation_cfg: ComputationConfig | None = None,
576
+ save_cfg=None,
577
+ save_labels_dir=None,
578
+ labels_fmt=None,
579
+ ) -> Self:
580
+ assert clustering_cfg is not None
581
+ return cls(
582
+ sklearn_class_name=clustering_cfg.sklearn_class_name,
583
+ sklearn_kwargs=clustering_cfg.sklearn_kwargs,
584
+ computation_cfg=computation_cfg,
585
+ save_cfg=save_cfg,
586
+ save_labels_dir=save_labels_dir,
587
+ labels_fmt=labels_fmt,
588
+ sampling_cfg=clustering_cfg.sampling_cfg,
589
+ )
590
+
591
+ def _cluster(
592
+ self,
593
+ features: SimpleMatrixFeatures,
594
+ sorting: DARTsortSorting,
595
+ recording: BaseRecording,
596
+ motion_est=None,
597
+ ) -> np.ndarray:
598
+ skcls = getattr(sklearn.cluster, self.sklearn_class_name)
599
+ clus = skcls(**self.sklearn_kwargs)
600
+ return clus.fit_predict(features.features)
601
+
602
+
603
+ clustering_strategies["sklearn"] = ScikitLearnClusterer
604
+
605
+
606
+ class Refinement(Clusterer):
607
+ def __init__(
608
+ self, clusterer: Clusterer, refinement_cfg: RefinementConfig, **kwargs
609
+ ):
610
+ super().__init__(**kwargs)
611
+ self.clusterer = clusterer
612
+ self.refinement_cfg = refinement_cfg
613
+ self.sampling_cfg = refinement_cfg.sampling_cfg
614
+
615
+ def cluster(
616
+ self,
617
+ features: SimpleMatrixFeatures,
618
+ sorting: DARTsortSorting,
619
+ recording: BaseRecording,
620
+ motion_est=None,
621
+ ):
622
+ sorting = self.clusterer.cluster(features, sorting, recording, motion_est)
623
+ sorting = self.refine(features, sorting, recording, motion_est)
624
+ return sorting
625
+
626
+ def _refine(
627
+ self,
628
+ features: SimpleMatrixFeatures,
629
+ sorting: DARTsortSorting,
630
+ recording: BaseRecording,
631
+ motion_est=None,
632
+ ):
633
+ del features, recording, motion_est
634
+ return sorting
635
+
636
+ def refine(
637
+ self,
638
+ features: SimpleMatrixFeatures,
639
+ sorting: DARTsortSorting,
640
+ recording: BaseRecording,
641
+ motion_est=None,
642
+ ):
643
+ sorting = self._refine(features, sorting, recording, motion_est)
644
+ if self.labels_fmt and self.save_labels_dir is not None:
645
+ labels_fmt = self.labels_fmt.format(stepname="")
646
+ ds_save_intermediate_labels(
647
+ labels_fmt, sorting, self.save_labels_dir, self.save_cfg
648
+ )
649
+ return sorting
650
+
651
+
652
+ refinement_strategies["none"] = Refinement
653
+
654
+
655
+ class TMMRefinement(Refinement):
656
+ def _refine(
657
+ self,
658
+ features: SimpleMatrixFeatures,
659
+ sorting: DARTsortSorting,
660
+ recording: BaseRecording,
661
+ motion_est=None,
662
+ ):
663
+ assert features is not None
664
+ subsampling, ixs = self.handle_sampling(features)
665
+ ixs = cast(np.ndarray, ixs) if subsampling else None
666
+ sorting = mixture.tmm_demix(
667
+ sorting=sorting,
668
+ motion_est=motion_est,
669
+ refinement_cfg=self.refinement_cfg,
670
+ computation_cfg=self.computation_cfg,
671
+ fit_indices=ixs,
672
+ save_step_labels_format=self.labels_fmt,
673
+ save_step_labels_dir=self.save_labels_dir,
674
+ save_cfg=self.save_cfg,
675
+ )
676
+ gc.collect()
677
+ torch.cuda.empty_cache()
678
+ return sorting
679
+
680
+
681
+ refinement_strategies["tmm"] = TMMRefinement
682
+
683
+
684
+ class SplitMergeRefinement(Refinement):
685
+ def _refine(
686
+ self,
687
+ features: SimpleMatrixFeatures,
688
+ sorting: DARTsortSorting,
689
+ recording: BaseRecording,
690
+ motion_est=None,
691
+ ):
692
+ return refine_util.split_merge(
693
+ recording=recording,
694
+ sorting=sorting,
695
+ motion_est=motion_est,
696
+ split_cfg=self.refinement_cfg.split_cfg,
697
+ merge_cfg=self.refinement_cfg.merge_cfg,
698
+ merge_template_cfg=self.refinement_cfg.merge_template_cfg,
699
+ computation_cfg=self.computation_cfg,
700
+ )
701
+
702
+
703
+ # code too old. todo.
704
+ # refinement_strategies["splitmerge"] = SplitMergeRefinement
705
+
706
+
707
+ class PCMergeRefinement(Refinement):
708
+ def _refine(
709
+ self,
710
+ features: SimpleMatrixFeatures,
711
+ sorting: DARTsortSorting,
712
+ recording: BaseRecording,
713
+ motion_est=None,
714
+ ):
715
+ return refine_util.pc_merge(
716
+ sorting,
717
+ refinement_cfg=self.refinement_cfg,
718
+ motion_est=motion_est,
719
+ computation_cfg=self.computation_cfg,
720
+ )
721
+
722
+
723
+ refinement_strategies["pcmerge"] = PCMergeRefinement
724
+
725
+
726
+ class ForwardBackwardEnsembler(Refinement):
727
+ """If there are more time chunk ones, make a new ABC with this logic."""
728
+
729
+ def cluster(
730
+ self,
731
+ features: SimpleMatrixFeatures,
732
+ sorting: DARTsortSorting,
733
+ recording: BaseRecording,
734
+ motion_est=None,
735
+ ):
736
+ chunk_length_samples = (
737
+ recording.sampling_frequency * self.refinement_cfg.chunk_size_s
738
+ )
739
+ chunk_time_ranges_s = chunk_time_ranges(recording, chunk_length_samples)
740
+ times_seconds = sorting.times_seconds # type: ignore
741
+ assert features is not None
742
+
743
+ chunk_sortings = []
744
+ for lo, hi in chunk_time_ranges_s:
745
+ mask = np.flatnonzero(times_seconds == times_seconds.clip(lo, hi))
746
+ s = sorting.mask(mask)
747
+ f = features.mask(mask)
748
+ l = self.clusterer._cluster(f, s, recording, motion_est)
749
+ labels = np.full_like(sorting.labels, -1)
750
+ labels[mask] = l
751
+ chunk_sortings.append(sorting.ephemeral_replace(labels=labels))
752
+
753
+ labels = forward_backward.forward_backward(
754
+ chunk_time_ranges_s,
755
+ chunk_sortings,
756
+ log_c=self.refinement_cfg.log_c,
757
+ feature_scales=self.refinement_cfg.feature_scales,
758
+ adaptive_feature_scales=self.refinement_cfg.adaptive_feature_scales,
759
+ motion_est=motion_est,
760
+ )
761
+ sorting = sorting.ephemeral_replace(labels=labels)
762
+ return sorting
763
+
764
+
765
+ refinement_strategies["forwardbackward"] = ForwardBackwardEnsembler