DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1285 @@
1
+ from dataclasses import dataclass, replace
2
+ from pathlib import Path
3
+ from typing import Optional
4
+
5
+ import h5py
6
+ import numpy as np
7
+ from dartsort.util import drift_util, waveform_util
8
+ from dartsort.util.data_util import (
9
+ DARTsortSorting,
10
+ batched_h5_read,
11
+ get_featurization_pipeline,
12
+ )
13
+ from dartsort.util.multiprocessing_util import get_pool
14
+
15
+ try:
16
+ from hdbscan import HDBSCAN
17
+ except ImportError:
18
+ from sklearn.cluster import HDBSCAN
19
+ from scipy.spatial.distance import cdist, pdist
20
+ from sklearn.decomposition import PCA
21
+
22
+ # from sklearn.metrics.pairwise import nan_euclidean_distances
23
+ from tqdm.auto import tqdm
24
+
25
+ from . import cluster_util, density, relocate
26
+ from .forward_backward import forward_backward
27
+
28
+
29
+ def split_clusters(
30
+ sorting,
31
+ split_strategy="FeatureSplit",
32
+ split_strategy_kwargs=None,
33
+ recursive=False,
34
+ split_big=False,
35
+ split_big_kw=dict(dz=40, dx=48, min_size_split=50),
36
+ motion_est=None,
37
+ show_progress=True,
38
+ depth_order=True,
39
+ n_jobs=0,
40
+ ):
41
+ """Parallel split step runner function
42
+
43
+ Arguments
44
+ ---------
45
+ sorting : DARTsortSorting
46
+ split_step_class_name : str
47
+ One of split_steps_by_class_name.keys()
48
+ split_step_kwargs : dictionary
49
+ recursive : bool
50
+ If True, attempt to split newly created clusters again
51
+ show_progress : bool
52
+ n_jobs : int
53
+
54
+ Returns
55
+ -------
56
+ split_sorting : DARTsortSorting
57
+ """
58
+ # initialize split state
59
+ labels = sorting.labels.copy()
60
+ labels_to_process = np.unique(labels)
61
+ labels_to_process = list(labels_to_process[labels_to_process > 0])
62
+ cur_max_label = max(labels_to_process)
63
+
64
+ if split_strategy_kwargs is None:
65
+ split_strategy_kwargs = {}
66
+ if motion_est is not None:
67
+ split_strategy_kwargs["motion_est"] = motion_est
68
+
69
+ n_jobs, Executor, context = get_pool(n_jobs)
70
+ with Executor(
71
+ max_workers=n_jobs,
72
+ mp_context=context,
73
+ initializer=_split_job_init,
74
+ initargs=(
75
+ split_strategy,
76
+ sorting.parent_h5_path,
77
+ split_strategy_kwargs,
78
+ ),
79
+ ) as pool:
80
+ iterator = jobs = [
81
+ pool.submit(_split_job, np.flatnonzero(labels == i))
82
+ for i in labels_to_process
83
+ ]
84
+ if show_progress:
85
+ iterator = tqdm(
86
+ jobs,
87
+ desc=split_strategy,
88
+ total=len(labels_to_process),
89
+ smoothing=0,
90
+ )
91
+ for future in iterator:
92
+ split_result = future.result()
93
+ if not split_result.is_split:
94
+ continue
95
+
96
+ # assign new labels. -1 -> -1, 0 keeps current label,
97
+ # others start at cur_max_label + 1
98
+ in_unit = split_result.in_unit
99
+ new_labels = split_result.new_labels
100
+ triaged = split_result.new_labels < 0
101
+ labels[in_unit[triaged]] = new_labels[triaged]
102
+ labels[in_unit[new_labels > 0]] = cur_max_label + new_labels[new_labels > 0]
103
+ new_untriaged_labels = labels[in_unit[new_labels >= 0]]
104
+ cur_max_label = new_untriaged_labels.max()
105
+
106
+ # submit recursive jobs to the pool, if any
107
+ if recursive:
108
+ new_units = np.unique(new_untriaged_labels)
109
+ for i in new_units:
110
+ jobs.append(pool.submit(_split_job, np.flatnonzero(labels == i)))
111
+ if show_progress:
112
+ iterator.total += len(new_units) # type: ignore
113
+ elif split_big:
114
+ new_units = np.unique(new_untriaged_labels)
115
+ for i in new_units:
116
+ idx = np.flatnonzero(new_untriaged_labels == i)
117
+ tall = np.ptp(split_result.x[idx]) > split_big_kw["dx"]
118
+ wide = np.ptp(split_result.z_reg[idx]) > split_big_kw["dx"]
119
+ if (tall or wide) and len(idx) > split_big_kw["min_size_split"]:
120
+ jobs.append(
121
+ pool.submit(_split_job, np.flatnonzero(labels == i))
122
+ )
123
+ if show_progress:
124
+ iterator.total += 1 # type: ignore
125
+
126
+ new_sorting = sorting.ephemeral_replace(labels=labels)
127
+ if depth_order:
128
+ new_sorting = cluster_util.reorder_by_depth(new_sorting, motion_est=motion_est)
129
+ return new_sorting
130
+
131
+
132
+ # -- split steps
133
+
134
+ # the purpose of using classes here is that the class manages expensive
135
+ # resources that we don't want to set up every time we split a cluster.
136
+ # this way, these objects are created once on each worker process.
137
+ # we only have one split step so it is a bit silly to have the superclass,
138
+ # but it's just to illustrate the interface that's actually used by the
139
+ # main function split_clusters, in case someone wants to write another
140
+
141
+
142
+ @dataclass
143
+ class SplitResult:
144
+ """If not is_split, it is fine to leave other fields as None"""
145
+
146
+ is_split: bool = False
147
+ in_unit: Optional[np.ndarray] = None
148
+ new_labels: Optional[np.ndarray] = None
149
+ x: Optional[np.ndarray] = None
150
+ z_reg: Optional[np.ndarray] = None
151
+
152
+ def __post_init__(self):
153
+ """Runs after a SplitResult is constructed, check valid"""
154
+ if self.is_split:
155
+ assert self.in_unit is not None
156
+ assert self.new_labels is not None
157
+ assert self.in_unit.ndim == 1
158
+ assert self.in_unit.shape == self.new_labels.shape
159
+
160
+
161
+ class SplitStrategy:
162
+ """Split steps subclass this and implement split_cluster"""
163
+
164
+ def split_cluster(self, in_unit):
165
+ """
166
+ Arguments
167
+ ---------
168
+ in_unit : indices of spikes in a putative unit
169
+
170
+ Returns
171
+ -------
172
+ split_result : a SplitResult object
173
+ """
174
+ raise NotImplementedError
175
+
176
+
177
+ class FeatureSplit(SplitStrategy):
178
+ def __init__(
179
+ self,
180
+ peeling_hdf5_filename,
181
+ recording=None,
182
+ chunk_size_s=300,
183
+ peeling_featurization_pt=None,
184
+ motion_est=None,
185
+ use_localization_features=True,
186
+ return_localization_features=False,
187
+ n_pca_features=2,
188
+ pca_imputation=None,
189
+ pca_reassign=False,
190
+ relocated=True,
191
+ use_wfs_L2_norm=False,
192
+ localization_feature_scales=(1.0, 1.0, 50.0),
193
+ time_scale=3e-3,
194
+ use_time_feature=False,
195
+ log_c=5,
196
+ channel_selection_radius=75.0,
197
+ min_cluster_size=25,
198
+ min_samples=25,
199
+ cluster_selection_epsilon=1,
200
+ sigma_local=5,
201
+ sigma_local_low=None,
202
+ sigma_regional=None,
203
+ noise_density=0.1,
204
+ n_neighbors_search=20,
205
+ radius_search=5.0,
206
+ triage_quantile_per_cluster=0.2,
207
+ remove_clusters_smaller_than=25,
208
+ knn_reassign_outliers=False,
209
+ random_state=0,
210
+ rescale_all_features=False,
211
+ use_ptp=True,
212
+ amplitude_normalized=False,
213
+ use_spread=False,
214
+ max_spikes=None,
215
+ cluster_alg="hdbscan",
216
+ **dataset_name_kwargs,
217
+ ):
218
+ """Split clusters based on per-cluster PCA and localization features
219
+
220
+ Fits PCA to relocated waveforms within a cluster, combines these with
221
+ localization features (x, registered z, and max amplitude), and runs
222
+ HDBSCAN to refine the labeling.
223
+
224
+ Arguments
225
+ ---------
226
+ peeling_hdf5_filename : str or Path
227
+ Path to the HDF5 file output by a peeling step
228
+ peeling_featurization_pt : str or Path
229
+ Path to the .pt file where a featurization WaveformPipeline was
230
+ saved during peeling. Required if relocated=True.
231
+ motion_est : dredge.motion_util.MotionEstimate, optional
232
+ If supplied, registered depth positions will be used for clustering
233
+ and relocation
234
+ use_localization_features : bool
235
+ If false, only PCA features are used for clustering
236
+ n_pca_features : int
237
+ If 0, no PCA is used
238
+ relocated : bool
239
+ Whether to relocate waveforms according to the motion estimate.
240
+ Only used if motion_est is not None
241
+ localization_feature_scales : 3-tuple of float
242
+ Scales to apply to x, registered z, and log-scaled amplitude during
243
+ clustering to approximately sphere the clusters
244
+ channel_selection_radius : float
245
+ Spatial radius around the main channel used when selecting channels
246
+ used when extracting local PCA features
247
+ min_cluster_size, min_samples, cluster_selection_epsilon
248
+ HDBSCAN parameters. See their documentation for lots of info.
249
+ """
250
+ # method parameters
251
+ self.channel_selection_radius = channel_selection_radius
252
+ self.use_localization_features = use_localization_features
253
+ self.n_pca_features = n_pca_features
254
+ self.pca_imputation = pca_imputation
255
+ self.pca_reassign = pca_reassign
256
+ self.relocated = relocated and motion_est is not None
257
+ self.motion_est = motion_est
258
+ self.localization_feature_scales = localization_feature_scales
259
+ self.log_c = log_c
260
+ self.rg = np.random.default_rng(random_state)
261
+ self.knn_reassign_outliers = knn_reassign_outliers
262
+ self.max_spikes = max_spikes
263
+ self.use_wfs_L2_norm = use_wfs_L2_norm
264
+ self.rescale_all_features = rescale_all_features
265
+ self.use_ptp = use_ptp
266
+ self.amplitude_normalized = amplitude_normalized
267
+ self.use_spread = use_spread
268
+ self.use_time_feature = use_time_feature
269
+ self.time_scale = time_scale
270
+ self.return_localization_features = return_localization_features
271
+
272
+ # hdbscan parameters
273
+ self.min_cluster_size = min_cluster_size
274
+ self.min_samples = min_samples
275
+ self.cluster_selection_epsilon = cluster_selection_epsilon
276
+
277
+ # DPC parameters
278
+ self.sigma_local = sigma_local
279
+ self.sigma_local_low = sigma_local_low
280
+ self.sigma_regional = sigma_regional
281
+ self.noise_density = noise_density
282
+ self.n_neighbors_search = n_neighbors_search
283
+ self.radius_search = radius_search
284
+ self.triage_quantile_per_cluster = triage_quantile_per_cluster
285
+ self.remove_clusters_smaller_than = remove_clusters_smaller_than
286
+
287
+ assert cluster_alg in ("hdbscan", "dpc")
288
+ self.cluster_alg = cluster_alg
289
+
290
+ # load up the required h5 datasets
291
+ self.initialize_from_h5(
292
+ peeling_hdf5_filename,
293
+ peeling_featurization_pt,
294
+ **dataset_name_kwargs,
295
+ )
296
+
297
+ def split_cluster(self, in_unit_all):
298
+ n_spikes = in_unit_all.size
299
+ subsampling = self.max_spikes and n_spikes > self.max_spikes
300
+ if subsampling:
301
+ # TODO: max_spikes could be chosen automatically
302
+ # based on available memory and number of spikes
303
+ idx_subsample = self.rg.choice(n_spikes, self.max_spikes, replace=False)
304
+ idx_subsample.sort()
305
+ in_unit = in_unit_all[idx_subsample]
306
+ else:
307
+ in_unit = in_unit_all
308
+
309
+ if n_spikes < self.min_cluster_size:
310
+ return SplitResult()
311
+
312
+ do_pca = self.n_pca_features > 0
313
+
314
+ if do_pca or self.relocated or self.use_wfs_L2_norm or self.pca_reassign:
315
+ (
316
+ max_registered_channel,
317
+ n_pitches_shift,
318
+ reloc_amplitudes,
319
+ kept,
320
+ ) = self.get_registered_channels(in_unit)
321
+ if not kept.size:
322
+ return SplitResult()
323
+ else:
324
+ kept = np.arange(in_unit.shape[0])
325
+
326
+ features = []
327
+
328
+ if self.use_localization_features:
329
+ loc_features = self.localization_features[in_unit]
330
+ if self.relocated:
331
+ loc_features[kept, 2] = self.localization_feature_scales[2] * np.log(
332
+ self.log_c + reloc_amplitudes
333
+ )
334
+ if self.rescale_all_features:
335
+ mad0 = mad_sigma(loc_features[kept, 0])
336
+ loc_features[kept] *= mad0 / mad_sigma(loc_features[kept])
337
+ if not self.use_ptp:
338
+ loc_features = loc_features[:, :2]
339
+ features.append(loc_features)
340
+
341
+ if self.use_time_feature:
342
+ features.append(self.t_s[in_unit] * self.time_scale)
343
+
344
+ if do_pca:
345
+ enough_good_spikes, pca_kept, pca_embeds = self.pca_features(
346
+ in_unit[kept],
347
+ max_registered_channel,
348
+ n_pitches_shift,
349
+ amplitude_normalized=self.amplitude_normalized,
350
+ )
351
+ do_pca = enough_good_spikes
352
+
353
+ if do_pca:
354
+ pca_embeds_ = np.full(
355
+ (in_unit.size, pca_embeds.shape[1]),
356
+ np.nan,
357
+ dtype=pca_embeds.dtype,
358
+ )
359
+ pca_embeds_[kept[pca_kept]] = pca_embeds
360
+ pca_embeds = pca_embeds_
361
+ kept = kept[pca_kept]
362
+ # scale pc features to match localization features
363
+ if self.rescale_all_features:
364
+ mad0 = mad_sigma(loc_features[kept, 0])
365
+ for k in range(self.n_pca_features):
366
+ pca_embeds[:, k] *= mad0 / mad_sigma(pca_embeds[:, k])
367
+ elif self.use_localization_features:
368
+ pca_embeds *= mad_sigma(loc_features[kept]).mean()
369
+ features.append(pca_embeds)
370
+
371
+ l2_norm = None
372
+ if self.use_wfs_L2_norm:
373
+ enough_good_spikes, wf2_norm_kept, l2_norm = self.waveforms_L2norm(
374
+ in_unit,
375
+ max_registered_channel,
376
+ n_pitches_shift,
377
+ )
378
+ kept = kept[wf2_norm_kept]
379
+
380
+ if self.use_spread:
381
+ spread = self.spread_feature(in_unit)
382
+ if self.rescale_all_features:
383
+ mad0 = mad_sigma(loc_features[kept, 0])
384
+ spread *= mad0 / mad_sigma(spread[kept])
385
+ elif self.use_localization_features:
386
+ spread *= mad_sigma(loc_features[kept]).mean()
387
+ features.append(spread)
388
+
389
+ if not features:
390
+ return SplitResult()
391
+ features = np.column_stack([f[kept] for f in features])
392
+
393
+ if self.cluster_alg == "hdbscan" and features.shape[0] > self.min_cluster_size:
394
+ clust = HDBSCAN(
395
+ min_cluster_size=self.min_cluster_size,
396
+ min_samples=self.min_samples,
397
+ cluster_selection_epsilon=self.cluster_selection_epsilon,
398
+ core_dist_n_jobs=1, # let's just use our parallelism
399
+ prediction_data=self.knn_reassign_outliers,
400
+ )
401
+ clust_labels = clust.fit_predict(features)
402
+ new_ids = np.unique(clust_labels)
403
+ new_ids = new_ids[new_ids >= 0]
404
+ is_split = new_ids.size > 1
405
+ elif (
406
+ self.cluster_alg == "dpc"
407
+ and features.shape[0] > self.remove_clusters_smaller_than
408
+ ):
409
+ clust_labels = density.density_peaks_clustering(
410
+ features,
411
+ l2_norm=l2_norm,
412
+ sigma_local=self.sigma_local,
413
+ sigma_local_low=self.sigma_local_low,
414
+ sigma_regional=self.sigma_regional,
415
+ noise_density=self.noise_density,
416
+ n_neighbors_search=self.n_neighbors_search,
417
+ radius_search=self.radius_search,
418
+ triage_quantile_per_cluster=self.triage_quantile_per_cluster,
419
+ remove_clusters_smaller_than=self.remove_clusters_smaller_than,
420
+ )
421
+ new_ids = np.unique(clust_labels)
422
+ new_ids = new_ids[new_ids >= 0]
423
+ is_split = new_ids.size > 1
424
+ else:
425
+ is_split = False
426
+
427
+ if is_split and self.knn_reassign_outliers:
428
+ clust_labels = cluster_util.knn_reassign_outliers(clust_labels, features)
429
+
430
+ new_labels = None
431
+ if is_split:
432
+ new_labels = np.full(n_spikes, -1)
433
+ if subsampling:
434
+ new_labels[idx_subsample[kept]] = clust_labels
435
+ else:
436
+ new_labels[kept] = clust_labels
437
+
438
+ if is_split and self.pca_reassign:
439
+ if subsampling:
440
+ new_labels[idx_subsample] = self.tpca_reassignment(
441
+ new_labels[idx_subsample], in_unit, n_pitches_shift
442
+ )
443
+ else:
444
+ new_labels = self.tpca_reassignment(
445
+ new_labels, in_unit, n_pitches_shift
446
+ )
447
+
448
+ if self.return_localization_features and self.use_localization_features:
449
+ return SplitResult(
450
+ is_split=is_split,
451
+ in_unit=in_unit_all,
452
+ new_labels=new_labels,
453
+ x=loc_features[:, 0],
454
+ z_reg=loc_features[:, 1],
455
+ )
456
+ return SplitResult(
457
+ is_split=is_split, in_unit=in_unit_all, new_labels=new_labels
458
+ )
459
+
460
+ def get_registered_channels(self, in_unit, n_samples=1000):
461
+ n_pitches_shift = drift_util.get_spike_pitch_shifts(
462
+ self.z[in_unit],
463
+ geom=self.geom,
464
+ registered_depths_um=self.z_reg[in_unit],
465
+ )
466
+
467
+ amp_samples = slice(None)
468
+ if in_unit.size > n_samples:
469
+ amp_samples = self.rg.choice(in_unit.size, size=n_samples, replace=False)
470
+ amp_samples.sort()
471
+ amp_vecs = batched_h5_read(self.amplitude_vectors, in_unit)
472
+ amplitude_template = drift_util.registered_average(
473
+ amp_vecs[amp_samples],
474
+ n_pitches_shift[amp_samples],
475
+ self.geom,
476
+ self.registered_geom,
477
+ main_channels=self.channels[in_unit[amp_samples]],
478
+ channel_index=self.channel_index,
479
+ )
480
+ max_registered_channel = amplitude_template.argmax()
481
+
482
+ if self.relocated:
483
+ targ_chans = self.registered_channel_index[max_registered_channel]
484
+ targ_chans = targ_chans[targ_chans < len(self.registered_geom)]
485
+ reloc_amp_vecs = relocate.relocated_waveforms_on_static_channels(
486
+ amp_vecs,
487
+ main_channels=self.channels[in_unit],
488
+ channel_index=self.channel_index,
489
+ xyza_from=self.xyza[in_unit],
490
+ target_channels=targ_chans,
491
+ z_to=self.z_reg[in_unit],
492
+ geom=self.geom,
493
+ registered_geom=self.registered_geom,
494
+ match_distance=self.match_distance,
495
+ )
496
+ kept = np.flatnonzero(~np.isnan(reloc_amp_vecs).all(axis=1))
497
+ reloc_amplitudes = np.nanmax(reloc_amp_vecs[kept], axis=1)
498
+ else:
499
+ reloc_amplitudes = None
500
+ kept = np.arange(in_unit.size)
501
+
502
+ return max_registered_channel, n_pitches_shift, reloc_amplitudes, kept
503
+
504
+ def spread_feature(
505
+ self,
506
+ in_unit,
507
+ max_value_dist=70,
508
+ ):
509
+ spread = np.zeros(in_unit.shape[0])
510
+ amp_vecs = batched_h5_read(self.amplitude_vectors, in_unit)
511
+ main_channels = self.channels[in_unit]
512
+
513
+ channel_distances_index = np.sqrt(
514
+ (
515
+ (
516
+ np.pad(
517
+ self.geom,
518
+ [[0, 1], [0, 0]],
519
+ mode="constant",
520
+ constant_values=np.nan,
521
+ )[self.channel_index]
522
+ - self.geom[:, None]
523
+ )
524
+ ** 2
525
+ ).sum(2)
526
+ )
527
+
528
+ for k in range(in_unit.shape[0]):
529
+ max_chan = main_channels[k]
530
+ channels_effective = np.flatnonzero(self.channel_index[max_chan] < 384)
531
+ channels_effective = channels_effective[
532
+ channel_distances_index[max_chan][channels_effective] < max_value_dist
533
+ ]
534
+ spread[k] = np.nansum(
535
+ amp_vecs[k, channels_effective]
536
+ * channel_distances_index[max_chan][channels_effective]
537
+ ) / np.nansum(amp_vecs[k, channels_effective])
538
+
539
+ return spread
540
+
541
+ def waveforms_L2norm(
542
+ self,
543
+ in_unit,
544
+ max_registered_channel,
545
+ n_pitches_shift,
546
+ batch_size=1_000,
547
+ ):
548
+ """Compute relocated waveforms on a drift-invariant channel set, for computing L2 distance"""
549
+ # figure out which set of channels to use
550
+ # we use the stored amplitudes to do this rather than computing a
551
+ # template, which can be expensive
552
+ # max_batch_size set to avoid memory errors
553
+ pca_channels = self.registered_channel_index[max_registered_channel]
554
+ pca_channels = pca_channels[pca_channels < len(self.registered_geom)]
555
+
556
+ # load waveform embeddings and invert TPCA if we are relocating
557
+ waveforms = None # will allocate output array when we know its size
558
+ for bs in range(0, in_unit.size, batch_size):
559
+ be = min(in_unit.size, bs + batch_size)
560
+
561
+ batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
562
+ n_batch, rank, c = batch.shape
563
+
564
+ # invert TPCA for relocation
565
+ if self.relocated:
566
+ batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
567
+ batch = self.tpca.inverse_transform(batch)
568
+ t = batch.shape[1]
569
+ batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
570
+
571
+ # relocate or just restrict to channel subset
572
+ if self.relocated:
573
+ batch = relocate.relocated_waveforms_on_static_channels(
574
+ batch,
575
+ main_channels=self.channels[in_unit[bs:be]],
576
+ channel_index=self.channel_index,
577
+ target_channels=pca_channels,
578
+ xyza_from=self.xyza[in_unit[bs:be]],
579
+ z_to=self.z_reg[in_unit[bs:be]],
580
+ geom=self.geom,
581
+ registered_geom=self.registered_geom,
582
+ match_distance=self.match_distance,
583
+ )
584
+ else:
585
+ batch = drift_util.get_waveforms_on_static_channels(
586
+ batch,
587
+ self.geom,
588
+ main_channels=self.channels[in_unit[bs:be]],
589
+ channel_index=self.channel_index,
590
+ target_channels=pca_channels,
591
+ n_pitches_shift=n_pitches_shift[bs:be],
592
+ registered_geom=self.registered_geom,
593
+ match_distance=self.match_distance,
594
+ )
595
+ t = batch.shape[1]
596
+
597
+ if waveforms is None:
598
+ waveforms = np.empty(
599
+ (in_unit.size, t * pca_channels.size), dtype=batch.dtype
600
+ )
601
+ waveforms[bs:be] = batch.reshape(n_batch, -1)
602
+
603
+ # remove channels which are entirely nan
604
+ not_entirely_nan_channels = np.flatnonzero(np.isfinite(waveforms).any(axis=0))
605
+ if not_entirely_nan_channels.size < waveforms.shape[1]:
606
+ waveforms = waveforms[:, not_entirely_nan_channels]
607
+
608
+ # figure out which waveforms overlap completely with the remaining channels
609
+ no_nan = np.flatnonzero(np.isfinite(waveforms).all(axis=1))
610
+ if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
611
+ return False, no_nan, None
612
+
613
+ else:
614
+ size_wf = waveforms.shape[1]
615
+ waveforms_l2_norm = cdist(waveforms[no_nan], waveforms[no_nan]) / size_wf
616
+ return True, no_nan, waveforms_l2_norm
617
+
618
+ def pca_features(
619
+ self,
620
+ in_unit,
621
+ max_registered_channel,
622
+ n_pitches_shift,
623
+ batch_size=128,
624
+ max_samples_pca=5_000,
625
+ amplitude_normalized=False,
626
+ ):
627
+ """Compute relocated PCA features on a drift-invariant channel set
628
+
629
+ Returns
630
+ -------
631
+ enough_good_spikes : bool
632
+ kept : integer index array, size < in_unit.size, indices into in_unit
633
+ pca_embeds : array of shape (kept.size, rank)
634
+
635
+ """
636
+ # figure out which set of channels to use
637
+ # we use the stored amplitudes to do this rather than computing a
638
+ # template, which can be expensive
639
+ # max_batch_size set to avoid memory errors
640
+ pca_channels = self.registered_channel_index[max_registered_channel]
641
+ pca_channels = pca_channels[pca_channels < len(self.registered_geom)]
642
+
643
+ # fast path for small data case
644
+ if in_unit.size < max_samples_pca:
645
+ waveforms = batched_h5_read(self.tpca_features, in_unit)
646
+ n, rank, c = waveforms.shape
647
+
648
+ # invert TPCA for relocation
649
+ if self.relocated:
650
+ waveforms = waveforms.transpose(0, 2, 1).reshape(n * c, rank)
651
+ waveforms = self.tpca.inverse_transform(waveforms)
652
+ t = waveforms.shape[1]
653
+ waveforms = waveforms.reshape(n, c, t).transpose(0, 2, 1)
654
+
655
+ # relocate or just restrict to channel subset
656
+ if self.relocated:
657
+ waveforms = relocate.relocated_waveforms_on_static_channels(
658
+ waveforms,
659
+ main_channels=self.channels[in_unit],
660
+ channel_index=self.channel_index,
661
+ target_channels=pca_channels,
662
+ xyza_from=self.xyza[in_unit],
663
+ z_to=self.z_reg[in_unit],
664
+ geom=self.geom,
665
+ registered_geom=self.registered_geom,
666
+ match_distance=self.match_distance,
667
+ )
668
+ else:
669
+ waveforms = drift_util.get_waveforms_on_static_channels(
670
+ waveforms,
671
+ self.geom,
672
+ main_channels=self.channels[in_unit],
673
+ channel_index=self.channel_index,
674
+ target_channels=pca_channels,
675
+ n_pitches_shift=n_pitches_shift,
676
+ registered_geom=self.registered_geom,
677
+ match_distance=self.match_distance,
678
+ )
679
+ t = waveforms.shape[1]
680
+
681
+ # remove channels which are entirely nan
682
+ not_entirely_nan_channels = np.flatnonzero(
683
+ np.isfinite(waveforms[:, 0, :]).any(axis=0)
684
+ )
685
+ if not not_entirely_nan_channels.size:
686
+ return False, None, None
687
+ if not_entirely_nan_channels.size < waveforms.shape[2]:
688
+ waveforms = waveforms[:, :, not_entirely_nan_channels]
689
+
690
+ # figure out which waveforms overlap completely with the remaining channels
691
+ no_nan = np.flatnonzero(np.isfinite(waveforms[:, 0, :]).all(axis=1))
692
+ if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
693
+ return False, no_nan, None
694
+
695
+ # fit the per-cluster small rank PCA
696
+ pca = PCA(
697
+ self.n_pca_features,
698
+ random_state=np.random.RandomState(seed=self.rg.bit_generator),
699
+ whiten=True,
700
+ )
701
+ if self.pca_imputation is None:
702
+ waveforms = waveforms[no_nan]
703
+ waveforms = waveforms.reshape(waveforms.shape[0], -1)
704
+ if self.amplitude_normalized:
705
+ waveforms /= self.amplitudes[in_unit[no_nan]][:, None]
706
+ pca_projs = pca.fit_transform(waveforms)
707
+ return True, no_nan, pca_projs
708
+ elif self.pca_imputation in ("mean", "iterative"):
709
+ wfs = waveforms.reshape(waveforms.shape[0], -1)
710
+ if self.amplitude_normalized:
711
+ wfs /= self.amplitudes[in_unit][:, None]
712
+ pca.fit(wfs[no_nan])
713
+ if self.pca_imputation == "iterative":
714
+ wfs = np.where(np.isfinite(wfs), wfs, pca.mean_[None])
715
+ isnan = np.setdiff1d(np.arange(wfs.shape[0]), no_nan)
716
+ for i in range(10):
717
+ emb = pca.fit_transform(wfs)
718
+ wfs[isnan] = pca.inverse_transform(emb[isnan])
719
+ pca_projs = pca.transform(wfs)
720
+ else:
721
+ pca_projs = pca.transform(
722
+ np.where(np.isfinite(wfs), wfs, pca.mean_[None])
723
+ )
724
+ return True, slice(None), pca_projs
725
+ else:
726
+ assert False
727
+
728
+ # otherwise, we have bigger data. in that case, first fit the PCA
729
+ fit_choices = self.rg.choice(in_unit.size, size=max_samples_pca, replace=False)
730
+ fit_choices.sort()
731
+ fit_waveforms = batched_h5_read(self.tpca_features, in_unit[fit_choices])
732
+ n, rank, c = fit_waveforms.shape
733
+
734
+ # invert TPCA for relocation
735
+ if self.relocated:
736
+ fit_waveforms = fit_waveforms.transpose(0, 2, 1).reshape(n * c, rank)
737
+ fit_waveforms = self.tpca.inverse_transform(fit_waveforms)
738
+ t = fit_waveforms.shape[1]
739
+ fit_waveforms = fit_waveforms.reshape(n, c, t).transpose(0, 2, 1)
740
+
741
+ # relocate or just restrict to channel subset
742
+ if self.relocated:
743
+ fit_waveforms = relocate.relocated_waveforms_on_static_channels(
744
+ fit_waveforms,
745
+ main_channels=self.channels[in_unit[fit_choices]],
746
+ channel_index=self.channel_index,
747
+ target_channels=pca_channels,
748
+ xyza_from=self.xyza[in_unit[fit_choices]],
749
+ z_to=self.z_reg[in_unit[fit_choices]],
750
+ geom=self.geom,
751
+ registered_geom=self.registered_geom,
752
+ match_distance=self.match_distance,
753
+ )
754
+ else:
755
+ fit_waveforms = drift_util.get_waveforms_on_static_channels(
756
+ fit_waveforms,
757
+ self.geom,
758
+ main_channels=self.channels[in_unit[fit_choices]],
759
+ channel_index=self.channel_index,
760
+ target_channels=pca_channels,
761
+ n_pitches_shift=n_pitches_shift[fit_choices],
762
+ registered_geom=self.registered_geom,
763
+ match_distance=self.match_distance,
764
+ )
765
+ t = fit_waveforms.shape[1]
766
+
767
+ # remove channels which are entirely nan
768
+ not_entirely_nan_channels = np.flatnonzero(
769
+ np.isfinite(fit_waveforms[:, 0, :]).any(axis=0)
770
+ )
771
+ if not not_entirely_nan_channels.size:
772
+ print("All empty chans")
773
+ return False, None, None
774
+
775
+ subset_chans = not_entirely_nan_channels.size < fit_waveforms.shape[2]
776
+ if subset_chans:
777
+ fit_waveforms = fit_waveforms[:, :, not_entirely_nan_channels]
778
+
779
+ # figure out which fit_waveforms overlap completely with the remaining channels
780
+ no_nan = np.flatnonzero(np.isfinite(fit_waveforms[:, 0, :]).all(axis=1))
781
+ if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
782
+ return False, no_nan, None
783
+ fit_waveforms = fit_waveforms.reshape(fit_waveforms.shape[0], -1)
784
+ if self.amplitude_normalized:
785
+ amp_ix = in_unit[fit_choices]
786
+ fit_waveforms /= self.amplitudes[amp_ix][:, None]
787
+
788
+ # fit the per-cluster small rank PCA
789
+ pca = PCA(
790
+ self.n_pca_features,
791
+ random_state=np.random.RandomState(seed=self.rg.bit_generator),
792
+ whiten=True,
793
+ )
794
+ pca.fit(fit_waveforms[no_nan])
795
+ if self.pca_imputation == "iterative":
796
+ fit_waveforms = np.where(
797
+ np.isfinite(fit_waveforms), fit_waveforms, pca.mean_[None]
798
+ )
799
+ isnan = np.setdiff1d(np.arange(fit_waveforms.shape[0]), no_nan)
800
+ for i in range(10):
801
+ emb = pca.fit_transform(fit_waveforms)
802
+ fit_waveforms[isnan] = pca.inverse_transform(emb[isnan])
803
+
804
+ # now, we have the PCA. let's embed our data in batches.
805
+ pca_projs = np.full(
806
+ (in_unit.size, self.n_pca_features),
807
+ np.nan,
808
+ dtype=fit_waveforms.dtype,
809
+ )
810
+ all_kept = []
811
+
812
+ # load waveform embeddings and invert TPCA if we are relocating
813
+ for bs in range(0, in_unit.size, batch_size):
814
+ be = min(in_unit.size, bs + batch_size)
815
+
816
+ batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
817
+ n_batch, rank, c = batch.shape
818
+
819
+ # invert TPCA for relocation
820
+ if self.relocated:
821
+ batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
822
+ batch = self.tpca.inverse_transform(batch)
823
+ t = batch.shape[1]
824
+ batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
825
+
826
+ # relocate or just restrict to channel subset
827
+ if self.relocated:
828
+ batch = relocate.relocated_waveforms_on_static_channels(
829
+ batch,
830
+ main_channels=self.channels[in_unit[bs:be]],
831
+ channel_index=self.channel_index,
832
+ target_channels=pca_channels,
833
+ xyza_from=self.xyza[in_unit[bs:be]],
834
+ z_to=self.z_reg[in_unit[bs:be]],
835
+ geom=self.geom,
836
+ registered_geom=self.registered_geom,
837
+ match_distance=self.match_distance,
838
+ )
839
+ else:
840
+ batch = drift_util.get_waveforms_on_static_channels(
841
+ batch,
842
+ self.geom,
843
+ main_channels=self.channels[in_unit[bs:be]],
844
+ channel_index=self.channel_index,
845
+ target_channels=pca_channels,
846
+ n_pitches_shift=n_pitches_shift[bs:be],
847
+ registered_geom=self.registered_geom,
848
+ match_distance=self.match_distance,
849
+ )
850
+ t = batch.shape[1]
851
+
852
+ if subset_chans:
853
+ batch = batch[:, :, not_entirely_nan_channels]
854
+
855
+ if self.pca_imputation is None:
856
+ no_nan = np.flatnonzero(np.isfinite(batch[:, 0, :]).all(axis=1))
857
+ if not no_nan.size:
858
+ continue
859
+ all_kept.append(bs + no_nan)
860
+ batch = batch[no_nan].reshape(no_nan.size, -1)
861
+ if self.amplitude_normalized:
862
+ amp_ix = in_unit[bs:be][no_nan]
863
+ batch /= self.amplitudes[amp_ix][:, None]
864
+ pca_projs[bs + no_nan] = pca.transform(batch)
865
+ elif self.pca_imputation in ("mean", "iterative"):
866
+ batch = batch.reshape(batch.shape[0], -1)
867
+ batch /= self.amplitudes[in_unit[bs:be]][:, None]
868
+ pca_projs[bs:be] = pca.transform(
869
+ np.where(np.isfinite(batch), batch, pca.mean_[None])
870
+ )
871
+ else:
872
+ assert False
873
+ if self.pca_imputation is None:
874
+ kept = np.concatenate(all_kept)
875
+ elif self.pca_imputation in ("mean", "iterative"):
876
+ kept = slice(None)
877
+ else:
878
+ assert False
879
+ return True, kept, pca_projs[kept]
880
+
881
+ def tpca_reassignment(
882
+ self,
883
+ clust_labels,
884
+ in_unit,
885
+ n_pitches_shift,
886
+ centroid_samples=500,
887
+ batch_size=128,
888
+ ):
889
+ unit_ids = np.unique(clust_labels)
890
+ unit_ids = unit_ids[unit_ids >= 0]
891
+ n_units = unit_ids.size
892
+
893
+ # compute centroids in TPCA space
894
+ centroids = []
895
+ for split_label in unit_ids:
896
+ in_split = np.flatnonzero(clust_labels == split_label)
897
+ if in_split.size > centroid_samples:
898
+ in_split = self.rg.choice(
899
+ in_split, size=centroid_samples, replace=False
900
+ )
901
+ in_split.sort()
902
+
903
+ ix = in_unit[in_split]
904
+ split_feats = batched_h5_read(self.tpca_features, ix)
905
+ n_batch, rank, c = split_feats.shape
906
+
907
+ # invert TPCA for relocation
908
+ if self.relocated:
909
+ split_feats = split_feats.transpose(0, 2, 1).reshape(n_batch * c, rank)
910
+ split_feats = self.tpca.inverse_transform(split_feats)
911
+ t = split_feats.shape[1]
912
+ split_feats = split_feats.reshape(n_batch, c, t).transpose(0, 2, 1)
913
+
914
+ # relocate or just restrict to channel subset
915
+ if self.relocated:
916
+ split_feats = relocate.relocated_waveforms_on_static_channels(
917
+ split_feats,
918
+ target_channels=slice(None),
919
+ main_channels=self.channels[ix],
920
+ channel_index=self.channel_index,
921
+ xyza_from=self.xyza[ix],
922
+ z_to=self.z_reg[ix],
923
+ geom=self.geom,
924
+ registered_geom=self.registered_geom,
925
+ match_distance=self.match_distance,
926
+ )
927
+ else:
928
+ split_feats = drift_util.get_waveforms_on_static_channels(
929
+ split_feats,
930
+ self.geom,
931
+ main_channels=self.channels[ix],
932
+ channel_index=self.channel_index,
933
+ n_pitches_shift=n_pitches_shift[in_split],
934
+ registered_geom=self.registered_geom,
935
+ match_distance=self.match_distance,
936
+ )
937
+ centroids.append(np.nan_to_num(np.nanmean(split_feats, axis=0)))
938
+ centroids = np.array(centroids)
939
+ chans = np.flatnonzero(np.isfinite(centroids[:, 0, :]).any(0))
940
+ centroids = centroids[:, :, chans]
941
+
942
+ # load waveform embeddings and invert TPCA if we are relocating
943
+ new_labels = np.full_like(clust_labels, -1)
944
+ assert clust_labels.shape == in_unit.shape
945
+ for bs in range(0, in_unit.size, batch_size):
946
+ be = min(in_unit.size, bs + batch_size)
947
+
948
+ batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
949
+ n_batch, rank, c = batch.shape
950
+
951
+ # invert TPCA for relocation
952
+ if self.relocated:
953
+ batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
954
+ batch = self.tpca.inverse_transform(batch)
955
+ t = batch.shape[1]
956
+ batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
957
+
958
+ # relocate or just restrict to channel subset
959
+ if self.relocated:
960
+ batch = relocate.relocated_waveforms_on_static_channels(
961
+ batch,
962
+ main_channels=self.channels[in_unit[bs:be]],
963
+ channel_index=self.channel_index,
964
+ target_channels=chans,
965
+ xyza_from=self.xyza[in_unit[bs:be]],
966
+ z_to=self.z_reg[in_unit[bs:be]],
967
+ geom=self.geom,
968
+ registered_geom=self.registered_geom,
969
+ match_distance=self.match_distance,
970
+ )
971
+ else:
972
+ batch = drift_util.get_waveforms_on_static_channels(
973
+ batch,
974
+ self.geom,
975
+ main_channels=self.channels[in_unit[bs:be]],
976
+ channel_index=self.channel_index,
977
+ target_channels=chans,
978
+ n_pitches_shift=n_pitches_shift[bs:be],
979
+ registered_geom=self.registered_geom,
980
+ match_distance=self.match_distance,
981
+ )
982
+
983
+ d = nan_sqeuclidean_distances(
984
+ batch.reshape(batch.shape[0], -1), centroids.reshape(n_units, -1)
985
+ )
986
+ # d = batch.reshape(batch.shape[0], -1)[:, None] - centroids.reshape(n_units, -1)[None, :]
987
+ # np.nan_to_num(d, copy=False)
988
+ # d *= d
989
+ # d = d.sum(2)
990
+ new_labels[bs:be] = d.argmin(1)
991
+ # for j, i in enumerate(range(bs, be)):
992
+ # wf = batch[j]
993
+ # mask = np.isfinite(wf[0])
994
+ # wf = wf[:, mask].ravel()[None]
995
+ # c = centroids[:, :, mask].reshape(n_units, -1)
996
+ # new_labels[i] = cdist(wf, c, metric="sqeuclidean").ravel().argmin()
997
+
998
+ return new_labels
999
+
1000
+ def initialize_from_h5(
1001
+ self,
1002
+ peeling_hdf5_filename,
1003
+ peeling_featurization_pt,
1004
+ tpca_features_dataset_name="collisioncleaned_tpca_features",
1005
+ localizations_dataset_name="point_source_localizations",
1006
+ amplitudes_dataset_name="denoised_ptp_amplitudes",
1007
+ amplitude_vectors_dataset_name="denoised_ptp_amplitude_vectors",
1008
+ ):
1009
+ peeling_hdf5_filename = Path(peeling_hdf5_filename)
1010
+ h5 = h5py.File(peeling_hdf5_filename, "r", libver="latest", locking=False)
1011
+ self.geom = h5["geom"][:]
1012
+ self.channel_index = h5["channel_index"][:]
1013
+ self.channels = h5["channels"][:]
1014
+ self.match_distance = pdist(self.geom).min() / 2
1015
+
1016
+ if self.use_localization_features or self.relocated or self.use_time_feature:
1017
+ self.t_s = h5["times_seconds"][:]
1018
+
1019
+ if self.use_localization_features or self.relocated:
1020
+ self.xyza = h5[localizations_dataset_name][:]
1021
+ # registered spike positions (=originals if not relocated)
1022
+ self.z_reg = self.z = self.xyza[:, 2]
1023
+
1024
+ self.amplitudes = h5[amplitudes_dataset_name][:]
1025
+ self.registered_geom = self.geom
1026
+ self.registered_channel_index = self.channel_index
1027
+
1028
+ if self.motion_est is not None:
1029
+ self.z_reg = self.motion_est.correct_s(self.t_s, self.z)
1030
+ if self.relocated or self.n_pca_features or self.use_wfs_L2_norm:
1031
+ self.registered_geom = drift_util.registered_geometry(
1032
+ self.geom, self.motion_est
1033
+ )
1034
+ self.registered_channel_index = waveform_util.make_channel_index(
1035
+ self.registered_geom, self.channel_selection_radius
1036
+ )
1037
+
1038
+ if (
1039
+ (self.use_localization_features and self.relocated)
1040
+ or self.n_pca_features
1041
+ or self.use_ptp
1042
+ or self.use_wfs_L2_norm
1043
+ ):
1044
+ self.amplitude_vectors = h5[amplitude_vectors_dataset_name]
1045
+
1046
+ if self.use_localization_features:
1047
+ self.localization_features = np.c_[
1048
+ self.xyza[:, 0],
1049
+ self.z_reg,
1050
+ np.log(self.log_c + self.amplitudes),
1051
+ ]
1052
+ self.localization_features *= self.localization_feature_scales
1053
+
1054
+ if self.n_pca_features or self.use_wfs_L2_norm:
1055
+ # don't load these one into memory, since it's a bit heavier
1056
+ self.tpca_features = h5[tpca_features_dataset_name]
1057
+
1058
+ if peeling_featurization_pt is None:
1059
+ mdir = peeling_hdf5_filename.parent / f"{peeling_hdf5_filename.stem}_models"
1060
+ peeling_featurization_pt = mdir / "featurization_pipeline.pt"
1061
+ assert peeling_featurization_pt.exists()
1062
+
1063
+ if (self.n_pca_features or self.use_wfs_L2_norm) and self.relocated:
1064
+ # load up featurization pipeline for tpca inversion
1065
+ assert peeling_featurization_pt is not None
1066
+ feature_pipeline = get_featurization_pipeline(
1067
+ peeling_hdf5_filename, featurization_pipeline_pt
1068
+ )
1069
+ tpca_feature = [
1070
+ f
1071
+ for f in feature_pipeline.transformers
1072
+ if f.name == tpca_features_dataset_name
1073
+ ]
1074
+ assert len(tpca_feature) == 1
1075
+ self.tpca = tpca_feature[0].to_sklearn()
1076
+
1077
+
1078
+ class ChunkForwardBackwardFeatureSplit(FeatureSplit):
1079
+ def __init__(
1080
+ self,
1081
+ peeling_hdf5_filename,
1082
+ recording=None,
1083
+ chunk_size_s=300.0,
1084
+ **split_strategy_kwargs,
1085
+ ):
1086
+ """Split clusters based on per-cluster PCA and localization features
1087
+
1088
+ Fits PCA to relocated waveforms within a cluster, combines these with
1089
+ localization features (x, registered z, and max amplitude), and runs
1090
+ HDBSCAN to refine the labeling.
1091
+
1092
+ Arguments
1093
+ ---------
1094
+ peeling_hdf5_filename : str or Path
1095
+ Path to the HDF5 file output by a peeling step
1096
+ peeling_featurization_pt : str or Path
1097
+ Path to the .pt file where a featurization WaveformPipeline was
1098
+ saved during peeling. Required if relocated=True.
1099
+ motion_est : dredge.motion_util.MotionEstimate, optional
1100
+ If supplied, registered depth positions will be used for clustering
1101
+ and relocation
1102
+ use_localization_features : bool
1103
+ If false, only PCA features are used for clustering
1104
+ n_pca_features : int
1105
+ If 0, no PCA is used
1106
+ relocated : bool
1107
+ Whether to relocate waveforms according to the motion estimate.
1108
+ Only used if motion_est is not None
1109
+ localization_feature_scales : 3-tuple of float
1110
+ Scales to apply to x, registered z, and log-scaled amplitude during
1111
+ clustering to approximately sphere the clusters
1112
+ channel_selection_radius : float
1113
+ Spatial radius around the main channel used when selecting channels
1114
+ used when extracting local PCA features
1115
+ min_cluster_size, min_samples, cluster_selection_epsilon
1116
+ HDBSCAN parameters. See their documentation for lots of info.
1117
+ """
1118
+ split_strategy_kwargs["return_localization_features"] = True
1119
+ super().__init__(
1120
+ peeling_hdf5_filename, recording=recording, **split_strategy_kwargs
1121
+ )
1122
+
1123
+ # Check for ensembling
1124
+ self.chunk_size_s = chunk_size_s
1125
+ assert (
1126
+ self.use_localization_features
1127
+ ), "Need to use loc features for ensembling over chunks"
1128
+ assert (
1129
+ self.recording is not None
1130
+ ), "Need to input recording for ensembling over chunks"
1131
+ assert (
1132
+ self.chunk_size_s is not None
1133
+ ), "Need to input chunk size for ensembling over chunks"
1134
+
1135
+ def split_cluster(self, in_unit):
1136
+ chunk_samples = self.recording.sampling_frequency * self.chunk_size_s
1137
+
1138
+ n_chunks = self.recording.get_num_samples() / chunk_samples
1139
+ # we'll count the remainder as a chunk if it's at least 2/3 of one
1140
+ n_chunks = np.floor(n_chunks) + (n_chunks - np.floor(n_chunks) > 0.66)
1141
+ n_chunks = int(max(1, n_chunks))
1142
+
1143
+ # evenly divide the recording into chunks
1144
+ assert self.recording.get_num_segments() == 1
1145
+ start_time_s, end_time_s = self.recording._recording_segments[
1146
+ 0
1147
+ ].sample_index_to_time(np.array([0, self.recording.get_num_samples() - 1]))
1148
+ chunk_times_s = np.linspace(start_time_s, end_time_s, num=n_chunks + 1)
1149
+ chunk_time_ranges_s = list(zip(chunk_times_s[:-1], chunk_times_s[1:]))
1150
+
1151
+ chunks_indices_unit = [
1152
+ np.flatnonzero(
1153
+ np.logical_and(
1154
+ self.t_s[in_unit] >= chunk_range[0],
1155
+ self.t_s[in_unit] < chunk_range[1],
1156
+ )
1157
+ )
1158
+ for chunk_range in chunk_time_ranges_s
1159
+ ]
1160
+
1161
+ # cluster each chunk. can be parallelized in the future.
1162
+ splits = [
1163
+ super().split_cluster(in_unit[idx_unit_chunk])
1164
+ for idx_unit_chunk in chunks_indices_unit
1165
+ ]
1166
+
1167
+ new_labels_chunks = []
1168
+ for k in range(len(splits)):
1169
+ new_labels = np.full(len(in_unit), -1)
1170
+ if len(chunks_indices_unit[k]) and splits[k].new_labels is not None:
1171
+ new_labels[chunks_indices_unit[k]] = splits[k].new_labels
1172
+ elif len(chunks_indices_unit[k]) and splits[k].new_labels is None:
1173
+ # moved the 0 logic from split_cluster to here
1174
+ new_labels[chunks_indices_unit[k]] = 0
1175
+ new_labels_chunks.append(new_labels)
1176
+
1177
+ split_sortings = [
1178
+ DARTsortSorting(
1179
+ times_samples=self.t_s[
1180
+ in_unit
1181
+ ], # Needed for initialization but not for anything else
1182
+ channels=self.channels[in_unit],
1183
+ labels=new_labels,
1184
+ ephemeral_features={
1185
+ "point_source_localizations": self.xyza[in_unit],
1186
+ "denoised_ptp_amplitudes": self.amplitudes[in_unit],
1187
+ "times_seconds": self.t_s[in_unit],
1188
+ },
1189
+ )
1190
+ for new_labels in new_labels_chunks
1191
+ ]
1192
+
1193
+ labels_ensembled = forward_backward(
1194
+ self.recording,
1195
+ chunk_time_ranges_s,
1196
+ split_sortings,
1197
+ log_c=self.log_c,
1198
+ feature_scales=self.localization_feature_scales,
1199
+ adaptive_feature_scales=self.rescale_all_features,
1200
+ motion_est=self.motion_est,
1201
+ verbose=False,
1202
+ )
1203
+
1204
+ is_split = np.setdiff1d(np.unique(labels_ensembled), [-1]).size > 1
1205
+
1206
+ return SplitResult(
1207
+ is_split=is_split,
1208
+ in_unit=in_unit,
1209
+ new_labels=labels_ensembled,
1210
+ x=self.localization_features[in_unit, 0],
1211
+ z_reg=self.localization_features[in_unit, 2],
1212
+ )
1213
+
1214
+
1215
+ class NullSplit(SplitStrategy):
1216
+ def __init__(self, *args, **kwargs):
1217
+ pass
1218
+
1219
+ def split_cluster(self, in_unit):
1220
+ return SplitResult(in_unit=in_unit)
1221
+
1222
+
1223
+ # this is to help split_clusters take a string argument
1224
+ all_split_strategies = [
1225
+ FeatureSplit,
1226
+ ChunkForwardBackwardFeatureSplit,
1227
+ NullSplit,
1228
+ ]
1229
+ split_strategies_by_class_name = {cls.__name__: cls for cls in all_split_strategies}
1230
+
1231
+ # -- parallelism widgets
1232
+
1233
+
1234
+ class SplitJobContext:
1235
+ def __init__(self, split_strategy):
1236
+ self.split_strategy = split_strategy
1237
+
1238
+
1239
+ _split_job_context = None
1240
+
1241
+
1242
+ def _split_job_init(
1243
+ split_strategy_class_name, peeling_hdf5_filename, split_strategy_kwargs
1244
+ ):
1245
+ global _split_job_context
1246
+ SplitStrategy = split_strategies_by_class_name[split_strategy_class_name]
1247
+ if split_strategy_kwargs is None:
1248
+ split_strategy_kwargs = {}
1249
+ split_strategy_kwargs["peeling_hdf5_filename"] = peeling_hdf5_filename
1250
+ _split_job_context = SplitJobContext(SplitStrategy(**split_strategy_kwargs))
1251
+
1252
+
1253
+ def _split_job(in_unit):
1254
+ return _split_job_context.split_strategy.split_cluster(in_unit)
1255
+
1256
+
1257
+ # other helpers
1258
+
1259
+
1260
+ def mad_sigma(x, axis=0, keepdims=False, gauss_correct=True):
1261
+ med = np.median(x, axis=axis, keepdims=True)
1262
+ mad = np.median(np.abs(x - med), axis=axis, keepdims=keepdims)
1263
+ if gauss_correct:
1264
+ mad *= 1.4826
1265
+ return mad
1266
+
1267
+
1268
+ def nan_sqeuclidean_distances(X, Y, copy_X=False, copy_Y=True):
1269
+ missing_X = np.isnan(X)
1270
+ missing_Y = np.isnan(Y)
1271
+ if copy_X:
1272
+ X = X.copy()
1273
+ if copy_Y:
1274
+ Y = Y.copy()
1275
+ X[missing_X] = 0
1276
+ Y[missing_Y] = 0
1277
+ distances = cdist(X, Y, metric="sqeuclidean")
1278
+
1279
+ # Adjust distances for missing values
1280
+ X *= X
1281
+ Y *= Y
1282
+ distances -= np.dot(X, missing_Y.T)
1283
+ distances -= np.dot(missing_X, Y.T)
1284
+
1285
+ return distances