DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1299 @@
1
+ from typing import Literal, cast
2
+ from threading import local
3
+
4
+ import numba
5
+ import numpy as np
6
+ import torch
7
+ from torch import Tensor
8
+ import torch.nn.functional as F
9
+ from scipy.interpolate import RegularGridInterpolator
10
+ from scipy.ndimage import gaussian_filter
11
+ from scipy.sparse import coo_array
12
+ from scipy.sparse.csgraph import connected_components
13
+ from scipy.spatial import KDTree
14
+ from scipy.spatial.distance import pdist, squareform
15
+ from scipy.stats import bernoulli
16
+ from tqdm.auto import trange, tqdm
17
+
18
+ from ..util.logging_util import get_logger
19
+ from ..util.py_util import timer
20
+ from ..util.internal_config import ComputationConfig
21
+ from ..util.job_util import ensure_computation_config
22
+ from ..util.multiprocessing_util import get_pool
23
+ from .cluster_util import decrumb
24
+
25
+ logger = get_logger(__name__)
26
+
27
+
28
+ def kdtree_inliers(
29
+ X,
30
+ kdtree=None,
31
+ n_neighbors: int | None = 10,
32
+ distance_upper_bound: float | None = 25.0,
33
+ workers=1,
34
+ batch_size=2**16,
35
+ leafsize=10,
36
+ show_progress=False,
37
+ ) -> tuple[np.ndarray, KDTree]:
38
+ """Mark outlying points by a neighbors distance criterion
39
+
40
+ Returns
41
+ -------
42
+ inliers : bool array
43
+ inliers[i] = X[i] has at least n_neighbors neighbors within distance
44
+ distance_upper_bound
45
+ kdtree : KDTree
46
+ """
47
+ if kdtree is None:
48
+ kdtree = KDTree(X, leafsize=leafsize)
49
+
50
+ if distance_upper_bound is None or not n_neighbors:
51
+ inliers = np.ones(kdtree.n, dtype=bool)
52
+ return inliers, kdtree
53
+
54
+ inliers = np.zeros(kdtree.n, dtype=bool)
55
+ if show_progress:
56
+ iters = trange(0, kdtree.n, batch_size, desc="KDTin")
57
+ else:
58
+ iters = range(0, kdtree.n, batch_size)
59
+ for i0 in iters:
60
+ i1 = min(kdtree.n, i0 + batch_size)
61
+
62
+ _, indices = kdtree.query(
63
+ X[i0:i1],
64
+ k=1 + n_neighbors,
65
+ distance_upper_bound=distance_upper_bound,
66
+ workers=workers,
67
+ )
68
+ indices = cast(np.ndarray, indices)
69
+ inliers[i0:i1] = indices[:, -1] < kdtree.n
70
+
71
+ return inliers, kdtree
72
+
73
+
74
+ def get_smoothed_density(
75
+ X,
76
+ inliers: np.ndarray | slice = slice(None),
77
+ sigma: float = 1.0,
78
+ sigma_low: float | None = None,
79
+ sigma_ramp_ax=-1,
80
+ bin_sizes=None,
81
+ bin_size_ratio=10.0,
82
+ min_bin_size=0.1,
83
+ ramp_min_bin_size=5.0,
84
+ max_n_bins=128,
85
+ min_n_bins=5,
86
+ ) -> np.ndarray:
87
+ """Get RBF density estimates for each X[i] and bandwidth in sigmas
88
+
89
+ Outliers will be marked with NaN KDEs. Please pass inliers, or else your
90
+ histogram is liable to be way too big.
91
+ """
92
+ # figure out what bandwidths we'll be working on
93
+
94
+ if do_ramp := bool(sigma_low):
95
+ min_sigma = min(sigma, sigma_low)
96
+ else:
97
+ min_sigma = sigma
98
+
99
+ # histogram bin sizes -- not too big, not too small
100
+ infeats = X[inliers]
101
+ bin_sizes = np.full(X.shape[1], min_sigma / bin_size_ratio)
102
+ bin_sizes = np.maximum(min_bin_size, bin_sizes)
103
+ if do_ramp:
104
+ bin_sizes[sigma_ramp_ax] = max(bin_sizes[sigma_ramp_ax], ramp_min_bin_size)
105
+
106
+ # select bin edges
107
+ extents = np.c_[np.floor(infeats.min(0)), np.ceil(infeats.max(0))]
108
+ dextents = np.ptp(extents, 1)
109
+ if not (dextents > 0).all():
110
+ raise ValueError(
111
+ f"Issue in KDE. {dextents=} {infeats.shape=} {sigma=} {bin_sizes=}."
112
+ )
113
+ nbins = np.ceil(dextents / bin_sizes).astype(int)
114
+ nbins = nbins.clip(min_n_bins, max_n_bins)
115
+ bin_edges = [np.linspace(e[0], e[1], num=nb + 1) for e, nb in zip(extents, nbins)]
116
+
117
+ # compute histogram and figure out how big the bins actually were
118
+ raw_histogram, bin_edges = np.histogramdd(infeats, bins=bin_edges)
119
+ bin_sizes = np.array([(be[1] - be[0]) for be in bin_edges])
120
+ bin_centers = [0.5 * (be[1:] + be[:-1]) for be in bin_edges]
121
+
122
+ # normalize histogram to samples / volume
123
+ raw_histogram = raw_histogram / bin_sizes.prod()
124
+
125
+ # figure out how histogram should be filtered
126
+ hist = raw_histogram
127
+ if sigma is not None and sigma_low is None:
128
+ sigma_hist = sigma * np.reciprocal(bin_sizes)
129
+ hist = gaussian_filter(raw_histogram, sigma_hist)
130
+ elif sigma is not None and sigma_low is not None:
131
+ # filter by a sequence of bandwidths
132
+ sigma_hist = sigma * np.reciprocal(bin_sizes)
133
+ sigma_low_hist = sigma_low * np.reciprocal(bin_sizes)
134
+ ramp = np.linspace(
135
+ sigma_low_hist[sigma_ramp_ax],
136
+ sigma_hist[sigma_ramp_ax],
137
+ num=hist.shape[sigma_ramp_ax],
138
+ )
139
+
140
+ # operate along the ramp axis
141
+ hist_move = np.moveaxis(hist, sigma_ramp_ax, 0)
142
+ hist_smoothed = hist_move.copy()
143
+ for j, sig in enumerate(ramp):
144
+ sig_move = sig.copy()
145
+ sig_move[0] = sig[sigma_ramp_ax]
146
+ sig_move[sigma_ramp_ax] = sig[0]
147
+ hist_smoothed[j] = gaussian_filter(hist_move, sig_move)[j] # sig_move
148
+ hist = np.moveaxis(hist_smoothed, 0, sigma_ramp_ax)
149
+
150
+ lerp = RegularGridInterpolator(bin_centers, hist, bounds_error=False)
151
+ low = np.array([np.min(bc) for bc in bin_centers])
152
+ high = np.array([np.max(bc) for bc in bin_centers])
153
+ dens = lerp(X.clip(low, high))
154
+
155
+ return dens
156
+
157
+
158
+ def get_smoothed_density_ratio(
159
+ X,
160
+ *,
161
+ inliers: np.ndarray | slice = slice(None),
162
+ sigmas: list[float],
163
+ sigma_lows: list[float] | list[None] | None = None,
164
+ sigma_ramp_ax=-1,
165
+ bin_sizes=None,
166
+ bin_size_ratio=10.0,
167
+ min_bin_size=0.1,
168
+ ramp_min_bin_size=5.0,
169
+ max_n_bins=128,
170
+ min_n_bins=5,
171
+ ) -> np.ndarray:
172
+ assert 0 < len(sigmas) <= 2
173
+ sigma_lows = sigma_lows or [None] * len(sigmas)
174
+ dens = [
175
+ get_smoothed_density(
176
+ X=X,
177
+ inliers=inliers,
178
+ sigma=sigma,
179
+ sigma_low=sigma_low,
180
+ sigma_ramp_ax=sigma_ramp_ax,
181
+ bin_sizes=bin_sizes,
182
+ bin_size_ratio=bin_size_ratio,
183
+ min_bin_size=min_bin_size,
184
+ ramp_min_bin_size=ramp_min_bin_size,
185
+ max_n_bins=max_n_bins,
186
+ min_n_bins=min_n_bins,
187
+ )
188
+ for sigma, sigma_low in zip(sigmas, sigma_lows)
189
+ ]
190
+ if len(sigmas) == 1:
191
+ return dens[0]
192
+ else:
193
+ dens_ = dens[0]
194
+ dens_ /= dens[1]
195
+ np.nan_to_num(dens_, out=dens_) # type: ignore
196
+ return dens_
197
+
198
+
199
+ def nearest_higher_density_neighbor(
200
+ kdtree,
201
+ density,
202
+ n_neighbors_search=20,
203
+ distance_upper_bound=5.0,
204
+ workers=1,
205
+ batch_size=2**16,
206
+ ) -> np.ndarray:
207
+ nhdn = np.full(kdtree.n, kdtree.n, dtype=np.intp)
208
+ density_padded = np.pad(density, (0, 1), constant_values=np.inf)
209
+
210
+ for i0 in range(0, kdtree.n, batch_size):
211
+ i1 = min(kdtree.n, i0 + batch_size)
212
+
213
+ distances, indices = kdtree.query(
214
+ kdtree.data[i0:i1],
215
+ k=1 + n_neighbors_search,
216
+ distance_upper_bound=distance_upper_bound,
217
+ workers=workers,
218
+ )
219
+ # exclude self
220
+ distances = distances[:, 1:]
221
+ indices = indices[:, 1:]
222
+ missing = indices == kdtree.n
223
+ distances[missing] = np.inf
224
+
225
+ # find lowest distance higher density neighbor
226
+ bdens = density[i0:i1]
227
+ bneighb_dens = density_padded[indices]
228
+ is_lower_density = np.logical_or(
229
+ bneighb_dens <= bdens[:, None], np.isneginf(bneighb_dens)
230
+ )
231
+ distances[is_lower_density] = np.inf
232
+ indices[is_lower_density] = kdtree.n
233
+
234
+ nearest = distances.argmin(1, keepdims=True)
235
+ nhdn[i0:i1] = np.take_along_axis(indices, nearest, axis=1)[:, 0]
236
+
237
+ return nhdn
238
+
239
+
240
+ def remove_border_points(
241
+ labels, density, kdtree, search_radius=1.0, n_neighbors_search=5, workers=1
242
+ ):
243
+ distances, indices = kdtree.query(
244
+ kdtree.data,
245
+ k=1 + n_neighbors_search,
246
+ distance_upper_bound=search_radius,
247
+ workers=workers,
248
+ )
249
+
250
+ labels_padded = np.pad(labels, (0, 1), constant_values=-2)
251
+ in_border = np.any(
252
+ labels_padded[indices] != labels_padded[indices[:, 0, None]], axis=1
253
+ )
254
+ in_border = np.flatnonzero(in_border)
255
+ labels_in_border = labels[in_border]
256
+
257
+ units = np.unique(labels)
258
+ new_labels = labels.copy()
259
+ for i, u in enumerate(units[units > 0]):
260
+ in_unit = np.flatnonzero(labels == u)
261
+ in_unit_border = in_border[labels_in_border == u]
262
+ if not in_unit_border.size:
263
+ continue
264
+
265
+ to_remove = density[in_unit] < density[in_unit_border].max()
266
+ new_labels[in_unit[to_remove]] = -1
267
+
268
+ return new_labels
269
+
270
+
271
+ def guess_mode(
272
+ X,
273
+ sigma: float | Literal["rule_of_thumb"] = "rule_of_thumb",
274
+ outlier_neighbor_count=10,
275
+ outlier_sigma=3.0,
276
+ kdtree=None,
277
+ workers=1,
278
+ ):
279
+ """Use a KDE to guess the highest density point."""
280
+ n = len(X)
281
+ sigma0 = np.sqrt(X.var(axis=0).sum())
282
+ inliers, kdtree = kdtree_inliers(
283
+ X,
284
+ kdtree=kdtree,
285
+ n_neighbors=outlier_neighbor_count,
286
+ distance_upper_bound=outlier_sigma * sigma0,
287
+ workers=workers,
288
+ )
289
+ inliers = cast(np.ndarray, inliers)
290
+
291
+ if sigma == "rule_of_thumb":
292
+ sigma_dens = (
293
+ 1.06
294
+ * np.linalg.norm(np.std(X[inliers], axis=0))
295
+ * np.power(inliers.sum(), -0.2)
296
+ )
297
+ else:
298
+ sigma_dens = sigma
299
+
300
+ density = get_smoothed_density(X, inliers=inliers, sigma=sigma_dens)
301
+ assert isinstance(density, np.ndarray)
302
+ assert density.shape == (n,)
303
+
304
+ return np.argmax(density)
305
+
306
+
307
+ def bucket_density_ratio(
308
+ X: np.ndarray,
309
+ scaled_geom: np.ndarray,
310
+ sigma: float,
311
+ sigma_regional: float,
312
+ max_sigma: float = 3.0,
313
+ computation_cfg: ComputationConfig | None = None,
314
+ workers=-1,
315
+ batch_size=8192,
316
+ ):
317
+ """
318
+
319
+ Algorithm:
320
+ - Assign spikes to channels with a geom-kdtree (make sure geom's x
321
+ scale matches the clustering feature's x scale.)
322
+ - Let Dc be pairwise distances between channels
323
+ - Compute sparse distance matrix. If Dx is pairwise distance between
324
+ all pairs of spikes, we just want the entries with Dx<max_dist.
325
+ - Note
326
+ |ci-cj| = |(ci-xi) + (xj-cj) + (xi-xj)|
327
+ <= 2C + |xi-xj|
328
+ Thus we need not consider i,j s.t. |ci-cj|-2C>max_dist.
329
+ """
330
+ computation_cfg = ensure_computation_config(computation_cfg)
331
+ with timer("geom bucketing"):
332
+ geom_kdt = KDTree(scaled_geom, leafsize=1)
333
+ dists, channels = geom_kdt.query(X[:, :2], workers=workers)
334
+
335
+ max_dist_by_chan = np.zeros(len(scaled_geom))
336
+ for c in range(len(scaled_geom)):
337
+ inc = np.flatnonzero(channels == c)
338
+ if inc.size:
339
+ max_dist_by_chan[c] = dists[inc].max()
340
+
341
+ larger_sigma = max(sigma, sigma_regional or 0.0)
342
+ max_dist = max_sigma * larger_sigma
343
+
344
+ lb = squareform(pdist(scaled_geom))
345
+ lb -= max_dist_by_chan[:, None]
346
+ lb -= max_dist_by_chan
347
+ chans_pair_mask = lb <= max_dist
348
+ print(f"{chans_pair_mask.sum(1).min()=}")
349
+ print(f"{chans_pair_mask.sum(1).max()=}")
350
+ print(f"{max_dist_by_chan.max()=}")
351
+ print(f"{max_dist_by_chan.min()=}")
352
+
353
+ device = computation_cfg.actual_device()
354
+ channels = torch.asarray(channels, device=device, dtype=torch.long)
355
+ Xt = torch.asarray(X, device=device, dtype=torch.float)
356
+ chans_pair_mask = torch.asarray(chans_pair_mask, device=device, dtype=torch.bool)
357
+ density_ratio = torch.full((len(X),), -torch.inf, device=device)
358
+
359
+ # for c in trange(len(scaled_geom), desc="bktdens"):
360
+ for c in trange(25, desc="bktdens"):
361
+ (inc,) = (channels == c).nonzero(as_tuple=True)
362
+ (friends,) = chans_pair_mask[c][channels].nonzero(as_tuple=True)
363
+ density_ratio[inc] = _local_sparse_dens_ratio(
364
+ Xt, inc, friends, max_dist, sigma, sigma_regional, batch_size=batch_size
365
+ )
366
+
367
+ return density_ratio
368
+
369
+
370
+ @torch.jit.script
371
+ def _local_sparse_dens_ratio(
372
+ X: Tensor,
373
+ inc: Tensor,
374
+ inf: Tensor,
375
+ max_dist: float,
376
+ sigma0: float,
377
+ sigma1: float,
378
+ batch_size: int = 1024,
379
+ ):
380
+ nc = inc.shape[0]
381
+ nf = inf.shape[0]
382
+ dens_ratio = X.new_empty(nc)
383
+
384
+ s0_factor = (1.0 / (torch.sqrt(torch.tensor(2.0)) * sigma0)).to(X)
385
+ s1_factor = (1.0 / (torch.sqrt(torch.tensor(2.0)) * sigma1)).to(X)
386
+
387
+ for ic0 in range(0, nc, batch_size):
388
+ ic1 = min(nc, ic0 + batch_size)
389
+ Xc = X[inc[ic0:ic1]]
390
+ for if0 in range(0, nf, batch_size):
391
+ if1 = min(nf, if0 + batch_size)
392
+ Xf = X[inf[if0:if1]]
393
+
394
+ d = torch.cdist(Xc, Xf)
395
+ F.threshold(d, threshold=max_dist, value=torch.inf, inplace=True)
396
+
397
+ d0 = d * s0_factor
398
+ d1 = d * s1_factor
399
+ d0.square_().neg_().exp_()
400
+ d1.square_().neg_().exp_()
401
+ d0 = d0.sum(1)
402
+ d1 = d1.sum(1)
403
+ torch.divide(d0, d1, out=dens_ratio[ic0:ic1])
404
+
405
+ return dens_ratio.nan_to_num_()
406
+
407
+
408
+ def kdt_density(
409
+ kdtree: KDTree,
410
+ X: np.ndarray,
411
+ sigma: float,
412
+ sigma_regional: float,
413
+ batch_size=2048,
414
+ max_sigma: float = 3.0,
415
+ n_threads=8,
416
+ ):
417
+ n, d = X.shape
418
+ max_dist = max_sigma * sigma_regional
419
+
420
+ jobs = range(0, n, batch_size)
421
+ n_jobs, Executor, context = get_pool(n_jobs=n_threads, cls="ThreadPoolExecutor") # type: ignore
422
+ with Executor( # type: ignore
423
+ max_workers=n_jobs,
424
+ mp_context=context,
425
+ initializer=_kdtdens_init,
426
+ initargs=(kdtree, X, batch_size, sigma, sigma_regional, max_dist),
427
+ ) as pool:
428
+ density = np.full((n,), -np.inf)
429
+ for i0, i1, dens in tqdm(
430
+ pool.map(_kdtdens_job, jobs),
431
+ total=len(jobs),
432
+ smoothing=0.0,
433
+ desc=f"KDTdens[{n_jobs}]",
434
+ ):
435
+ density[i0:i1] = dens
436
+ np.nan_to_num(density, copy=False)
437
+ return density
438
+
439
+
440
+ _kdtdens_ctx = local()
441
+ _kdtdens_ctx.sargs = None
442
+
443
+
444
+ def _kdtdens_init(kdtree, X, batch_size, sigma, sigma_regional, max_dist):
445
+ global _kdtdens_ctx
446
+ _kdtdens_ctx.sargs = (kdtree, X, batch_size, sigma, sigma_regional, max_dist)
447
+
448
+
449
+ def _kdtdens_job(i0):
450
+ global _kdtdens_ctx
451
+ kdtree, X, batch_size, sigma, sigma_regional, max_dist = _kdtdens_ctx.sargs
452
+ i1 = min(kdtree.n, i0 + batch_size)
453
+ batch_kdt = KDTree(X[i0:i1])
454
+ sdm = batch_kdt.sparse_distance_matrix(kdtree, max_dist, output_type="ndarray")
455
+ ii = sdm["i"]
456
+ jj = sdm["j"]
457
+ v_local = sdm["v"]
458
+ v_regional = sdm["v"].copy()
459
+ _gausskernel(v_local, sigma)
460
+ _gausskernel(v_regional, sigma_regional)
461
+ coo_local = coo_array(
462
+ (v_local, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
463
+ )
464
+ coo_regional = coo_array(
465
+ (v_regional, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
466
+ )
467
+ dens_local = coo_local.sum(axis=1)
468
+ dens_regional = coo_regional.sum(axis=1)
469
+ dens_local /= dens_regional
470
+ return i0, i1, dens_local
471
+
472
+
473
+ @numba.jit(
474
+ parallel=False, nopython=True, boundscheck=False, nogil=True, error_model="numpy"
475
+ )
476
+ def _gausskernel(v: np.ndarray, sigma: float):
477
+ const = 1.0 / (np.sqrt(2.0) * sigma)
478
+ for i in range(v.size):
479
+ v[i] = np.exp(-np.square(v[i] * const))
480
+
481
+
482
+ def knn_density(
483
+ kdtree, X, k, distance_upper_bound, batch_size=2**12, workers=-1, sigma=None
484
+ ):
485
+ n, d = X.shape
486
+ density = np.full(n, -np.inf, dtype="float32")
487
+
488
+ for i0 in range(0, n, batch_size):
489
+ i1 = min(n, i0 + batch_size)
490
+ distances, indices = kdtree.query(
491
+ X[i0:i1],
492
+ k=k,
493
+ distance_upper_bound=distance_upper_bound,
494
+ workers=workers,
495
+ )
496
+ if sigma is None:
497
+ np.nan_to_num(distances, copy=False, posinf=-np.inf)
498
+ n_neighbs = (indices < kdtree.n).sum(axis=1)
499
+ max_dist = distances[:, 1:].max(1)
500
+ np.nan_to_num(max_dist, copy=False, posinf=-np.inf)
501
+ # forget the sphere factor and the factor of n.
502
+ density[i0:i1] = n_neighbs / (max_dist**d)
503
+ else:
504
+ kernel = np.square(distances, out=distances)
505
+ kernel *= -(sigma**-2)
506
+ np.exp(kernel, out=kernel)
507
+ density[i0:i1] = kernel.sum(1)
508
+
509
+ return density
510
+
511
+
512
+ def density_peaks(
513
+ X,
514
+ kdtree=None,
515
+ density=None,
516
+ knn_k=None,
517
+ use_knn=False,
518
+ use_histograms=False,
519
+ sigma_local=5.0,
520
+ sigma_regional=None,
521
+ outlier_neighbor_count: int | None = 10,
522
+ outlier_radius: float | None = 25.0,
523
+ n_neighbors_search=20,
524
+ radius_search=25.0,
525
+ noise_density=0.0,
526
+ remove_clusters_smaller_than=10,
527
+ remove_borders=False,
528
+ border_search_radius=10.0,
529
+ border_search_neighbors=3,
530
+ inlier_dims=slice(0, 2),
531
+ leafsize=24,
532
+ workers=-1,
533
+ ):
534
+ """Density peaks clustering as described by Rodriguez and Laio, but...
535
+
536
+ This implementation has three tweaks from original DPC
537
+ - KDE density estimation, implemented by smoothing histograms
538
+ - "Sharpening": density can be a ratio of KDEs, like a sharpening kernel
539
+ - Noise: you can throw away points with too low of a density (ratio)
540
+ """
541
+ n = len(X)
542
+
543
+ radius_search = radius_search * np.sqrt(X.shape[1])
544
+ dim_arange = np.arange(X.shape[1])
545
+ inlier_dim_ixs = dim_arange[inlier_dims]
546
+ if outlier_radius is not None:
547
+ outlier_radius = outlier_radius * np.sqrt(inlier_dim_ixs.size)
548
+ inliers, kdtree = kdtree_inliers(
549
+ X[:, inlier_dims],
550
+ kdtree=kdtree,
551
+ n_neighbors=outlier_neighbor_count,
552
+ distance_upper_bound=outlier_radius,
553
+ workers=workers,
554
+ leafsize=leafsize,
555
+ )
556
+ if not np.array_equal(inlier_dim_ixs, dim_arange):
557
+ kdtree = KDTree(X)
558
+
559
+ if density is None:
560
+ if sigma_regional and not use_histograms:
561
+ density = kdt_density(
562
+ kdtree,
563
+ X,
564
+ sigma=sigma_local,
565
+ sigma_regional=sigma_regional,
566
+ n_threads=workers,
567
+ )
568
+ elif use_knn:
569
+ density = knn_density(
570
+ kdtree, X, k=knn_k, distance_upper_bound=radius_search, workers=workers
571
+ )
572
+ elif use_histograms:
573
+ sigmas = [sigma_local]
574
+ if sigma_regional is not None:
575
+ sigmas.append(sigma_regional)
576
+ density = get_smoothed_density_ratio(X, inliers=inliers, sigmas=sigmas)
577
+ else:
578
+ density = knn_density(
579
+ kdtree,
580
+ X,
581
+ k=n_neighbors_search,
582
+ distance_upper_bound=radius_search,
583
+ workers=workers,
584
+ sigma=sigma_local,
585
+ )
586
+
587
+ nhdn = nearest_higher_density_neighbor(
588
+ kdtree,
589
+ density,
590
+ n_neighbors_search=n_neighbors_search,
591
+ distance_upper_bound=radius_search,
592
+ workers=workers,
593
+ )
594
+ if noise_density:
595
+ nhdn[density < noise_density] = n
596
+
597
+ nhdn = nhdn.astype(np.intc)
598
+ has_nhdn = np.flatnonzero(nhdn < n).astype(np.intc)
599
+ data = np.ones(has_nhdn.size)
600
+ graph = coo_array((data, (nhdn[has_nhdn], has_nhdn)), shape=(n, n))
601
+ ncc, labels = connected_components(graph)
602
+
603
+ if remove_borders:
604
+ labels = remove_border_points(
605
+ labels,
606
+ density,
607
+ kdtree,
608
+ search_radius=border_search_radius,
609
+ n_neighbors_search=border_search_neighbors,
610
+ workers=workers,
611
+ )
612
+
613
+ if remove_clusters_smaller_than:
614
+ labels = decrumb(labels, min_size=remove_clusters_smaller_than, in_place=True)
615
+
616
+ return dict(
617
+ density=density,
618
+ nhdn=nhdn,
619
+ labels=labels,
620
+ kdtree=kdtree,
621
+ )
622
+
623
+
624
+ def nearest_neighbor_assign(
625
+ kdtree, tree_labels, X_other, radius_search=5.0, workers=-1
626
+ ):
627
+ _, inds = kdtree.query(
628
+ X_other, k=1, distance_upper_bound=radius_search, workers=workers
629
+ )
630
+ found = np.flatnonzero(inds < kdtree.n)
631
+ other_labels = np.full(len(X_other), -1, dtype=tree_labels.dtype)
632
+ other_labels[found] = tree_labels[inds[found]]
633
+ return other_labels
634
+
635
+
636
+ def sparse_iso_hellinger(centroids, sigma, hellinger_threshold=0.25, centroids_b=None):
637
+ """Sparse Hellinger distance between isotropic Gaussian components"""
638
+ c = centroids * (1.0 / (sigma * np.sqrt(8.0)))
639
+ kdt = KDTree(c)
640
+ if centroids_b is None:
641
+ kdt_b = kdt
642
+ else:
643
+ c_b = centroids_b * (1.0 / (sigma * np.sqrt(8.0)))
644
+ kdt_b = KDTree(c_b)
645
+ max_distance = np.sqrt(-np.log(1 - hellinger_threshold))
646
+ dists = kdt.sparse_distance_matrix(
647
+ kdt_b, max_distance=max_distance, output_type="ndarray"
648
+ )
649
+ vals = dists["v"]
650
+ np.square(vals, out=vals)
651
+ vals *= -1
652
+ np.exp(vals, out=vals) # now BC
653
+ np.subtract(1, vals, out=vals) # and now hell^2.
654
+ np.abs(vals, out=vals) # numerical -0
655
+ np.sqrt(vals, out=vals) # and now hell
656
+ dists = coo_array((vals, (dists["j"], dists["i"])), shape=(kdt_b.n, kdt.n))
657
+ return dists
658
+
659
+
660
+ def coo_nhdn(coo, densities) -> np.ndarray:
661
+ ii, jj = coo.coords
662
+ dists = coo.data.copy()
663
+ M = dists.max()
664
+ dens_ii = densities[ii]
665
+ dens_jj = densities[jj]
666
+ not_higher = dens_jj <= dens_ii
667
+
668
+ # only higher density
669
+ dists[not_higher] = np.inf
670
+ dists[ii == jj] = 2 * M + 1 # i am my last resort
671
+ coo2 = coo_array((dists, (ii, jj)), shape=coo.shape)
672
+ # argmin == nearest
673
+ nhdn = coo2.tocsc().argmin(axis=1, explicit=True)
674
+ nhdn = np.atleast_1d(nhdn)
675
+ assert nhdn.shape == densities.shape[:1]
676
+ return nhdn
677
+
678
+
679
+ def gmm_density_peaks(
680
+ X,
681
+ channels,
682
+ outlier_neighbor_count=10,
683
+ outlier_radius=25.0,
684
+ remove_clusters_smaller_than=50,
685
+ workers=-1,
686
+ n_initializations=10,
687
+ n_iter=50,
688
+ max_components_per_channel=20,
689
+ min_spikes_per_component=10,
690
+ random_state=0,
691
+ kmeanspp_min_dist=0.0,
692
+ hellinger_cutoff=0.95,
693
+ hellinger_strong=0.0,
694
+ hellinger_weak=0.999,
695
+ max_sigma=5.0,
696
+ sigma_atol=1e-4,
697
+ max_samples=2_000_000,
698
+ noise_const_dims=None,
699
+ show_progress=True,
700
+ use_hellinger=False,
701
+ mop=True,
702
+ n_neighbors_search=20,
703
+ device=None,
704
+ ):
705
+ """Density peaks clustering via an isotropic GMM density estimate
706
+
707
+ Idea: use an overfitted isotropic GMM (i.e. lots of components) to capture the
708
+ density in a point cloud. Then, each GMM component is grouped together with its
709
+ nearest higher density (i.e., higher mixing proportion, since isotropic) neighbor,
710
+ much like the usual density peaks clustering algorithm.
711
+
712
+ Neighbor-ness is determined by a Hellinger overlap criterion. Contrasted with the
713
+ smoothed histogram DPC algorithm elsewhere in this file, this algorithm is a bit
714
+ more adaptive, and that neighbor criterion is easier to tune than the k-d tree query
715
+ parameters which define the point-to-point neighbor criterion for the other algorithm.
716
+
717
+ TODO: parametrize as max_components_per_square_micron or something instead of per
718
+ channel so that this has more hope of transferring from probe to probe.
719
+
720
+ TODO: see parallelism todo in gmm_kmeans. This could be a lot faster.
721
+
722
+ Arguments
723
+ ---------
724
+ X : (N_spikes, n_features) array
725
+ For instance, the features property of a SimpleMatrix object as obtained
726
+ from get_clustering_features()
727
+ channels: (N_spikes,) array
728
+ The channels to which each spike belongs.
729
+ TODO: currently used in controlling the number of components, but it may
730
+ change when the TODO above is implemented.
731
+ """
732
+ from .kmeans import truncated_kmeans
733
+
734
+ if device is None:
735
+ device = "cuda" if torch.cuda.is_available() else "cpu"
736
+ device = torch.device(device)
737
+
738
+ n = len(X)
739
+ if n > max_samples:
740
+ random_state = np.random.default_rng(random_state)
741
+ choices = random_state.choice(n, size=max_samples, replace=False)
742
+ choices.sort()
743
+ else:
744
+ choices = slice(None)
745
+
746
+ Xi = X[choices]
747
+ nc = len(Xi)
748
+ inliers, kdtree = kdtree_inliers(
749
+ Xi,
750
+ n_neighbors=outlier_neighbor_count,
751
+ distance_upper_bound=outlier_radius * np.sqrt(X.shape[1]),
752
+ workers=workers or 1,
753
+ )
754
+ if not isinstance(choices, slice):
755
+ inliers = choices[inliers]
756
+
757
+ Xi = X[inliers]
758
+ ni = len(Xi)
759
+ uchans, cchans = np.unique(channels[inliers], return_counts=True)
760
+ comps_per_chan = np.minimum(
761
+ max_components_per_channel,
762
+ np.ceil(cchans / min_spikes_per_component).astype(int),
763
+ )
764
+ n_components = min(
765
+ comps_per_chan.sum(), int(np.ceil(ni / min_spikes_per_component))
766
+ )
767
+ logger.dartsortdebug(
768
+ f"gmmdpc: {n_components} components for {cchans.size} channels"
769
+ f"over {ni}/{nc} inliers"
770
+ )
771
+ res = truncated_kmeans(
772
+ Xi,
773
+ max_sigma=max_sigma,
774
+ n_components=n_components,
775
+ n_initializations=n_initializations,
776
+ random_state=random_state,
777
+ n_iter=n_iter,
778
+ dirichlet_alpha=1.0 / n_components,
779
+ kmeanspp_min_dist=kmeanspp_min_dist,
780
+ device=device,
781
+ show_progress=show_progress,
782
+ noise_const_dims=noise_const_dims,
783
+ with_log_likelihoods=mop or not use_hellinger,
784
+ sigma_atol=sigma_atol,
785
+ )
786
+ res["n_components"] = n_components
787
+ res["kmeans_labels"] = res["labels"]
788
+ n_components = len(cast(torch.Tensor, res["centroids"]))
789
+ res["n_components_kept"] = n_components
790
+ print("nosqrt")
791
+ maxdist = max_sigma * float(res["sigma"]) # * np.sqrt(X.shape[1])
792
+ if use_hellinger:
793
+ centroids = cast(torch.Tensor, res["centroids"]).numpy(force=True)
794
+ log_proportions = cast(torch.Tensor, res["log_proportions"]).numpy(force=True)
795
+ labels = gmmdpc_hellinger(
796
+ X=X,
797
+ centroids=centroids,
798
+ sigma=res["sigma"],
799
+ log_proportions=log_proportions,
800
+ maxdist=maxdist,
801
+ workers=workers,
802
+ hellinger_cutoff=hellinger_cutoff,
803
+ hellinger_strong=hellinger_strong,
804
+ hellinger_weak=hellinger_weak,
805
+ )
806
+ else:
807
+ log_likelihoods = cast(torch.Tensor, res["log_likelihoods"]).numpy(force=True)
808
+ labels = gmmdpc_classic(
809
+ X=X,
810
+ inliers=inliers,
811
+ log_likelihoods=log_likelihoods,
812
+ maxdist=maxdist,
813
+ workers=workers,
814
+ n_neighbors_search=n_neighbors_search,
815
+ )
816
+
817
+ if remove_clusters_smaller_than:
818
+ if show_progress:
819
+ logger.info("Clean...")
820
+ labels = decrumb(labels, min_size=remove_clusters_smaller_than, in_place=True)
821
+ res["labels"] = labels # type: ignore
822
+
823
+ if mop:
824
+ # use k-dtree dpc to mop up any remaining clusters...
825
+ inliers = np.flatnonzero(inliers)
826
+ discarded = np.flatnonzero(labels[inliers] < 0)
827
+ log_likelihoods = cast(torch.Tensor, res["log_likelihoods"]).numpy(force=True)
828
+ mop_res = density_peaks(
829
+ Xi[discarded],
830
+ density=log_likelihoods[discarded],
831
+ outlier_radius=outlier_radius,
832
+ outlier_neighbor_count=outlier_neighbor_count,
833
+ radius_search=maxdist,
834
+ workers=workers,
835
+ remove_clusters_smaller_than=remove_clusters_smaller_than,
836
+ n_neighbors_search=n_neighbors_search,
837
+ )
838
+ mopped = np.flatnonzero(mop_res["labels"] >= 0)
839
+ mopped_labels = mop_res["labels"][mopped]
840
+ mopped_labels += labels.max() + 1
841
+ labels[inliers[discarded[mopped]]] = mopped_labels
842
+
843
+ return res
844
+
845
+
846
+ def gmmdpc_classic(
847
+ X,
848
+ inliers,
849
+ log_likelihoods,
850
+ maxdist,
851
+ workers=-1,
852
+ n_neighbors_search=20,
853
+ ):
854
+ kdtree_res = density_peaks(
855
+ X[inliers],
856
+ density=log_likelihoods,
857
+ outlier_radius=None,
858
+ outlier_neighbor_count=None,
859
+ radius_search=maxdist,
860
+ workers=workers,
861
+ remove_clusters_smaller_than=0,
862
+ n_neighbors_search=n_neighbors_search,
863
+ )
864
+ labels = nearest_neighbor_assign(
865
+ kdtree=kdtree_res["kdtree"],
866
+ tree_labels=kdtree_res["labels"],
867
+ X_other=X,
868
+ radius_search=maxdist,
869
+ workers=workers,
870
+ )
871
+ return labels
872
+
873
+
874
+ def gmmdpc_hellinger(
875
+ X,
876
+ centroids,
877
+ sigma,
878
+ log_proportions,
879
+ maxdist: float,
880
+ workers=-1,
881
+ hellinger_cutoff=0.95,
882
+ hellinger_strong=0.0,
883
+ hellinger_weak=0.999,
884
+ ) -> np.ndarray:
885
+ """Density peaks clustering via isotropic GMM (Hellinger version)
886
+
887
+ Spikes are assigned to isotropic Gaussian clusters by k-means. Then
888
+ density peaks is carried out on centroids, where the notion of density
889
+ for a centroid is its log proportion in the mixture.
890
+
891
+ As in any DPC implementation, the key is the neighbor criterion. Here,
892
+ components are neighbors if they meet certain criteria about Hellinger
893
+ distance, which is like cosine distance for probability distributions.
894
+ """
895
+ logger.dartsortdebug("GMMDPC: Hellinger...")
896
+ coo = sparse_iso_hellinger(
897
+ centroids,
898
+ sigma,
899
+ hellinger_threshold=hellinger_cutoff,
900
+ )
901
+ nhdn = coo_nhdn(coo, log_proportions)
902
+ assert nhdn.shape == (len(centroids),)
903
+ n_components = len(centroids)
904
+ ii = np.arange(len(nhdn))
905
+ jj = nhdn
906
+
907
+ if hellinger_strong:
908
+ strong = np.flatnonzero(coo.data < hellinger_strong)
909
+ ii = np.concatenate([ii, coo.coords[0][strong]])
910
+ jj = np.concatenate([jj, coo.coords[1][strong]])
911
+ order = np.argsort(ii, kind="stable")
912
+ ii = ii[order]
913
+ jj = jj[order]
914
+
915
+ if hellinger_weak:
916
+ disconnected = nhdn == ii
917
+ disconnected = np.logical_and(
918
+ disconnected,
919
+ np.logical_not(np.isin(ii, nhdn[np.logical_not(disconnected)])),
920
+ )
921
+ disconnected = np.flatnonzero(disconnected)
922
+ else:
923
+ disconnected = np.array([], dtype=int)
924
+
925
+ if hellinger_weak and disconnected.size:
926
+ coo_weak = sparse_iso_hellinger(
927
+ centroids[disconnected],
928
+ sigma,
929
+ hellinger_threshold=hellinger_weak,
930
+ # centroids_b=centroids,
931
+ )
932
+ coo_weak = coo_array(
933
+ (coo_weak.data, (coo_weak.coords[1], coo_weak.coords[0])),
934
+ shape=(disconnected.size, disconnected.size),
935
+ # (coo_weak.data, (disconnected[coo_weak.coords[1]], coo_weak.coords[0])),
936
+ # shape=(n_components, n_components),
937
+ )
938
+ weak_nhdn = coo_nhdn(coo_weak, log_proportions[disconnected])
939
+ # jj[disconnected] = weak_nhdn[disconnected]
940
+ print("weak2")
941
+ jj[disconnected] = disconnected[weak_nhdn]
942
+
943
+ _1 = np.ones((1,), dtype="float32")
944
+ _1 = np.broadcast_to(_1, jj.shape)
945
+ nhdn_coo = coo_array((_1, (ii, jj)), shape=(n_components, n_components))
946
+ _, labels = connected_components(nhdn_coo)
947
+ assert labels.shape == (n_components,)
948
+ labels_padded = np.pad(labels, [(0, 1)], constant_values=-1)
949
+
950
+ ckdt = KDTree(centroids)
951
+ _, q = ckdt.query(X, workers=workers or 1, distance_upper_bound=maxdist)
952
+ labels = labels_padded[q]
953
+ return labels
954
+
955
+
956
+ # -- versions used in UHD project
957
+
958
+
959
+ def density_peaks_fancy(
960
+ xyza,
961
+ amps,
962
+ sorting,
963
+ motion_est,
964
+ geom,
965
+ sigma_local=5.0,
966
+ sigma_regional=None,
967
+ outlier_neighbor_count=10,
968
+ outlier_radius=25.0,
969
+ n_neighbors_search=20,
970
+ radius_search=5.0,
971
+ noise_density=0.0,
972
+ remove_clusters_smaller_than=10,
973
+ workers=-1,
974
+ scales=(1.0, 1.0, 50.0),
975
+ amp_log_c=5.0,
976
+ sigma_local_low: float | None = None,
977
+ sigma_regional_low: float | None = None,
978
+ distance_dependent_noise_density=False,
979
+ attach_density_feature=False,
980
+ triage_quantile_per_cluster=0.0,
981
+ ramp_triage_per_cluster=False,
982
+ triage_quantile_before_clustering=0.0,
983
+ amp_no_triaging_before_clustering=6.0,
984
+ amp_no_triaging_after_clustering=8.0,
985
+ use_y_triaging=False,
986
+ ):
987
+ z = xyza[:, 2]
988
+ if motion_est is not None:
989
+ z = motion_est.correct_s(sorting.times_seconds, z)
990
+ z_not_reg = xyza[:, 2]
991
+ ampfeat = scales[2] * np.log(amp_log_c + amps[:])
992
+ res = _density_peaks_clustering_uhd_implementation(
993
+ np.c_[scales[0] * xyza[:, 0], scales[1] * z, ampfeat],
994
+ geom=geom,
995
+ y=xyza[:, 1],
996
+ z_not_reg=z_not_reg,
997
+ use_y_triaging=use_y_triaging,
998
+ sigma_local=sigma_local,
999
+ sigma_local_low=sigma_local_low,
1000
+ sigma_regional=sigma_regional,
1001
+ sigma_regional_low=sigma_regional_low,
1002
+ n_neighbors_search=n_neighbors_search,
1003
+ radius_search=radius_search,
1004
+ remove_clusters_smaller_than=remove_clusters_smaller_than,
1005
+ noise_density=noise_density,
1006
+ triage_quantile_per_cluster=triage_quantile_per_cluster,
1007
+ ramp_triage_per_cluster=ramp_triage_per_cluster,
1008
+ triage_quantile_before_clustering=triage_quantile_before_clustering,
1009
+ amp_no_triaging_before_clustering=amp_no_triaging_before_clustering,
1010
+ amp_no_triaging_after_clustering=amp_no_triaging_after_clustering,
1011
+ distance_dependent_noise_density=distance_dependent_noise_density,
1012
+ outlier_radius=outlier_radius,
1013
+ outlier_neighbor_count=outlier_neighbor_count,
1014
+ scales=scales,
1015
+ log_c=amp_log_c,
1016
+ workers=workers,
1017
+ return_extra=attach_density_feature,
1018
+ )
1019
+ return res
1020
+
1021
+
1022
+ def _density_peaks_clustering_uhd_implementation(
1023
+ X,
1024
+ geom=None,
1025
+ y=None,
1026
+ z_not_reg=None,
1027
+ use_y_triaging=False,
1028
+ l2_norm=None,
1029
+ kdtree=None,
1030
+ sigma_local=5.0,
1031
+ sigma_local_low=None,
1032
+ sigma_regional=None,
1033
+ sigma_regional_low=None,
1034
+ min_bin_size=1.0,
1035
+ outlier_neighbor_count=10,
1036
+ outlier_radius=25.0,
1037
+ n_neighbors_search=20,
1038
+ radius_search=5.0,
1039
+ noise_density=0.0,
1040
+ remove_clusters_smaller_than=10,
1041
+ remove_borders=False,
1042
+ border_search_radius=10.0,
1043
+ border_search_neighbors=3,
1044
+ workers=1,
1045
+ return_extra=False,
1046
+ triage_quantile_before_clustering=0.0,
1047
+ amp_no_triaging_before_clustering=6.0,
1048
+ ramp_triage_before_clustering=False,
1049
+ radius_triage_before_clustering=75.0,
1050
+ triage_quantile_per_cluster=0.0,
1051
+ amp_no_triaging_after_clustering=12.0,
1052
+ ramp_triage_per_cluster=False,
1053
+ distance_dependent_noise_density=False,
1054
+ amp_lowest_noise_density=8.0,
1055
+ min_distance_noise_density=0.0,
1056
+ min_distance_noise_density_10=200,
1057
+ max_noise_density=10.0,
1058
+ max_n_bins=128,
1059
+ scales=None,
1060
+ log_c=None,
1061
+ ):
1062
+ """
1063
+ if l2_norm is passed as argument, it will be used to compute density and nhdn
1064
+ """
1065
+ # n = len(X)
1066
+
1067
+ if ramp_triage_before_clustering and geom is not None:
1068
+ assert scales is not None
1069
+ assert log_c is not None
1070
+ inliers_first = np.ones(len(X)).astype("bool")
1071
+ idx_low_ptp = np.flatnonzero(
1072
+ X[:, 2] < scales[2] * np.log(log_c + amp_no_triaging_before_clustering)
1073
+ )
1074
+ distances_to_geom = (
1075
+ np.sqrt((X[idx_low_ptp, :2][None] - geom[:, None]) ** 2).sum(2)
1076
+ ).min(0)
1077
+ probabilities = (
1078
+ np.minimum(distances_to_geom, radius_triage_before_clustering)
1079
+ / radius_triage_before_clustering
1080
+ )
1081
+ which_to_discard = bernoulli.rvs(probabilities).astype("bool") # type: ignore
1082
+ inliers_first[idx_low_ptp[which_to_discard]] = False
1083
+ inliers_first = np.arange(len(X))[inliers_first]
1084
+ else:
1085
+ inliers_first = np.arange(len(X))
1086
+
1087
+ n = len(inliers_first)
1088
+ if n <= 1:
1089
+ return dict(labels=np.full(X.shape[0], -1))
1090
+
1091
+ if use_y_triaging and y is not None:
1092
+ inliers, _ = kdtree_inliers(
1093
+ np.c_[X[inliers_first], y[inliers_first]],
1094
+ kdtree=kdtree,
1095
+ n_neighbors=outlier_neighbor_count,
1096
+ distance_upper_bound=outlier_radius * np.sqrt(4 / 3),
1097
+ workers=workers,
1098
+ )
1099
+ _, kdtree = kdtree_inliers(
1100
+ X[inliers_first],
1101
+ kdtree=kdtree,
1102
+ n_neighbors=outlier_neighbor_count,
1103
+ distance_upper_bound=outlier_radius,
1104
+ workers=workers,
1105
+ )
1106
+ else:
1107
+ inliers, kdtree = kdtree_inliers(
1108
+ X[inliers_first],
1109
+ kdtree=kdtree,
1110
+ n_neighbors=outlier_neighbor_count,
1111
+ distance_upper_bound=outlier_radius,
1112
+ workers=workers,
1113
+ )
1114
+
1115
+ if not inliers.sum() > 1:
1116
+ return dict(labels=np.full(X.shape[0], -1))
1117
+
1118
+ # inliers = inliers_first[inliers]
1119
+
1120
+ if isinstance(sigma_local, str) and sigma_local.startswith("rule_of_thumb"):
1121
+ factor = 1
1122
+ if "*" in sigma_local:
1123
+ factor = float(sigma_local.split("*")[1].strip())
1124
+ sigma_local = (
1125
+ 1.06
1126
+ * factor
1127
+ * np.linalg.norm(np.std(X[inliers_first][inliers], axis=0))
1128
+ * np.power(inliers.sum(), -0.2)
1129
+ )
1130
+ if sigma_local <= 0:
1131
+ raise ValueError(
1132
+ f"rule of thumb problems. {sigma_local=} "
1133
+ f"{np.std(X[inliers_first][inliers], axis=0)=} "
1134
+ f"{len(X)=} {np.power(len(X), -0.2)=} "
1135
+ f"{X.shape=} {inliers_first.shape=} {inliers.sum()=} "
1136
+ f"{X[inliers_first][inliers].shape=}"
1137
+ )
1138
+ if sigma_regional == "rule_of_thumb":
1139
+ radius_search = radius_search * sigma_local
1140
+ sigma_regional = 10 * sigma_local
1141
+ else:
1142
+ radius_search = radius_search * sigma_local
1143
+
1144
+ if l2_norm is None:
1145
+ do_ratio = sigma_regional is not None
1146
+ density = get_smoothed_density(
1147
+ X[inliers_first],
1148
+ inliers=inliers,
1149
+ sigma=sigma_local,
1150
+ sigma_low=sigma_local_low,
1151
+ min_bin_size=min_bin_size,
1152
+ max_n_bins=max_n_bins,
1153
+ )
1154
+ if density is None:
1155
+ return dict(labels=np.full(X.shape[0], -1))
1156
+ if do_ratio:
1157
+ reg_density = get_smoothed_density(
1158
+ X[inliers_first],
1159
+ inliers=inliers,
1160
+ sigma=sigma_regional,
1161
+ sigma_low=sigma_regional_low,
1162
+ min_bin_size=min_bin_size,
1163
+ )
1164
+ density = density / reg_density
1165
+ density = np.nan_to_num(density)
1166
+
1167
+ nhdn = nearest_higher_density_neighbor(
1168
+ kdtree,
1169
+ density,
1170
+ n_neighbors_search=n_neighbors_search,
1171
+ distance_upper_bound=radius_search,
1172
+ workers=workers,
1173
+ )
1174
+ else:
1175
+ # inliers don't matter here? Or should we still remove them?...
1176
+ # indices = np.full(l2_norm.shape[0], n)
1177
+ # l2_norm_inliers = l2_norm[inliers] #?? NO -> keep everything but only compute for inliers
1178
+ l2_norm = l2_norm[inliers_first][:, inliers_first]
1179
+ n = l2_norm.shape[0]
1180
+ indices = l2_norm.argsort()[:, : 1 + n_neighbors_search]
1181
+ distances = l2_norm[np.arange(n)[:, None], indices]
1182
+ assert distances.shape == (l2_norm.shape[0], 1 + n_neighbors_search)
1183
+ density = np.median(distances, axis=1)
1184
+ density_padded = np.pad(density, (0, 1), constant_values=np.inf)
1185
+ is_higher_density = density_padded[indices] >= density[:, None]
1186
+ distances[is_higher_density] = np.inf
1187
+ indices[is_higher_density] = n
1188
+ nhdn = indices[np.arange(n), distances.argmin(1)]
1189
+ del distances, indices
1190
+
1191
+ if noise_density and l2_norm is None:
1192
+ nhdn[density <= noise_density] = n
1193
+
1194
+ if distance_dependent_noise_density and noise_density is not None:
1195
+ assert scales is not None
1196
+ assert log_c is not None
1197
+ assert geom is not None
1198
+ dist = np.sqrt(
1199
+ (
1200
+ (
1201
+ np.c_[X[inliers_first, 0], z_not_reg[inliers_first]][:, None] # type: ignore
1202
+ - geom[None]
1203
+ )
1204
+ ** 2
1205
+ ).sum(2)
1206
+ ).min(1)
1207
+ noise_density_dist = (
1208
+ np.minimum(
1209
+ np.maximum(dist - min_distance_noise_density, 0),
1210
+ min_distance_noise_density_10 - min_distance_noise_density,
1211
+ )
1212
+ * (max_noise_density - noise_density)
1213
+ / (min_distance_noise_density_10 - min_distance_noise_density)
1214
+ + noise_density
1215
+ )
1216
+ noise_density_dist[
1217
+ X[inliers_first, 2] > scales[2] * np.log(log_c + amp_lowest_noise_density)
1218
+ ] = 2
1219
+ nhdn[density <= noise_density_dist] = n
1220
+ if triage_quantile_before_clustering and l2_norm is None:
1221
+ assert scales is not None
1222
+ assert log_c is not None
1223
+ q = np.quantile(density, triage_quantile_before_clustering)
1224
+ idx_triaging = np.flatnonzero(
1225
+ np.logical_and(
1226
+ X[inliers_first, 2]
1227
+ < scales[2] * np.log(log_c + amp_no_triaging_before_clustering),
1228
+ density < q,
1229
+ )
1230
+ )
1231
+ nhdn[idx_triaging] = n
1232
+
1233
+ nhdn = nhdn.astype(np.intc)
1234
+ has_nhdn = np.flatnonzero(nhdn < n).astype(np.intc)
1235
+ rc = (
1236
+ np.concatenate([nhdn[has_nhdn], has_nhdn]),
1237
+ np.concatenate([has_nhdn, nhdn[has_nhdn]]),
1238
+ )
1239
+
1240
+ graph = coo_array((np.ones(2 * has_nhdn.size), rc), shape=(n, n))
1241
+ ncc, labels = connected_components(graph, directed=False)
1242
+
1243
+ if remove_borders:
1244
+ labels = remove_border_points(
1245
+ labels,
1246
+ density,
1247
+ kdtree,
1248
+ search_radius=border_search_radius,
1249
+ n_neighbors_search=border_search_neighbors,
1250
+ workers=workers,
1251
+ )
1252
+
1253
+ if remove_clusters_smaller_than:
1254
+ labels = decrumb(labels, min_size=remove_clusters_smaller_than)
1255
+
1256
+ if triage_quantile_per_cluster:
1257
+ assert scales is not None
1258
+ assert log_c is not None
1259
+ amp_no_triaging_after_clustering = scales[2] * np.log(
1260
+ log_c + amp_no_triaging_after_clustering
1261
+ )
1262
+ for k in np.unique(labels[labels > -1]):
1263
+ idx_label = np.flatnonzero(labels == k)
1264
+ amp_vec = X[inliers_first][idx_label, 2]
1265
+ med_amp = np.median(amp_vec)
1266
+ if med_amp < amp_no_triaging_after_clustering:
1267
+ if ramp_triage_per_cluster:
1268
+ triage_quantile_per_cluster = (
1269
+ amp_no_triaging_after_clustering - med_amp
1270
+ ) / amp_no_triaging_after_clustering
1271
+ # triage_quantile_unit = triage_quantile_per_cluster
1272
+ if l2_norm is None:
1273
+ q = np.quantile(density[idx_label], triage_quantile_per_cluster)
1274
+ spikes_to_remove = np.flatnonzero(
1275
+ np.logical_and(
1276
+ density[idx_label] < q,
1277
+ amp_vec < amp_no_triaging_after_clustering,
1278
+ )
1279
+ )
1280
+ else:
1281
+ q = np.quantile(density[idx_label], 1 - triage_quantile_per_cluster)
1282
+ spikes_to_remove = np.flatnonzero(
1283
+ np.logical_and(
1284
+ density[idx_label] > q,
1285
+ amp_vec < amp_no_triaging_after_clustering,
1286
+ )
1287
+ )
1288
+
1289
+ labels[idx_label[spikes_to_remove]] = -1
1290
+
1291
+ labels_all = np.full(len(X), -1)
1292
+ labels_all[inliers_first] = labels
1293
+
1294
+ density_all = np.zeros(len(X))
1295
+ nhdn_all = np.full(len(X), n)
1296
+ density_all[inliers_first] = density
1297
+ nhdn_all[inliers_first] = nhdn
1298
+
1299
+ return dict(density=density_all, nhdn=nhdn_all, labels=labels_all, kdtree=kdtree)