DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1299 @@
|
|
|
1
|
+
from typing import Literal, cast
|
|
2
|
+
from threading import local
|
|
3
|
+
|
|
4
|
+
import numba
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
import torch.nn.functional as F
|
|
9
|
+
from scipy.interpolate import RegularGridInterpolator
|
|
10
|
+
from scipy.ndimage import gaussian_filter
|
|
11
|
+
from scipy.sparse import coo_array
|
|
12
|
+
from scipy.sparse.csgraph import connected_components
|
|
13
|
+
from scipy.spatial import KDTree
|
|
14
|
+
from scipy.spatial.distance import pdist, squareform
|
|
15
|
+
from scipy.stats import bernoulli
|
|
16
|
+
from tqdm.auto import trange, tqdm
|
|
17
|
+
|
|
18
|
+
from ..util.logging_util import get_logger
|
|
19
|
+
from ..util.py_util import timer
|
|
20
|
+
from ..util.internal_config import ComputationConfig
|
|
21
|
+
from ..util.job_util import ensure_computation_config
|
|
22
|
+
from ..util.multiprocessing_util import get_pool
|
|
23
|
+
from .cluster_util import decrumb
|
|
24
|
+
|
|
25
|
+
logger = get_logger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def kdtree_inliers(
|
|
29
|
+
X,
|
|
30
|
+
kdtree=None,
|
|
31
|
+
n_neighbors: int | None = 10,
|
|
32
|
+
distance_upper_bound: float | None = 25.0,
|
|
33
|
+
workers=1,
|
|
34
|
+
batch_size=2**16,
|
|
35
|
+
leafsize=10,
|
|
36
|
+
show_progress=False,
|
|
37
|
+
) -> tuple[np.ndarray, KDTree]:
|
|
38
|
+
"""Mark outlying points by a neighbors distance criterion
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
inliers : bool array
|
|
43
|
+
inliers[i] = X[i] has at least n_neighbors neighbors within distance
|
|
44
|
+
distance_upper_bound
|
|
45
|
+
kdtree : KDTree
|
|
46
|
+
"""
|
|
47
|
+
if kdtree is None:
|
|
48
|
+
kdtree = KDTree(X, leafsize=leafsize)
|
|
49
|
+
|
|
50
|
+
if distance_upper_bound is None or not n_neighbors:
|
|
51
|
+
inliers = np.ones(kdtree.n, dtype=bool)
|
|
52
|
+
return inliers, kdtree
|
|
53
|
+
|
|
54
|
+
inliers = np.zeros(kdtree.n, dtype=bool)
|
|
55
|
+
if show_progress:
|
|
56
|
+
iters = trange(0, kdtree.n, batch_size, desc="KDTin")
|
|
57
|
+
else:
|
|
58
|
+
iters = range(0, kdtree.n, batch_size)
|
|
59
|
+
for i0 in iters:
|
|
60
|
+
i1 = min(kdtree.n, i0 + batch_size)
|
|
61
|
+
|
|
62
|
+
_, indices = kdtree.query(
|
|
63
|
+
X[i0:i1],
|
|
64
|
+
k=1 + n_neighbors,
|
|
65
|
+
distance_upper_bound=distance_upper_bound,
|
|
66
|
+
workers=workers,
|
|
67
|
+
)
|
|
68
|
+
indices = cast(np.ndarray, indices)
|
|
69
|
+
inliers[i0:i1] = indices[:, -1] < kdtree.n
|
|
70
|
+
|
|
71
|
+
return inliers, kdtree
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_smoothed_density(
|
|
75
|
+
X,
|
|
76
|
+
inliers: np.ndarray | slice = slice(None),
|
|
77
|
+
sigma: float = 1.0,
|
|
78
|
+
sigma_low: float | None = None,
|
|
79
|
+
sigma_ramp_ax=-1,
|
|
80
|
+
bin_sizes=None,
|
|
81
|
+
bin_size_ratio=10.0,
|
|
82
|
+
min_bin_size=0.1,
|
|
83
|
+
ramp_min_bin_size=5.0,
|
|
84
|
+
max_n_bins=128,
|
|
85
|
+
min_n_bins=5,
|
|
86
|
+
) -> np.ndarray:
|
|
87
|
+
"""Get RBF density estimates for each X[i] and bandwidth in sigmas
|
|
88
|
+
|
|
89
|
+
Outliers will be marked with NaN KDEs. Please pass inliers, or else your
|
|
90
|
+
histogram is liable to be way too big.
|
|
91
|
+
"""
|
|
92
|
+
# figure out what bandwidths we'll be working on
|
|
93
|
+
|
|
94
|
+
if do_ramp := bool(sigma_low):
|
|
95
|
+
min_sigma = min(sigma, sigma_low)
|
|
96
|
+
else:
|
|
97
|
+
min_sigma = sigma
|
|
98
|
+
|
|
99
|
+
# histogram bin sizes -- not too big, not too small
|
|
100
|
+
infeats = X[inliers]
|
|
101
|
+
bin_sizes = np.full(X.shape[1], min_sigma / bin_size_ratio)
|
|
102
|
+
bin_sizes = np.maximum(min_bin_size, bin_sizes)
|
|
103
|
+
if do_ramp:
|
|
104
|
+
bin_sizes[sigma_ramp_ax] = max(bin_sizes[sigma_ramp_ax], ramp_min_bin_size)
|
|
105
|
+
|
|
106
|
+
# select bin edges
|
|
107
|
+
extents = np.c_[np.floor(infeats.min(0)), np.ceil(infeats.max(0))]
|
|
108
|
+
dextents = np.ptp(extents, 1)
|
|
109
|
+
if not (dextents > 0).all():
|
|
110
|
+
raise ValueError(
|
|
111
|
+
f"Issue in KDE. {dextents=} {infeats.shape=} {sigma=} {bin_sizes=}."
|
|
112
|
+
)
|
|
113
|
+
nbins = np.ceil(dextents / bin_sizes).astype(int)
|
|
114
|
+
nbins = nbins.clip(min_n_bins, max_n_bins)
|
|
115
|
+
bin_edges = [np.linspace(e[0], e[1], num=nb + 1) for e, nb in zip(extents, nbins)]
|
|
116
|
+
|
|
117
|
+
# compute histogram and figure out how big the bins actually were
|
|
118
|
+
raw_histogram, bin_edges = np.histogramdd(infeats, bins=bin_edges)
|
|
119
|
+
bin_sizes = np.array([(be[1] - be[0]) for be in bin_edges])
|
|
120
|
+
bin_centers = [0.5 * (be[1:] + be[:-1]) for be in bin_edges]
|
|
121
|
+
|
|
122
|
+
# normalize histogram to samples / volume
|
|
123
|
+
raw_histogram = raw_histogram / bin_sizes.prod()
|
|
124
|
+
|
|
125
|
+
# figure out how histogram should be filtered
|
|
126
|
+
hist = raw_histogram
|
|
127
|
+
if sigma is not None and sigma_low is None:
|
|
128
|
+
sigma_hist = sigma * np.reciprocal(bin_sizes)
|
|
129
|
+
hist = gaussian_filter(raw_histogram, sigma_hist)
|
|
130
|
+
elif sigma is not None and sigma_low is not None:
|
|
131
|
+
# filter by a sequence of bandwidths
|
|
132
|
+
sigma_hist = sigma * np.reciprocal(bin_sizes)
|
|
133
|
+
sigma_low_hist = sigma_low * np.reciprocal(bin_sizes)
|
|
134
|
+
ramp = np.linspace(
|
|
135
|
+
sigma_low_hist[sigma_ramp_ax],
|
|
136
|
+
sigma_hist[sigma_ramp_ax],
|
|
137
|
+
num=hist.shape[sigma_ramp_ax],
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# operate along the ramp axis
|
|
141
|
+
hist_move = np.moveaxis(hist, sigma_ramp_ax, 0)
|
|
142
|
+
hist_smoothed = hist_move.copy()
|
|
143
|
+
for j, sig in enumerate(ramp):
|
|
144
|
+
sig_move = sig.copy()
|
|
145
|
+
sig_move[0] = sig[sigma_ramp_ax]
|
|
146
|
+
sig_move[sigma_ramp_ax] = sig[0]
|
|
147
|
+
hist_smoothed[j] = gaussian_filter(hist_move, sig_move)[j] # sig_move
|
|
148
|
+
hist = np.moveaxis(hist_smoothed, 0, sigma_ramp_ax)
|
|
149
|
+
|
|
150
|
+
lerp = RegularGridInterpolator(bin_centers, hist, bounds_error=False)
|
|
151
|
+
low = np.array([np.min(bc) for bc in bin_centers])
|
|
152
|
+
high = np.array([np.max(bc) for bc in bin_centers])
|
|
153
|
+
dens = lerp(X.clip(low, high))
|
|
154
|
+
|
|
155
|
+
return dens
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def get_smoothed_density_ratio(
|
|
159
|
+
X,
|
|
160
|
+
*,
|
|
161
|
+
inliers: np.ndarray | slice = slice(None),
|
|
162
|
+
sigmas: list[float],
|
|
163
|
+
sigma_lows: list[float] | list[None] | None = None,
|
|
164
|
+
sigma_ramp_ax=-1,
|
|
165
|
+
bin_sizes=None,
|
|
166
|
+
bin_size_ratio=10.0,
|
|
167
|
+
min_bin_size=0.1,
|
|
168
|
+
ramp_min_bin_size=5.0,
|
|
169
|
+
max_n_bins=128,
|
|
170
|
+
min_n_bins=5,
|
|
171
|
+
) -> np.ndarray:
|
|
172
|
+
assert 0 < len(sigmas) <= 2
|
|
173
|
+
sigma_lows = sigma_lows or [None] * len(sigmas)
|
|
174
|
+
dens = [
|
|
175
|
+
get_smoothed_density(
|
|
176
|
+
X=X,
|
|
177
|
+
inliers=inliers,
|
|
178
|
+
sigma=sigma,
|
|
179
|
+
sigma_low=sigma_low,
|
|
180
|
+
sigma_ramp_ax=sigma_ramp_ax,
|
|
181
|
+
bin_sizes=bin_sizes,
|
|
182
|
+
bin_size_ratio=bin_size_ratio,
|
|
183
|
+
min_bin_size=min_bin_size,
|
|
184
|
+
ramp_min_bin_size=ramp_min_bin_size,
|
|
185
|
+
max_n_bins=max_n_bins,
|
|
186
|
+
min_n_bins=min_n_bins,
|
|
187
|
+
)
|
|
188
|
+
for sigma, sigma_low in zip(sigmas, sigma_lows)
|
|
189
|
+
]
|
|
190
|
+
if len(sigmas) == 1:
|
|
191
|
+
return dens[0]
|
|
192
|
+
else:
|
|
193
|
+
dens_ = dens[0]
|
|
194
|
+
dens_ /= dens[1]
|
|
195
|
+
np.nan_to_num(dens_, out=dens_) # type: ignore
|
|
196
|
+
return dens_
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def nearest_higher_density_neighbor(
|
|
200
|
+
kdtree,
|
|
201
|
+
density,
|
|
202
|
+
n_neighbors_search=20,
|
|
203
|
+
distance_upper_bound=5.0,
|
|
204
|
+
workers=1,
|
|
205
|
+
batch_size=2**16,
|
|
206
|
+
) -> np.ndarray:
|
|
207
|
+
nhdn = np.full(kdtree.n, kdtree.n, dtype=np.intp)
|
|
208
|
+
density_padded = np.pad(density, (0, 1), constant_values=np.inf)
|
|
209
|
+
|
|
210
|
+
for i0 in range(0, kdtree.n, batch_size):
|
|
211
|
+
i1 = min(kdtree.n, i0 + batch_size)
|
|
212
|
+
|
|
213
|
+
distances, indices = kdtree.query(
|
|
214
|
+
kdtree.data[i0:i1],
|
|
215
|
+
k=1 + n_neighbors_search,
|
|
216
|
+
distance_upper_bound=distance_upper_bound,
|
|
217
|
+
workers=workers,
|
|
218
|
+
)
|
|
219
|
+
# exclude self
|
|
220
|
+
distances = distances[:, 1:]
|
|
221
|
+
indices = indices[:, 1:]
|
|
222
|
+
missing = indices == kdtree.n
|
|
223
|
+
distances[missing] = np.inf
|
|
224
|
+
|
|
225
|
+
# find lowest distance higher density neighbor
|
|
226
|
+
bdens = density[i0:i1]
|
|
227
|
+
bneighb_dens = density_padded[indices]
|
|
228
|
+
is_lower_density = np.logical_or(
|
|
229
|
+
bneighb_dens <= bdens[:, None], np.isneginf(bneighb_dens)
|
|
230
|
+
)
|
|
231
|
+
distances[is_lower_density] = np.inf
|
|
232
|
+
indices[is_lower_density] = kdtree.n
|
|
233
|
+
|
|
234
|
+
nearest = distances.argmin(1, keepdims=True)
|
|
235
|
+
nhdn[i0:i1] = np.take_along_axis(indices, nearest, axis=1)[:, 0]
|
|
236
|
+
|
|
237
|
+
return nhdn
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def remove_border_points(
|
|
241
|
+
labels, density, kdtree, search_radius=1.0, n_neighbors_search=5, workers=1
|
|
242
|
+
):
|
|
243
|
+
distances, indices = kdtree.query(
|
|
244
|
+
kdtree.data,
|
|
245
|
+
k=1 + n_neighbors_search,
|
|
246
|
+
distance_upper_bound=search_radius,
|
|
247
|
+
workers=workers,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
labels_padded = np.pad(labels, (0, 1), constant_values=-2)
|
|
251
|
+
in_border = np.any(
|
|
252
|
+
labels_padded[indices] != labels_padded[indices[:, 0, None]], axis=1
|
|
253
|
+
)
|
|
254
|
+
in_border = np.flatnonzero(in_border)
|
|
255
|
+
labels_in_border = labels[in_border]
|
|
256
|
+
|
|
257
|
+
units = np.unique(labels)
|
|
258
|
+
new_labels = labels.copy()
|
|
259
|
+
for i, u in enumerate(units[units > 0]):
|
|
260
|
+
in_unit = np.flatnonzero(labels == u)
|
|
261
|
+
in_unit_border = in_border[labels_in_border == u]
|
|
262
|
+
if not in_unit_border.size:
|
|
263
|
+
continue
|
|
264
|
+
|
|
265
|
+
to_remove = density[in_unit] < density[in_unit_border].max()
|
|
266
|
+
new_labels[in_unit[to_remove]] = -1
|
|
267
|
+
|
|
268
|
+
return new_labels
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def guess_mode(
|
|
272
|
+
X,
|
|
273
|
+
sigma: float | Literal["rule_of_thumb"] = "rule_of_thumb",
|
|
274
|
+
outlier_neighbor_count=10,
|
|
275
|
+
outlier_sigma=3.0,
|
|
276
|
+
kdtree=None,
|
|
277
|
+
workers=1,
|
|
278
|
+
):
|
|
279
|
+
"""Use a KDE to guess the highest density point."""
|
|
280
|
+
n = len(X)
|
|
281
|
+
sigma0 = np.sqrt(X.var(axis=0).sum())
|
|
282
|
+
inliers, kdtree = kdtree_inliers(
|
|
283
|
+
X,
|
|
284
|
+
kdtree=kdtree,
|
|
285
|
+
n_neighbors=outlier_neighbor_count,
|
|
286
|
+
distance_upper_bound=outlier_sigma * sigma0,
|
|
287
|
+
workers=workers,
|
|
288
|
+
)
|
|
289
|
+
inliers = cast(np.ndarray, inliers)
|
|
290
|
+
|
|
291
|
+
if sigma == "rule_of_thumb":
|
|
292
|
+
sigma_dens = (
|
|
293
|
+
1.06
|
|
294
|
+
* np.linalg.norm(np.std(X[inliers], axis=0))
|
|
295
|
+
* np.power(inliers.sum(), -0.2)
|
|
296
|
+
)
|
|
297
|
+
else:
|
|
298
|
+
sigma_dens = sigma
|
|
299
|
+
|
|
300
|
+
density = get_smoothed_density(X, inliers=inliers, sigma=sigma_dens)
|
|
301
|
+
assert isinstance(density, np.ndarray)
|
|
302
|
+
assert density.shape == (n,)
|
|
303
|
+
|
|
304
|
+
return np.argmax(density)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def bucket_density_ratio(
|
|
308
|
+
X: np.ndarray,
|
|
309
|
+
scaled_geom: np.ndarray,
|
|
310
|
+
sigma: float,
|
|
311
|
+
sigma_regional: float,
|
|
312
|
+
max_sigma: float = 3.0,
|
|
313
|
+
computation_cfg: ComputationConfig | None = None,
|
|
314
|
+
workers=-1,
|
|
315
|
+
batch_size=8192,
|
|
316
|
+
):
|
|
317
|
+
"""
|
|
318
|
+
|
|
319
|
+
Algorithm:
|
|
320
|
+
- Assign spikes to channels with a geom-kdtree (make sure geom's x
|
|
321
|
+
scale matches the clustering feature's x scale.)
|
|
322
|
+
- Let Dc be pairwise distances between channels
|
|
323
|
+
- Compute sparse distance matrix. If Dx is pairwise distance between
|
|
324
|
+
all pairs of spikes, we just want the entries with Dx<max_dist.
|
|
325
|
+
- Note
|
|
326
|
+
|ci-cj| = |(ci-xi) + (xj-cj) + (xi-xj)|
|
|
327
|
+
<= 2C + |xi-xj|
|
|
328
|
+
Thus we need not consider i,j s.t. |ci-cj|-2C>max_dist.
|
|
329
|
+
"""
|
|
330
|
+
computation_cfg = ensure_computation_config(computation_cfg)
|
|
331
|
+
with timer("geom bucketing"):
|
|
332
|
+
geom_kdt = KDTree(scaled_geom, leafsize=1)
|
|
333
|
+
dists, channels = geom_kdt.query(X[:, :2], workers=workers)
|
|
334
|
+
|
|
335
|
+
max_dist_by_chan = np.zeros(len(scaled_geom))
|
|
336
|
+
for c in range(len(scaled_geom)):
|
|
337
|
+
inc = np.flatnonzero(channels == c)
|
|
338
|
+
if inc.size:
|
|
339
|
+
max_dist_by_chan[c] = dists[inc].max()
|
|
340
|
+
|
|
341
|
+
larger_sigma = max(sigma, sigma_regional or 0.0)
|
|
342
|
+
max_dist = max_sigma * larger_sigma
|
|
343
|
+
|
|
344
|
+
lb = squareform(pdist(scaled_geom))
|
|
345
|
+
lb -= max_dist_by_chan[:, None]
|
|
346
|
+
lb -= max_dist_by_chan
|
|
347
|
+
chans_pair_mask = lb <= max_dist
|
|
348
|
+
print(f"{chans_pair_mask.sum(1).min()=}")
|
|
349
|
+
print(f"{chans_pair_mask.sum(1).max()=}")
|
|
350
|
+
print(f"{max_dist_by_chan.max()=}")
|
|
351
|
+
print(f"{max_dist_by_chan.min()=}")
|
|
352
|
+
|
|
353
|
+
device = computation_cfg.actual_device()
|
|
354
|
+
channels = torch.asarray(channels, device=device, dtype=torch.long)
|
|
355
|
+
Xt = torch.asarray(X, device=device, dtype=torch.float)
|
|
356
|
+
chans_pair_mask = torch.asarray(chans_pair_mask, device=device, dtype=torch.bool)
|
|
357
|
+
density_ratio = torch.full((len(X),), -torch.inf, device=device)
|
|
358
|
+
|
|
359
|
+
# for c in trange(len(scaled_geom), desc="bktdens"):
|
|
360
|
+
for c in trange(25, desc="bktdens"):
|
|
361
|
+
(inc,) = (channels == c).nonzero(as_tuple=True)
|
|
362
|
+
(friends,) = chans_pair_mask[c][channels].nonzero(as_tuple=True)
|
|
363
|
+
density_ratio[inc] = _local_sparse_dens_ratio(
|
|
364
|
+
Xt, inc, friends, max_dist, sigma, sigma_regional, batch_size=batch_size
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
return density_ratio
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@torch.jit.script
|
|
371
|
+
def _local_sparse_dens_ratio(
|
|
372
|
+
X: Tensor,
|
|
373
|
+
inc: Tensor,
|
|
374
|
+
inf: Tensor,
|
|
375
|
+
max_dist: float,
|
|
376
|
+
sigma0: float,
|
|
377
|
+
sigma1: float,
|
|
378
|
+
batch_size: int = 1024,
|
|
379
|
+
):
|
|
380
|
+
nc = inc.shape[0]
|
|
381
|
+
nf = inf.shape[0]
|
|
382
|
+
dens_ratio = X.new_empty(nc)
|
|
383
|
+
|
|
384
|
+
s0_factor = (1.0 / (torch.sqrt(torch.tensor(2.0)) * sigma0)).to(X)
|
|
385
|
+
s1_factor = (1.0 / (torch.sqrt(torch.tensor(2.0)) * sigma1)).to(X)
|
|
386
|
+
|
|
387
|
+
for ic0 in range(0, nc, batch_size):
|
|
388
|
+
ic1 = min(nc, ic0 + batch_size)
|
|
389
|
+
Xc = X[inc[ic0:ic1]]
|
|
390
|
+
for if0 in range(0, nf, batch_size):
|
|
391
|
+
if1 = min(nf, if0 + batch_size)
|
|
392
|
+
Xf = X[inf[if0:if1]]
|
|
393
|
+
|
|
394
|
+
d = torch.cdist(Xc, Xf)
|
|
395
|
+
F.threshold(d, threshold=max_dist, value=torch.inf, inplace=True)
|
|
396
|
+
|
|
397
|
+
d0 = d * s0_factor
|
|
398
|
+
d1 = d * s1_factor
|
|
399
|
+
d0.square_().neg_().exp_()
|
|
400
|
+
d1.square_().neg_().exp_()
|
|
401
|
+
d0 = d0.sum(1)
|
|
402
|
+
d1 = d1.sum(1)
|
|
403
|
+
torch.divide(d0, d1, out=dens_ratio[ic0:ic1])
|
|
404
|
+
|
|
405
|
+
return dens_ratio.nan_to_num_()
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def kdt_density(
|
|
409
|
+
kdtree: KDTree,
|
|
410
|
+
X: np.ndarray,
|
|
411
|
+
sigma: float,
|
|
412
|
+
sigma_regional: float,
|
|
413
|
+
batch_size=2048,
|
|
414
|
+
max_sigma: float = 3.0,
|
|
415
|
+
n_threads=8,
|
|
416
|
+
):
|
|
417
|
+
n, d = X.shape
|
|
418
|
+
max_dist = max_sigma * sigma_regional
|
|
419
|
+
|
|
420
|
+
jobs = range(0, n, batch_size)
|
|
421
|
+
n_jobs, Executor, context = get_pool(n_jobs=n_threads, cls="ThreadPoolExecutor") # type: ignore
|
|
422
|
+
with Executor( # type: ignore
|
|
423
|
+
max_workers=n_jobs,
|
|
424
|
+
mp_context=context,
|
|
425
|
+
initializer=_kdtdens_init,
|
|
426
|
+
initargs=(kdtree, X, batch_size, sigma, sigma_regional, max_dist),
|
|
427
|
+
) as pool:
|
|
428
|
+
density = np.full((n,), -np.inf)
|
|
429
|
+
for i0, i1, dens in tqdm(
|
|
430
|
+
pool.map(_kdtdens_job, jobs),
|
|
431
|
+
total=len(jobs),
|
|
432
|
+
smoothing=0.0,
|
|
433
|
+
desc=f"KDTdens[{n_jobs}]",
|
|
434
|
+
):
|
|
435
|
+
density[i0:i1] = dens
|
|
436
|
+
np.nan_to_num(density, copy=False)
|
|
437
|
+
return density
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
_kdtdens_ctx = local()
|
|
441
|
+
_kdtdens_ctx.sargs = None
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _kdtdens_init(kdtree, X, batch_size, sigma, sigma_regional, max_dist):
|
|
445
|
+
global _kdtdens_ctx
|
|
446
|
+
_kdtdens_ctx.sargs = (kdtree, X, batch_size, sigma, sigma_regional, max_dist)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _kdtdens_job(i0):
|
|
450
|
+
global _kdtdens_ctx
|
|
451
|
+
kdtree, X, batch_size, sigma, sigma_regional, max_dist = _kdtdens_ctx.sargs
|
|
452
|
+
i1 = min(kdtree.n, i0 + batch_size)
|
|
453
|
+
batch_kdt = KDTree(X[i0:i1])
|
|
454
|
+
sdm = batch_kdt.sparse_distance_matrix(kdtree, max_dist, output_type="ndarray")
|
|
455
|
+
ii = sdm["i"]
|
|
456
|
+
jj = sdm["j"]
|
|
457
|
+
v_local = sdm["v"]
|
|
458
|
+
v_regional = sdm["v"].copy()
|
|
459
|
+
_gausskernel(v_local, sigma)
|
|
460
|
+
_gausskernel(v_regional, sigma_regional)
|
|
461
|
+
coo_local = coo_array(
|
|
462
|
+
(v_local, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
|
|
463
|
+
)
|
|
464
|
+
coo_regional = coo_array(
|
|
465
|
+
(v_regional, (ii, jj)), shape=(batch_kdt.n, kdtree.n), dtype=v_local.dtype
|
|
466
|
+
)
|
|
467
|
+
dens_local = coo_local.sum(axis=1)
|
|
468
|
+
dens_regional = coo_regional.sum(axis=1)
|
|
469
|
+
dens_local /= dens_regional
|
|
470
|
+
return i0, i1, dens_local
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
@numba.jit(
|
|
474
|
+
parallel=False, nopython=True, boundscheck=False, nogil=True, error_model="numpy"
|
|
475
|
+
)
|
|
476
|
+
def _gausskernel(v: np.ndarray, sigma: float):
|
|
477
|
+
const = 1.0 / (np.sqrt(2.0) * sigma)
|
|
478
|
+
for i in range(v.size):
|
|
479
|
+
v[i] = np.exp(-np.square(v[i] * const))
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def knn_density(
|
|
483
|
+
kdtree, X, k, distance_upper_bound, batch_size=2**12, workers=-1, sigma=None
|
|
484
|
+
):
|
|
485
|
+
n, d = X.shape
|
|
486
|
+
density = np.full(n, -np.inf, dtype="float32")
|
|
487
|
+
|
|
488
|
+
for i0 in range(0, n, batch_size):
|
|
489
|
+
i1 = min(n, i0 + batch_size)
|
|
490
|
+
distances, indices = kdtree.query(
|
|
491
|
+
X[i0:i1],
|
|
492
|
+
k=k,
|
|
493
|
+
distance_upper_bound=distance_upper_bound,
|
|
494
|
+
workers=workers,
|
|
495
|
+
)
|
|
496
|
+
if sigma is None:
|
|
497
|
+
np.nan_to_num(distances, copy=False, posinf=-np.inf)
|
|
498
|
+
n_neighbs = (indices < kdtree.n).sum(axis=1)
|
|
499
|
+
max_dist = distances[:, 1:].max(1)
|
|
500
|
+
np.nan_to_num(max_dist, copy=False, posinf=-np.inf)
|
|
501
|
+
# forget the sphere factor and the factor of n.
|
|
502
|
+
density[i0:i1] = n_neighbs / (max_dist**d)
|
|
503
|
+
else:
|
|
504
|
+
kernel = np.square(distances, out=distances)
|
|
505
|
+
kernel *= -(sigma**-2)
|
|
506
|
+
np.exp(kernel, out=kernel)
|
|
507
|
+
density[i0:i1] = kernel.sum(1)
|
|
508
|
+
|
|
509
|
+
return density
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def density_peaks(
|
|
513
|
+
X,
|
|
514
|
+
kdtree=None,
|
|
515
|
+
density=None,
|
|
516
|
+
knn_k=None,
|
|
517
|
+
use_knn=False,
|
|
518
|
+
use_histograms=False,
|
|
519
|
+
sigma_local=5.0,
|
|
520
|
+
sigma_regional=None,
|
|
521
|
+
outlier_neighbor_count: int | None = 10,
|
|
522
|
+
outlier_radius: float | None = 25.0,
|
|
523
|
+
n_neighbors_search=20,
|
|
524
|
+
radius_search=25.0,
|
|
525
|
+
noise_density=0.0,
|
|
526
|
+
remove_clusters_smaller_than=10,
|
|
527
|
+
remove_borders=False,
|
|
528
|
+
border_search_radius=10.0,
|
|
529
|
+
border_search_neighbors=3,
|
|
530
|
+
inlier_dims=slice(0, 2),
|
|
531
|
+
leafsize=24,
|
|
532
|
+
workers=-1,
|
|
533
|
+
):
|
|
534
|
+
"""Density peaks clustering as described by Rodriguez and Laio, but...
|
|
535
|
+
|
|
536
|
+
This implementation has three tweaks from original DPC
|
|
537
|
+
- KDE density estimation, implemented by smoothing histograms
|
|
538
|
+
- "Sharpening": density can be a ratio of KDEs, like a sharpening kernel
|
|
539
|
+
- Noise: you can throw away points with too low of a density (ratio)
|
|
540
|
+
"""
|
|
541
|
+
n = len(X)
|
|
542
|
+
|
|
543
|
+
radius_search = radius_search * np.sqrt(X.shape[1])
|
|
544
|
+
dim_arange = np.arange(X.shape[1])
|
|
545
|
+
inlier_dim_ixs = dim_arange[inlier_dims]
|
|
546
|
+
if outlier_radius is not None:
|
|
547
|
+
outlier_radius = outlier_radius * np.sqrt(inlier_dim_ixs.size)
|
|
548
|
+
inliers, kdtree = kdtree_inliers(
|
|
549
|
+
X[:, inlier_dims],
|
|
550
|
+
kdtree=kdtree,
|
|
551
|
+
n_neighbors=outlier_neighbor_count,
|
|
552
|
+
distance_upper_bound=outlier_radius,
|
|
553
|
+
workers=workers,
|
|
554
|
+
leafsize=leafsize,
|
|
555
|
+
)
|
|
556
|
+
if not np.array_equal(inlier_dim_ixs, dim_arange):
|
|
557
|
+
kdtree = KDTree(X)
|
|
558
|
+
|
|
559
|
+
if density is None:
|
|
560
|
+
if sigma_regional and not use_histograms:
|
|
561
|
+
density = kdt_density(
|
|
562
|
+
kdtree,
|
|
563
|
+
X,
|
|
564
|
+
sigma=sigma_local,
|
|
565
|
+
sigma_regional=sigma_regional,
|
|
566
|
+
n_threads=workers,
|
|
567
|
+
)
|
|
568
|
+
elif use_knn:
|
|
569
|
+
density = knn_density(
|
|
570
|
+
kdtree, X, k=knn_k, distance_upper_bound=radius_search, workers=workers
|
|
571
|
+
)
|
|
572
|
+
elif use_histograms:
|
|
573
|
+
sigmas = [sigma_local]
|
|
574
|
+
if sigma_regional is not None:
|
|
575
|
+
sigmas.append(sigma_regional)
|
|
576
|
+
density = get_smoothed_density_ratio(X, inliers=inliers, sigmas=sigmas)
|
|
577
|
+
else:
|
|
578
|
+
density = knn_density(
|
|
579
|
+
kdtree,
|
|
580
|
+
X,
|
|
581
|
+
k=n_neighbors_search,
|
|
582
|
+
distance_upper_bound=radius_search,
|
|
583
|
+
workers=workers,
|
|
584
|
+
sigma=sigma_local,
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
nhdn = nearest_higher_density_neighbor(
|
|
588
|
+
kdtree,
|
|
589
|
+
density,
|
|
590
|
+
n_neighbors_search=n_neighbors_search,
|
|
591
|
+
distance_upper_bound=radius_search,
|
|
592
|
+
workers=workers,
|
|
593
|
+
)
|
|
594
|
+
if noise_density:
|
|
595
|
+
nhdn[density < noise_density] = n
|
|
596
|
+
|
|
597
|
+
nhdn = nhdn.astype(np.intc)
|
|
598
|
+
has_nhdn = np.flatnonzero(nhdn < n).astype(np.intc)
|
|
599
|
+
data = np.ones(has_nhdn.size)
|
|
600
|
+
graph = coo_array((data, (nhdn[has_nhdn], has_nhdn)), shape=(n, n))
|
|
601
|
+
ncc, labels = connected_components(graph)
|
|
602
|
+
|
|
603
|
+
if remove_borders:
|
|
604
|
+
labels = remove_border_points(
|
|
605
|
+
labels,
|
|
606
|
+
density,
|
|
607
|
+
kdtree,
|
|
608
|
+
search_radius=border_search_radius,
|
|
609
|
+
n_neighbors_search=border_search_neighbors,
|
|
610
|
+
workers=workers,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
if remove_clusters_smaller_than:
|
|
614
|
+
labels = decrumb(labels, min_size=remove_clusters_smaller_than, in_place=True)
|
|
615
|
+
|
|
616
|
+
return dict(
|
|
617
|
+
density=density,
|
|
618
|
+
nhdn=nhdn,
|
|
619
|
+
labels=labels,
|
|
620
|
+
kdtree=kdtree,
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def nearest_neighbor_assign(
|
|
625
|
+
kdtree, tree_labels, X_other, radius_search=5.0, workers=-1
|
|
626
|
+
):
|
|
627
|
+
_, inds = kdtree.query(
|
|
628
|
+
X_other, k=1, distance_upper_bound=radius_search, workers=workers
|
|
629
|
+
)
|
|
630
|
+
found = np.flatnonzero(inds < kdtree.n)
|
|
631
|
+
other_labels = np.full(len(X_other), -1, dtype=tree_labels.dtype)
|
|
632
|
+
other_labels[found] = tree_labels[inds[found]]
|
|
633
|
+
return other_labels
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def sparse_iso_hellinger(centroids, sigma, hellinger_threshold=0.25, centroids_b=None):
|
|
637
|
+
"""Sparse Hellinger distance between isotropic Gaussian components"""
|
|
638
|
+
c = centroids * (1.0 / (sigma * np.sqrt(8.0)))
|
|
639
|
+
kdt = KDTree(c)
|
|
640
|
+
if centroids_b is None:
|
|
641
|
+
kdt_b = kdt
|
|
642
|
+
else:
|
|
643
|
+
c_b = centroids_b * (1.0 / (sigma * np.sqrt(8.0)))
|
|
644
|
+
kdt_b = KDTree(c_b)
|
|
645
|
+
max_distance = np.sqrt(-np.log(1 - hellinger_threshold))
|
|
646
|
+
dists = kdt.sparse_distance_matrix(
|
|
647
|
+
kdt_b, max_distance=max_distance, output_type="ndarray"
|
|
648
|
+
)
|
|
649
|
+
vals = dists["v"]
|
|
650
|
+
np.square(vals, out=vals)
|
|
651
|
+
vals *= -1
|
|
652
|
+
np.exp(vals, out=vals) # now BC
|
|
653
|
+
np.subtract(1, vals, out=vals) # and now hell^2.
|
|
654
|
+
np.abs(vals, out=vals) # numerical -0
|
|
655
|
+
np.sqrt(vals, out=vals) # and now hell
|
|
656
|
+
dists = coo_array((vals, (dists["j"], dists["i"])), shape=(kdt_b.n, kdt.n))
|
|
657
|
+
return dists
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def coo_nhdn(coo, densities) -> np.ndarray:
|
|
661
|
+
ii, jj = coo.coords
|
|
662
|
+
dists = coo.data.copy()
|
|
663
|
+
M = dists.max()
|
|
664
|
+
dens_ii = densities[ii]
|
|
665
|
+
dens_jj = densities[jj]
|
|
666
|
+
not_higher = dens_jj <= dens_ii
|
|
667
|
+
|
|
668
|
+
# only higher density
|
|
669
|
+
dists[not_higher] = np.inf
|
|
670
|
+
dists[ii == jj] = 2 * M + 1 # i am my last resort
|
|
671
|
+
coo2 = coo_array((dists, (ii, jj)), shape=coo.shape)
|
|
672
|
+
# argmin == nearest
|
|
673
|
+
nhdn = coo2.tocsc().argmin(axis=1, explicit=True)
|
|
674
|
+
nhdn = np.atleast_1d(nhdn)
|
|
675
|
+
assert nhdn.shape == densities.shape[:1]
|
|
676
|
+
return nhdn
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def gmm_density_peaks(
|
|
680
|
+
X,
|
|
681
|
+
channels,
|
|
682
|
+
outlier_neighbor_count=10,
|
|
683
|
+
outlier_radius=25.0,
|
|
684
|
+
remove_clusters_smaller_than=50,
|
|
685
|
+
workers=-1,
|
|
686
|
+
n_initializations=10,
|
|
687
|
+
n_iter=50,
|
|
688
|
+
max_components_per_channel=20,
|
|
689
|
+
min_spikes_per_component=10,
|
|
690
|
+
random_state=0,
|
|
691
|
+
kmeanspp_min_dist=0.0,
|
|
692
|
+
hellinger_cutoff=0.95,
|
|
693
|
+
hellinger_strong=0.0,
|
|
694
|
+
hellinger_weak=0.999,
|
|
695
|
+
max_sigma=5.0,
|
|
696
|
+
sigma_atol=1e-4,
|
|
697
|
+
max_samples=2_000_000,
|
|
698
|
+
noise_const_dims=None,
|
|
699
|
+
show_progress=True,
|
|
700
|
+
use_hellinger=False,
|
|
701
|
+
mop=True,
|
|
702
|
+
n_neighbors_search=20,
|
|
703
|
+
device=None,
|
|
704
|
+
):
|
|
705
|
+
"""Density peaks clustering via an isotropic GMM density estimate
|
|
706
|
+
|
|
707
|
+
Idea: use an overfitted isotropic GMM (i.e. lots of components) to capture the
|
|
708
|
+
density in a point cloud. Then, each GMM component is grouped together with its
|
|
709
|
+
nearest higher density (i.e., higher mixing proportion, since isotropic) neighbor,
|
|
710
|
+
much like the usual density peaks clustering algorithm.
|
|
711
|
+
|
|
712
|
+
Neighbor-ness is determined by a Hellinger overlap criterion. Contrasted with the
|
|
713
|
+
smoothed histogram DPC algorithm elsewhere in this file, this algorithm is a bit
|
|
714
|
+
more adaptive, and that neighbor criterion is easier to tune than the k-d tree query
|
|
715
|
+
parameters which define the point-to-point neighbor criterion for the other algorithm.
|
|
716
|
+
|
|
717
|
+
TODO: parametrize as max_components_per_square_micron or something instead of per
|
|
718
|
+
channel so that this has more hope of transferring from probe to probe.
|
|
719
|
+
|
|
720
|
+
TODO: see parallelism todo in gmm_kmeans. This could be a lot faster.
|
|
721
|
+
|
|
722
|
+
Arguments
|
|
723
|
+
---------
|
|
724
|
+
X : (N_spikes, n_features) array
|
|
725
|
+
For instance, the features property of a SimpleMatrix object as obtained
|
|
726
|
+
from get_clustering_features()
|
|
727
|
+
channels: (N_spikes,) array
|
|
728
|
+
The channels to which each spike belongs.
|
|
729
|
+
TODO: currently used in controlling the number of components, but it may
|
|
730
|
+
change when the TODO above is implemented.
|
|
731
|
+
"""
|
|
732
|
+
from .kmeans import truncated_kmeans
|
|
733
|
+
|
|
734
|
+
if device is None:
|
|
735
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
736
|
+
device = torch.device(device)
|
|
737
|
+
|
|
738
|
+
n = len(X)
|
|
739
|
+
if n > max_samples:
|
|
740
|
+
random_state = np.random.default_rng(random_state)
|
|
741
|
+
choices = random_state.choice(n, size=max_samples, replace=False)
|
|
742
|
+
choices.sort()
|
|
743
|
+
else:
|
|
744
|
+
choices = slice(None)
|
|
745
|
+
|
|
746
|
+
Xi = X[choices]
|
|
747
|
+
nc = len(Xi)
|
|
748
|
+
inliers, kdtree = kdtree_inliers(
|
|
749
|
+
Xi,
|
|
750
|
+
n_neighbors=outlier_neighbor_count,
|
|
751
|
+
distance_upper_bound=outlier_radius * np.sqrt(X.shape[1]),
|
|
752
|
+
workers=workers or 1,
|
|
753
|
+
)
|
|
754
|
+
if not isinstance(choices, slice):
|
|
755
|
+
inliers = choices[inliers]
|
|
756
|
+
|
|
757
|
+
Xi = X[inliers]
|
|
758
|
+
ni = len(Xi)
|
|
759
|
+
uchans, cchans = np.unique(channels[inliers], return_counts=True)
|
|
760
|
+
comps_per_chan = np.minimum(
|
|
761
|
+
max_components_per_channel,
|
|
762
|
+
np.ceil(cchans / min_spikes_per_component).astype(int),
|
|
763
|
+
)
|
|
764
|
+
n_components = min(
|
|
765
|
+
comps_per_chan.sum(), int(np.ceil(ni / min_spikes_per_component))
|
|
766
|
+
)
|
|
767
|
+
logger.dartsortdebug(
|
|
768
|
+
f"gmmdpc: {n_components} components for {cchans.size} channels"
|
|
769
|
+
f"over {ni}/{nc} inliers"
|
|
770
|
+
)
|
|
771
|
+
res = truncated_kmeans(
|
|
772
|
+
Xi,
|
|
773
|
+
max_sigma=max_sigma,
|
|
774
|
+
n_components=n_components,
|
|
775
|
+
n_initializations=n_initializations,
|
|
776
|
+
random_state=random_state,
|
|
777
|
+
n_iter=n_iter,
|
|
778
|
+
dirichlet_alpha=1.0 / n_components,
|
|
779
|
+
kmeanspp_min_dist=kmeanspp_min_dist,
|
|
780
|
+
device=device,
|
|
781
|
+
show_progress=show_progress,
|
|
782
|
+
noise_const_dims=noise_const_dims,
|
|
783
|
+
with_log_likelihoods=mop or not use_hellinger,
|
|
784
|
+
sigma_atol=sigma_atol,
|
|
785
|
+
)
|
|
786
|
+
res["n_components"] = n_components
|
|
787
|
+
res["kmeans_labels"] = res["labels"]
|
|
788
|
+
n_components = len(cast(torch.Tensor, res["centroids"]))
|
|
789
|
+
res["n_components_kept"] = n_components
|
|
790
|
+
print("nosqrt")
|
|
791
|
+
maxdist = max_sigma * float(res["sigma"]) # * np.sqrt(X.shape[1])
|
|
792
|
+
if use_hellinger:
|
|
793
|
+
centroids = cast(torch.Tensor, res["centroids"]).numpy(force=True)
|
|
794
|
+
log_proportions = cast(torch.Tensor, res["log_proportions"]).numpy(force=True)
|
|
795
|
+
labels = gmmdpc_hellinger(
|
|
796
|
+
X=X,
|
|
797
|
+
centroids=centroids,
|
|
798
|
+
sigma=res["sigma"],
|
|
799
|
+
log_proportions=log_proportions,
|
|
800
|
+
maxdist=maxdist,
|
|
801
|
+
workers=workers,
|
|
802
|
+
hellinger_cutoff=hellinger_cutoff,
|
|
803
|
+
hellinger_strong=hellinger_strong,
|
|
804
|
+
hellinger_weak=hellinger_weak,
|
|
805
|
+
)
|
|
806
|
+
else:
|
|
807
|
+
log_likelihoods = cast(torch.Tensor, res["log_likelihoods"]).numpy(force=True)
|
|
808
|
+
labels = gmmdpc_classic(
|
|
809
|
+
X=X,
|
|
810
|
+
inliers=inliers,
|
|
811
|
+
log_likelihoods=log_likelihoods,
|
|
812
|
+
maxdist=maxdist,
|
|
813
|
+
workers=workers,
|
|
814
|
+
n_neighbors_search=n_neighbors_search,
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
if remove_clusters_smaller_than:
|
|
818
|
+
if show_progress:
|
|
819
|
+
logger.info("Clean...")
|
|
820
|
+
labels = decrumb(labels, min_size=remove_clusters_smaller_than, in_place=True)
|
|
821
|
+
res["labels"] = labels # type: ignore
|
|
822
|
+
|
|
823
|
+
if mop:
|
|
824
|
+
# use k-dtree dpc to mop up any remaining clusters...
|
|
825
|
+
inliers = np.flatnonzero(inliers)
|
|
826
|
+
discarded = np.flatnonzero(labels[inliers] < 0)
|
|
827
|
+
log_likelihoods = cast(torch.Tensor, res["log_likelihoods"]).numpy(force=True)
|
|
828
|
+
mop_res = density_peaks(
|
|
829
|
+
Xi[discarded],
|
|
830
|
+
density=log_likelihoods[discarded],
|
|
831
|
+
outlier_radius=outlier_radius,
|
|
832
|
+
outlier_neighbor_count=outlier_neighbor_count,
|
|
833
|
+
radius_search=maxdist,
|
|
834
|
+
workers=workers,
|
|
835
|
+
remove_clusters_smaller_than=remove_clusters_smaller_than,
|
|
836
|
+
n_neighbors_search=n_neighbors_search,
|
|
837
|
+
)
|
|
838
|
+
mopped = np.flatnonzero(mop_res["labels"] >= 0)
|
|
839
|
+
mopped_labels = mop_res["labels"][mopped]
|
|
840
|
+
mopped_labels += labels.max() + 1
|
|
841
|
+
labels[inliers[discarded[mopped]]] = mopped_labels
|
|
842
|
+
|
|
843
|
+
return res
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def gmmdpc_classic(
|
|
847
|
+
X,
|
|
848
|
+
inliers,
|
|
849
|
+
log_likelihoods,
|
|
850
|
+
maxdist,
|
|
851
|
+
workers=-1,
|
|
852
|
+
n_neighbors_search=20,
|
|
853
|
+
):
|
|
854
|
+
kdtree_res = density_peaks(
|
|
855
|
+
X[inliers],
|
|
856
|
+
density=log_likelihoods,
|
|
857
|
+
outlier_radius=None,
|
|
858
|
+
outlier_neighbor_count=None,
|
|
859
|
+
radius_search=maxdist,
|
|
860
|
+
workers=workers,
|
|
861
|
+
remove_clusters_smaller_than=0,
|
|
862
|
+
n_neighbors_search=n_neighbors_search,
|
|
863
|
+
)
|
|
864
|
+
labels = nearest_neighbor_assign(
|
|
865
|
+
kdtree=kdtree_res["kdtree"],
|
|
866
|
+
tree_labels=kdtree_res["labels"],
|
|
867
|
+
X_other=X,
|
|
868
|
+
radius_search=maxdist,
|
|
869
|
+
workers=workers,
|
|
870
|
+
)
|
|
871
|
+
return labels
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def gmmdpc_hellinger(
|
|
875
|
+
X,
|
|
876
|
+
centroids,
|
|
877
|
+
sigma,
|
|
878
|
+
log_proportions,
|
|
879
|
+
maxdist: float,
|
|
880
|
+
workers=-1,
|
|
881
|
+
hellinger_cutoff=0.95,
|
|
882
|
+
hellinger_strong=0.0,
|
|
883
|
+
hellinger_weak=0.999,
|
|
884
|
+
) -> np.ndarray:
|
|
885
|
+
"""Density peaks clustering via isotropic GMM (Hellinger version)
|
|
886
|
+
|
|
887
|
+
Spikes are assigned to isotropic Gaussian clusters by k-means. Then
|
|
888
|
+
density peaks is carried out on centroids, where the notion of density
|
|
889
|
+
for a centroid is its log proportion in the mixture.
|
|
890
|
+
|
|
891
|
+
As in any DPC implementation, the key is the neighbor criterion. Here,
|
|
892
|
+
components are neighbors if they meet certain criteria about Hellinger
|
|
893
|
+
distance, which is like cosine distance for probability distributions.
|
|
894
|
+
"""
|
|
895
|
+
logger.dartsortdebug("GMMDPC: Hellinger...")
|
|
896
|
+
coo = sparse_iso_hellinger(
|
|
897
|
+
centroids,
|
|
898
|
+
sigma,
|
|
899
|
+
hellinger_threshold=hellinger_cutoff,
|
|
900
|
+
)
|
|
901
|
+
nhdn = coo_nhdn(coo, log_proportions)
|
|
902
|
+
assert nhdn.shape == (len(centroids),)
|
|
903
|
+
n_components = len(centroids)
|
|
904
|
+
ii = np.arange(len(nhdn))
|
|
905
|
+
jj = nhdn
|
|
906
|
+
|
|
907
|
+
if hellinger_strong:
|
|
908
|
+
strong = np.flatnonzero(coo.data < hellinger_strong)
|
|
909
|
+
ii = np.concatenate([ii, coo.coords[0][strong]])
|
|
910
|
+
jj = np.concatenate([jj, coo.coords[1][strong]])
|
|
911
|
+
order = np.argsort(ii, kind="stable")
|
|
912
|
+
ii = ii[order]
|
|
913
|
+
jj = jj[order]
|
|
914
|
+
|
|
915
|
+
if hellinger_weak:
|
|
916
|
+
disconnected = nhdn == ii
|
|
917
|
+
disconnected = np.logical_and(
|
|
918
|
+
disconnected,
|
|
919
|
+
np.logical_not(np.isin(ii, nhdn[np.logical_not(disconnected)])),
|
|
920
|
+
)
|
|
921
|
+
disconnected = np.flatnonzero(disconnected)
|
|
922
|
+
else:
|
|
923
|
+
disconnected = np.array([], dtype=int)
|
|
924
|
+
|
|
925
|
+
if hellinger_weak and disconnected.size:
|
|
926
|
+
coo_weak = sparse_iso_hellinger(
|
|
927
|
+
centroids[disconnected],
|
|
928
|
+
sigma,
|
|
929
|
+
hellinger_threshold=hellinger_weak,
|
|
930
|
+
# centroids_b=centroids,
|
|
931
|
+
)
|
|
932
|
+
coo_weak = coo_array(
|
|
933
|
+
(coo_weak.data, (coo_weak.coords[1], coo_weak.coords[0])),
|
|
934
|
+
shape=(disconnected.size, disconnected.size),
|
|
935
|
+
# (coo_weak.data, (disconnected[coo_weak.coords[1]], coo_weak.coords[0])),
|
|
936
|
+
# shape=(n_components, n_components),
|
|
937
|
+
)
|
|
938
|
+
weak_nhdn = coo_nhdn(coo_weak, log_proportions[disconnected])
|
|
939
|
+
# jj[disconnected] = weak_nhdn[disconnected]
|
|
940
|
+
print("weak2")
|
|
941
|
+
jj[disconnected] = disconnected[weak_nhdn]
|
|
942
|
+
|
|
943
|
+
_1 = np.ones((1,), dtype="float32")
|
|
944
|
+
_1 = np.broadcast_to(_1, jj.shape)
|
|
945
|
+
nhdn_coo = coo_array((_1, (ii, jj)), shape=(n_components, n_components))
|
|
946
|
+
_, labels = connected_components(nhdn_coo)
|
|
947
|
+
assert labels.shape == (n_components,)
|
|
948
|
+
labels_padded = np.pad(labels, [(0, 1)], constant_values=-1)
|
|
949
|
+
|
|
950
|
+
ckdt = KDTree(centroids)
|
|
951
|
+
_, q = ckdt.query(X, workers=workers or 1, distance_upper_bound=maxdist)
|
|
952
|
+
labels = labels_padded[q]
|
|
953
|
+
return labels
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
# -- versions used in UHD project
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def density_peaks_fancy(
|
|
960
|
+
xyza,
|
|
961
|
+
amps,
|
|
962
|
+
sorting,
|
|
963
|
+
motion_est,
|
|
964
|
+
geom,
|
|
965
|
+
sigma_local=5.0,
|
|
966
|
+
sigma_regional=None,
|
|
967
|
+
outlier_neighbor_count=10,
|
|
968
|
+
outlier_radius=25.0,
|
|
969
|
+
n_neighbors_search=20,
|
|
970
|
+
radius_search=5.0,
|
|
971
|
+
noise_density=0.0,
|
|
972
|
+
remove_clusters_smaller_than=10,
|
|
973
|
+
workers=-1,
|
|
974
|
+
scales=(1.0, 1.0, 50.0),
|
|
975
|
+
amp_log_c=5.0,
|
|
976
|
+
sigma_local_low: float | None = None,
|
|
977
|
+
sigma_regional_low: float | None = None,
|
|
978
|
+
distance_dependent_noise_density=False,
|
|
979
|
+
attach_density_feature=False,
|
|
980
|
+
triage_quantile_per_cluster=0.0,
|
|
981
|
+
ramp_triage_per_cluster=False,
|
|
982
|
+
triage_quantile_before_clustering=0.0,
|
|
983
|
+
amp_no_triaging_before_clustering=6.0,
|
|
984
|
+
amp_no_triaging_after_clustering=8.0,
|
|
985
|
+
use_y_triaging=False,
|
|
986
|
+
):
|
|
987
|
+
z = xyza[:, 2]
|
|
988
|
+
if motion_est is not None:
|
|
989
|
+
z = motion_est.correct_s(sorting.times_seconds, z)
|
|
990
|
+
z_not_reg = xyza[:, 2]
|
|
991
|
+
ampfeat = scales[2] * np.log(amp_log_c + amps[:])
|
|
992
|
+
res = _density_peaks_clustering_uhd_implementation(
|
|
993
|
+
np.c_[scales[0] * xyza[:, 0], scales[1] * z, ampfeat],
|
|
994
|
+
geom=geom,
|
|
995
|
+
y=xyza[:, 1],
|
|
996
|
+
z_not_reg=z_not_reg,
|
|
997
|
+
use_y_triaging=use_y_triaging,
|
|
998
|
+
sigma_local=sigma_local,
|
|
999
|
+
sigma_local_low=sigma_local_low,
|
|
1000
|
+
sigma_regional=sigma_regional,
|
|
1001
|
+
sigma_regional_low=sigma_regional_low,
|
|
1002
|
+
n_neighbors_search=n_neighbors_search,
|
|
1003
|
+
radius_search=radius_search,
|
|
1004
|
+
remove_clusters_smaller_than=remove_clusters_smaller_than,
|
|
1005
|
+
noise_density=noise_density,
|
|
1006
|
+
triage_quantile_per_cluster=triage_quantile_per_cluster,
|
|
1007
|
+
ramp_triage_per_cluster=ramp_triage_per_cluster,
|
|
1008
|
+
triage_quantile_before_clustering=triage_quantile_before_clustering,
|
|
1009
|
+
amp_no_triaging_before_clustering=amp_no_triaging_before_clustering,
|
|
1010
|
+
amp_no_triaging_after_clustering=amp_no_triaging_after_clustering,
|
|
1011
|
+
distance_dependent_noise_density=distance_dependent_noise_density,
|
|
1012
|
+
outlier_radius=outlier_radius,
|
|
1013
|
+
outlier_neighbor_count=outlier_neighbor_count,
|
|
1014
|
+
scales=scales,
|
|
1015
|
+
log_c=amp_log_c,
|
|
1016
|
+
workers=workers,
|
|
1017
|
+
return_extra=attach_density_feature,
|
|
1018
|
+
)
|
|
1019
|
+
return res
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def _density_peaks_clustering_uhd_implementation(
|
|
1023
|
+
X,
|
|
1024
|
+
geom=None,
|
|
1025
|
+
y=None,
|
|
1026
|
+
z_not_reg=None,
|
|
1027
|
+
use_y_triaging=False,
|
|
1028
|
+
l2_norm=None,
|
|
1029
|
+
kdtree=None,
|
|
1030
|
+
sigma_local=5.0,
|
|
1031
|
+
sigma_local_low=None,
|
|
1032
|
+
sigma_regional=None,
|
|
1033
|
+
sigma_regional_low=None,
|
|
1034
|
+
min_bin_size=1.0,
|
|
1035
|
+
outlier_neighbor_count=10,
|
|
1036
|
+
outlier_radius=25.0,
|
|
1037
|
+
n_neighbors_search=20,
|
|
1038
|
+
radius_search=5.0,
|
|
1039
|
+
noise_density=0.0,
|
|
1040
|
+
remove_clusters_smaller_than=10,
|
|
1041
|
+
remove_borders=False,
|
|
1042
|
+
border_search_radius=10.0,
|
|
1043
|
+
border_search_neighbors=3,
|
|
1044
|
+
workers=1,
|
|
1045
|
+
return_extra=False,
|
|
1046
|
+
triage_quantile_before_clustering=0.0,
|
|
1047
|
+
amp_no_triaging_before_clustering=6.0,
|
|
1048
|
+
ramp_triage_before_clustering=False,
|
|
1049
|
+
radius_triage_before_clustering=75.0,
|
|
1050
|
+
triage_quantile_per_cluster=0.0,
|
|
1051
|
+
amp_no_triaging_after_clustering=12.0,
|
|
1052
|
+
ramp_triage_per_cluster=False,
|
|
1053
|
+
distance_dependent_noise_density=False,
|
|
1054
|
+
amp_lowest_noise_density=8.0,
|
|
1055
|
+
min_distance_noise_density=0.0,
|
|
1056
|
+
min_distance_noise_density_10=200,
|
|
1057
|
+
max_noise_density=10.0,
|
|
1058
|
+
max_n_bins=128,
|
|
1059
|
+
scales=None,
|
|
1060
|
+
log_c=None,
|
|
1061
|
+
):
|
|
1062
|
+
"""
|
|
1063
|
+
if l2_norm is passed as argument, it will be used to compute density and nhdn
|
|
1064
|
+
"""
|
|
1065
|
+
# n = len(X)
|
|
1066
|
+
|
|
1067
|
+
if ramp_triage_before_clustering and geom is not None:
|
|
1068
|
+
assert scales is not None
|
|
1069
|
+
assert log_c is not None
|
|
1070
|
+
inliers_first = np.ones(len(X)).astype("bool")
|
|
1071
|
+
idx_low_ptp = np.flatnonzero(
|
|
1072
|
+
X[:, 2] < scales[2] * np.log(log_c + amp_no_triaging_before_clustering)
|
|
1073
|
+
)
|
|
1074
|
+
distances_to_geom = (
|
|
1075
|
+
np.sqrt((X[idx_low_ptp, :2][None] - geom[:, None]) ** 2).sum(2)
|
|
1076
|
+
).min(0)
|
|
1077
|
+
probabilities = (
|
|
1078
|
+
np.minimum(distances_to_geom, radius_triage_before_clustering)
|
|
1079
|
+
/ radius_triage_before_clustering
|
|
1080
|
+
)
|
|
1081
|
+
which_to_discard = bernoulli.rvs(probabilities).astype("bool") # type: ignore
|
|
1082
|
+
inliers_first[idx_low_ptp[which_to_discard]] = False
|
|
1083
|
+
inliers_first = np.arange(len(X))[inliers_first]
|
|
1084
|
+
else:
|
|
1085
|
+
inliers_first = np.arange(len(X))
|
|
1086
|
+
|
|
1087
|
+
n = len(inliers_first)
|
|
1088
|
+
if n <= 1:
|
|
1089
|
+
return dict(labels=np.full(X.shape[0], -1))
|
|
1090
|
+
|
|
1091
|
+
if use_y_triaging and y is not None:
|
|
1092
|
+
inliers, _ = kdtree_inliers(
|
|
1093
|
+
np.c_[X[inliers_first], y[inliers_first]],
|
|
1094
|
+
kdtree=kdtree,
|
|
1095
|
+
n_neighbors=outlier_neighbor_count,
|
|
1096
|
+
distance_upper_bound=outlier_radius * np.sqrt(4 / 3),
|
|
1097
|
+
workers=workers,
|
|
1098
|
+
)
|
|
1099
|
+
_, kdtree = kdtree_inliers(
|
|
1100
|
+
X[inliers_first],
|
|
1101
|
+
kdtree=kdtree,
|
|
1102
|
+
n_neighbors=outlier_neighbor_count,
|
|
1103
|
+
distance_upper_bound=outlier_radius,
|
|
1104
|
+
workers=workers,
|
|
1105
|
+
)
|
|
1106
|
+
else:
|
|
1107
|
+
inliers, kdtree = kdtree_inliers(
|
|
1108
|
+
X[inliers_first],
|
|
1109
|
+
kdtree=kdtree,
|
|
1110
|
+
n_neighbors=outlier_neighbor_count,
|
|
1111
|
+
distance_upper_bound=outlier_radius,
|
|
1112
|
+
workers=workers,
|
|
1113
|
+
)
|
|
1114
|
+
|
|
1115
|
+
if not inliers.sum() > 1:
|
|
1116
|
+
return dict(labels=np.full(X.shape[0], -1))
|
|
1117
|
+
|
|
1118
|
+
# inliers = inliers_first[inliers]
|
|
1119
|
+
|
|
1120
|
+
if isinstance(sigma_local, str) and sigma_local.startswith("rule_of_thumb"):
|
|
1121
|
+
factor = 1
|
|
1122
|
+
if "*" in sigma_local:
|
|
1123
|
+
factor = float(sigma_local.split("*")[1].strip())
|
|
1124
|
+
sigma_local = (
|
|
1125
|
+
1.06
|
|
1126
|
+
* factor
|
|
1127
|
+
* np.linalg.norm(np.std(X[inliers_first][inliers], axis=0))
|
|
1128
|
+
* np.power(inliers.sum(), -0.2)
|
|
1129
|
+
)
|
|
1130
|
+
if sigma_local <= 0:
|
|
1131
|
+
raise ValueError(
|
|
1132
|
+
f"rule of thumb problems. {sigma_local=} "
|
|
1133
|
+
f"{np.std(X[inliers_first][inliers], axis=0)=} "
|
|
1134
|
+
f"{len(X)=} {np.power(len(X), -0.2)=} "
|
|
1135
|
+
f"{X.shape=} {inliers_first.shape=} {inliers.sum()=} "
|
|
1136
|
+
f"{X[inliers_first][inliers].shape=}"
|
|
1137
|
+
)
|
|
1138
|
+
if sigma_regional == "rule_of_thumb":
|
|
1139
|
+
radius_search = radius_search * sigma_local
|
|
1140
|
+
sigma_regional = 10 * sigma_local
|
|
1141
|
+
else:
|
|
1142
|
+
radius_search = radius_search * sigma_local
|
|
1143
|
+
|
|
1144
|
+
if l2_norm is None:
|
|
1145
|
+
do_ratio = sigma_regional is not None
|
|
1146
|
+
density = get_smoothed_density(
|
|
1147
|
+
X[inliers_first],
|
|
1148
|
+
inliers=inliers,
|
|
1149
|
+
sigma=sigma_local,
|
|
1150
|
+
sigma_low=sigma_local_low,
|
|
1151
|
+
min_bin_size=min_bin_size,
|
|
1152
|
+
max_n_bins=max_n_bins,
|
|
1153
|
+
)
|
|
1154
|
+
if density is None:
|
|
1155
|
+
return dict(labels=np.full(X.shape[0], -1))
|
|
1156
|
+
if do_ratio:
|
|
1157
|
+
reg_density = get_smoothed_density(
|
|
1158
|
+
X[inliers_first],
|
|
1159
|
+
inliers=inliers,
|
|
1160
|
+
sigma=sigma_regional,
|
|
1161
|
+
sigma_low=sigma_regional_low,
|
|
1162
|
+
min_bin_size=min_bin_size,
|
|
1163
|
+
)
|
|
1164
|
+
density = density / reg_density
|
|
1165
|
+
density = np.nan_to_num(density)
|
|
1166
|
+
|
|
1167
|
+
nhdn = nearest_higher_density_neighbor(
|
|
1168
|
+
kdtree,
|
|
1169
|
+
density,
|
|
1170
|
+
n_neighbors_search=n_neighbors_search,
|
|
1171
|
+
distance_upper_bound=radius_search,
|
|
1172
|
+
workers=workers,
|
|
1173
|
+
)
|
|
1174
|
+
else:
|
|
1175
|
+
# inliers don't matter here? Or should we still remove them?...
|
|
1176
|
+
# indices = np.full(l2_norm.shape[0], n)
|
|
1177
|
+
# l2_norm_inliers = l2_norm[inliers] #?? NO -> keep everything but only compute for inliers
|
|
1178
|
+
l2_norm = l2_norm[inliers_first][:, inliers_first]
|
|
1179
|
+
n = l2_norm.shape[0]
|
|
1180
|
+
indices = l2_norm.argsort()[:, : 1 + n_neighbors_search]
|
|
1181
|
+
distances = l2_norm[np.arange(n)[:, None], indices]
|
|
1182
|
+
assert distances.shape == (l2_norm.shape[0], 1 + n_neighbors_search)
|
|
1183
|
+
density = np.median(distances, axis=1)
|
|
1184
|
+
density_padded = np.pad(density, (0, 1), constant_values=np.inf)
|
|
1185
|
+
is_higher_density = density_padded[indices] >= density[:, None]
|
|
1186
|
+
distances[is_higher_density] = np.inf
|
|
1187
|
+
indices[is_higher_density] = n
|
|
1188
|
+
nhdn = indices[np.arange(n), distances.argmin(1)]
|
|
1189
|
+
del distances, indices
|
|
1190
|
+
|
|
1191
|
+
if noise_density and l2_norm is None:
|
|
1192
|
+
nhdn[density <= noise_density] = n
|
|
1193
|
+
|
|
1194
|
+
if distance_dependent_noise_density and noise_density is not None:
|
|
1195
|
+
assert scales is not None
|
|
1196
|
+
assert log_c is not None
|
|
1197
|
+
assert geom is not None
|
|
1198
|
+
dist = np.sqrt(
|
|
1199
|
+
(
|
|
1200
|
+
(
|
|
1201
|
+
np.c_[X[inliers_first, 0], z_not_reg[inliers_first]][:, None] # type: ignore
|
|
1202
|
+
- geom[None]
|
|
1203
|
+
)
|
|
1204
|
+
** 2
|
|
1205
|
+
).sum(2)
|
|
1206
|
+
).min(1)
|
|
1207
|
+
noise_density_dist = (
|
|
1208
|
+
np.minimum(
|
|
1209
|
+
np.maximum(dist - min_distance_noise_density, 0),
|
|
1210
|
+
min_distance_noise_density_10 - min_distance_noise_density,
|
|
1211
|
+
)
|
|
1212
|
+
* (max_noise_density - noise_density)
|
|
1213
|
+
/ (min_distance_noise_density_10 - min_distance_noise_density)
|
|
1214
|
+
+ noise_density
|
|
1215
|
+
)
|
|
1216
|
+
noise_density_dist[
|
|
1217
|
+
X[inliers_first, 2] > scales[2] * np.log(log_c + amp_lowest_noise_density)
|
|
1218
|
+
] = 2
|
|
1219
|
+
nhdn[density <= noise_density_dist] = n
|
|
1220
|
+
if triage_quantile_before_clustering and l2_norm is None:
|
|
1221
|
+
assert scales is not None
|
|
1222
|
+
assert log_c is not None
|
|
1223
|
+
q = np.quantile(density, triage_quantile_before_clustering)
|
|
1224
|
+
idx_triaging = np.flatnonzero(
|
|
1225
|
+
np.logical_and(
|
|
1226
|
+
X[inliers_first, 2]
|
|
1227
|
+
< scales[2] * np.log(log_c + amp_no_triaging_before_clustering),
|
|
1228
|
+
density < q,
|
|
1229
|
+
)
|
|
1230
|
+
)
|
|
1231
|
+
nhdn[idx_triaging] = n
|
|
1232
|
+
|
|
1233
|
+
nhdn = nhdn.astype(np.intc)
|
|
1234
|
+
has_nhdn = np.flatnonzero(nhdn < n).astype(np.intc)
|
|
1235
|
+
rc = (
|
|
1236
|
+
np.concatenate([nhdn[has_nhdn], has_nhdn]),
|
|
1237
|
+
np.concatenate([has_nhdn, nhdn[has_nhdn]]),
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
graph = coo_array((np.ones(2 * has_nhdn.size), rc), shape=(n, n))
|
|
1241
|
+
ncc, labels = connected_components(graph, directed=False)
|
|
1242
|
+
|
|
1243
|
+
if remove_borders:
|
|
1244
|
+
labels = remove_border_points(
|
|
1245
|
+
labels,
|
|
1246
|
+
density,
|
|
1247
|
+
kdtree,
|
|
1248
|
+
search_radius=border_search_radius,
|
|
1249
|
+
n_neighbors_search=border_search_neighbors,
|
|
1250
|
+
workers=workers,
|
|
1251
|
+
)
|
|
1252
|
+
|
|
1253
|
+
if remove_clusters_smaller_than:
|
|
1254
|
+
labels = decrumb(labels, min_size=remove_clusters_smaller_than)
|
|
1255
|
+
|
|
1256
|
+
if triage_quantile_per_cluster:
|
|
1257
|
+
assert scales is not None
|
|
1258
|
+
assert log_c is not None
|
|
1259
|
+
amp_no_triaging_after_clustering = scales[2] * np.log(
|
|
1260
|
+
log_c + amp_no_triaging_after_clustering
|
|
1261
|
+
)
|
|
1262
|
+
for k in np.unique(labels[labels > -1]):
|
|
1263
|
+
idx_label = np.flatnonzero(labels == k)
|
|
1264
|
+
amp_vec = X[inliers_first][idx_label, 2]
|
|
1265
|
+
med_amp = np.median(amp_vec)
|
|
1266
|
+
if med_amp < amp_no_triaging_after_clustering:
|
|
1267
|
+
if ramp_triage_per_cluster:
|
|
1268
|
+
triage_quantile_per_cluster = (
|
|
1269
|
+
amp_no_triaging_after_clustering - med_amp
|
|
1270
|
+
) / amp_no_triaging_after_clustering
|
|
1271
|
+
# triage_quantile_unit = triage_quantile_per_cluster
|
|
1272
|
+
if l2_norm is None:
|
|
1273
|
+
q = np.quantile(density[idx_label], triage_quantile_per_cluster)
|
|
1274
|
+
spikes_to_remove = np.flatnonzero(
|
|
1275
|
+
np.logical_and(
|
|
1276
|
+
density[idx_label] < q,
|
|
1277
|
+
amp_vec < amp_no_triaging_after_clustering,
|
|
1278
|
+
)
|
|
1279
|
+
)
|
|
1280
|
+
else:
|
|
1281
|
+
q = np.quantile(density[idx_label], 1 - triage_quantile_per_cluster)
|
|
1282
|
+
spikes_to_remove = np.flatnonzero(
|
|
1283
|
+
np.logical_and(
|
|
1284
|
+
density[idx_label] > q,
|
|
1285
|
+
amp_vec < amp_no_triaging_after_clustering,
|
|
1286
|
+
)
|
|
1287
|
+
)
|
|
1288
|
+
|
|
1289
|
+
labels[idx_label[spikes_to_remove]] = -1
|
|
1290
|
+
|
|
1291
|
+
labels_all = np.full(len(X), -1)
|
|
1292
|
+
labels_all[inliers_first] = labels
|
|
1293
|
+
|
|
1294
|
+
density_all = np.zeros(len(X))
|
|
1295
|
+
nhdn_all = np.full(len(X), n)
|
|
1296
|
+
density_all[inliers_first] = density
|
|
1297
|
+
nhdn_all[inliers_first] = nhdn
|
|
1298
|
+
|
|
1299
|
+
return dict(density=density_all, nhdn=nhdn_all, labels=labels_all, kdtree=kdtree)
|