DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1285 @@
|
|
|
1
|
+
from dataclasses import dataclass, replace
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import h5py
|
|
6
|
+
import numpy as np
|
|
7
|
+
from dartsort.util import drift_util, waveform_util
|
|
8
|
+
from dartsort.util.data_util import (
|
|
9
|
+
DARTsortSorting,
|
|
10
|
+
batched_h5_read,
|
|
11
|
+
get_featurization_pipeline,
|
|
12
|
+
)
|
|
13
|
+
from dartsort.util.multiprocessing_util import get_pool
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
from hdbscan import HDBSCAN
|
|
17
|
+
except ImportError:
|
|
18
|
+
from sklearn.cluster import HDBSCAN
|
|
19
|
+
from scipy.spatial.distance import cdist, pdist
|
|
20
|
+
from sklearn.decomposition import PCA
|
|
21
|
+
|
|
22
|
+
# from sklearn.metrics.pairwise import nan_euclidean_distances
|
|
23
|
+
from tqdm.auto import tqdm
|
|
24
|
+
|
|
25
|
+
from . import cluster_util, density, relocate
|
|
26
|
+
from .forward_backward import forward_backward
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def split_clusters(
|
|
30
|
+
sorting,
|
|
31
|
+
split_strategy="FeatureSplit",
|
|
32
|
+
split_strategy_kwargs=None,
|
|
33
|
+
recursive=False,
|
|
34
|
+
split_big=False,
|
|
35
|
+
split_big_kw=dict(dz=40, dx=48, min_size_split=50),
|
|
36
|
+
motion_est=None,
|
|
37
|
+
show_progress=True,
|
|
38
|
+
depth_order=True,
|
|
39
|
+
n_jobs=0,
|
|
40
|
+
):
|
|
41
|
+
"""Parallel split step runner function
|
|
42
|
+
|
|
43
|
+
Arguments
|
|
44
|
+
---------
|
|
45
|
+
sorting : DARTsortSorting
|
|
46
|
+
split_step_class_name : str
|
|
47
|
+
One of split_steps_by_class_name.keys()
|
|
48
|
+
split_step_kwargs : dictionary
|
|
49
|
+
recursive : bool
|
|
50
|
+
If True, attempt to split newly created clusters again
|
|
51
|
+
show_progress : bool
|
|
52
|
+
n_jobs : int
|
|
53
|
+
|
|
54
|
+
Returns
|
|
55
|
+
-------
|
|
56
|
+
split_sorting : DARTsortSorting
|
|
57
|
+
"""
|
|
58
|
+
# initialize split state
|
|
59
|
+
labels = sorting.labels.copy()
|
|
60
|
+
labels_to_process = np.unique(labels)
|
|
61
|
+
labels_to_process = list(labels_to_process[labels_to_process > 0])
|
|
62
|
+
cur_max_label = max(labels_to_process)
|
|
63
|
+
|
|
64
|
+
if split_strategy_kwargs is None:
|
|
65
|
+
split_strategy_kwargs = {}
|
|
66
|
+
if motion_est is not None:
|
|
67
|
+
split_strategy_kwargs["motion_est"] = motion_est
|
|
68
|
+
|
|
69
|
+
n_jobs, Executor, context = get_pool(n_jobs)
|
|
70
|
+
with Executor(
|
|
71
|
+
max_workers=n_jobs,
|
|
72
|
+
mp_context=context,
|
|
73
|
+
initializer=_split_job_init,
|
|
74
|
+
initargs=(
|
|
75
|
+
split_strategy,
|
|
76
|
+
sorting.parent_h5_path,
|
|
77
|
+
split_strategy_kwargs,
|
|
78
|
+
),
|
|
79
|
+
) as pool:
|
|
80
|
+
iterator = jobs = [
|
|
81
|
+
pool.submit(_split_job, np.flatnonzero(labels == i))
|
|
82
|
+
for i in labels_to_process
|
|
83
|
+
]
|
|
84
|
+
if show_progress:
|
|
85
|
+
iterator = tqdm(
|
|
86
|
+
jobs,
|
|
87
|
+
desc=split_strategy,
|
|
88
|
+
total=len(labels_to_process),
|
|
89
|
+
smoothing=0,
|
|
90
|
+
)
|
|
91
|
+
for future in iterator:
|
|
92
|
+
split_result = future.result()
|
|
93
|
+
if not split_result.is_split:
|
|
94
|
+
continue
|
|
95
|
+
|
|
96
|
+
# assign new labels. -1 -> -1, 0 keeps current label,
|
|
97
|
+
# others start at cur_max_label + 1
|
|
98
|
+
in_unit = split_result.in_unit
|
|
99
|
+
new_labels = split_result.new_labels
|
|
100
|
+
triaged = split_result.new_labels < 0
|
|
101
|
+
labels[in_unit[triaged]] = new_labels[triaged]
|
|
102
|
+
labels[in_unit[new_labels > 0]] = cur_max_label + new_labels[new_labels > 0]
|
|
103
|
+
new_untriaged_labels = labels[in_unit[new_labels >= 0]]
|
|
104
|
+
cur_max_label = new_untriaged_labels.max()
|
|
105
|
+
|
|
106
|
+
# submit recursive jobs to the pool, if any
|
|
107
|
+
if recursive:
|
|
108
|
+
new_units = np.unique(new_untriaged_labels)
|
|
109
|
+
for i in new_units:
|
|
110
|
+
jobs.append(pool.submit(_split_job, np.flatnonzero(labels == i)))
|
|
111
|
+
if show_progress:
|
|
112
|
+
iterator.total += len(new_units) # type: ignore
|
|
113
|
+
elif split_big:
|
|
114
|
+
new_units = np.unique(new_untriaged_labels)
|
|
115
|
+
for i in new_units:
|
|
116
|
+
idx = np.flatnonzero(new_untriaged_labels == i)
|
|
117
|
+
tall = np.ptp(split_result.x[idx]) > split_big_kw["dx"]
|
|
118
|
+
wide = np.ptp(split_result.z_reg[idx]) > split_big_kw["dx"]
|
|
119
|
+
if (tall or wide) and len(idx) > split_big_kw["min_size_split"]:
|
|
120
|
+
jobs.append(
|
|
121
|
+
pool.submit(_split_job, np.flatnonzero(labels == i))
|
|
122
|
+
)
|
|
123
|
+
if show_progress:
|
|
124
|
+
iterator.total += 1 # type: ignore
|
|
125
|
+
|
|
126
|
+
new_sorting = sorting.ephemeral_replace(labels=labels)
|
|
127
|
+
if depth_order:
|
|
128
|
+
new_sorting = cluster_util.reorder_by_depth(new_sorting, motion_est=motion_est)
|
|
129
|
+
return new_sorting
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# -- split steps
|
|
133
|
+
|
|
134
|
+
# the purpose of using classes here is that the class manages expensive
|
|
135
|
+
# resources that we don't want to set up every time we split a cluster.
|
|
136
|
+
# this way, these objects are created once on each worker process.
|
|
137
|
+
# we only have one split step so it is a bit silly to have the superclass,
|
|
138
|
+
# but it's just to illustrate the interface that's actually used by the
|
|
139
|
+
# main function split_clusters, in case someone wants to write another
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass
|
|
143
|
+
class SplitResult:
|
|
144
|
+
"""If not is_split, it is fine to leave other fields as None"""
|
|
145
|
+
|
|
146
|
+
is_split: bool = False
|
|
147
|
+
in_unit: Optional[np.ndarray] = None
|
|
148
|
+
new_labels: Optional[np.ndarray] = None
|
|
149
|
+
x: Optional[np.ndarray] = None
|
|
150
|
+
z_reg: Optional[np.ndarray] = None
|
|
151
|
+
|
|
152
|
+
def __post_init__(self):
|
|
153
|
+
"""Runs after a SplitResult is constructed, check valid"""
|
|
154
|
+
if self.is_split:
|
|
155
|
+
assert self.in_unit is not None
|
|
156
|
+
assert self.new_labels is not None
|
|
157
|
+
assert self.in_unit.ndim == 1
|
|
158
|
+
assert self.in_unit.shape == self.new_labels.shape
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class SplitStrategy:
|
|
162
|
+
"""Split steps subclass this and implement split_cluster"""
|
|
163
|
+
|
|
164
|
+
def split_cluster(self, in_unit):
|
|
165
|
+
"""
|
|
166
|
+
Arguments
|
|
167
|
+
---------
|
|
168
|
+
in_unit : indices of spikes in a putative unit
|
|
169
|
+
|
|
170
|
+
Returns
|
|
171
|
+
-------
|
|
172
|
+
split_result : a SplitResult object
|
|
173
|
+
"""
|
|
174
|
+
raise NotImplementedError
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class FeatureSplit(SplitStrategy):
|
|
178
|
+
def __init__(
|
|
179
|
+
self,
|
|
180
|
+
peeling_hdf5_filename,
|
|
181
|
+
recording=None,
|
|
182
|
+
chunk_size_s=300,
|
|
183
|
+
peeling_featurization_pt=None,
|
|
184
|
+
motion_est=None,
|
|
185
|
+
use_localization_features=True,
|
|
186
|
+
return_localization_features=False,
|
|
187
|
+
n_pca_features=2,
|
|
188
|
+
pca_imputation=None,
|
|
189
|
+
pca_reassign=False,
|
|
190
|
+
relocated=True,
|
|
191
|
+
use_wfs_L2_norm=False,
|
|
192
|
+
localization_feature_scales=(1.0, 1.0, 50.0),
|
|
193
|
+
time_scale=3e-3,
|
|
194
|
+
use_time_feature=False,
|
|
195
|
+
log_c=5,
|
|
196
|
+
channel_selection_radius=75.0,
|
|
197
|
+
min_cluster_size=25,
|
|
198
|
+
min_samples=25,
|
|
199
|
+
cluster_selection_epsilon=1,
|
|
200
|
+
sigma_local=5,
|
|
201
|
+
sigma_local_low=None,
|
|
202
|
+
sigma_regional=None,
|
|
203
|
+
noise_density=0.1,
|
|
204
|
+
n_neighbors_search=20,
|
|
205
|
+
radius_search=5.0,
|
|
206
|
+
triage_quantile_per_cluster=0.2,
|
|
207
|
+
remove_clusters_smaller_than=25,
|
|
208
|
+
knn_reassign_outliers=False,
|
|
209
|
+
random_state=0,
|
|
210
|
+
rescale_all_features=False,
|
|
211
|
+
use_ptp=True,
|
|
212
|
+
amplitude_normalized=False,
|
|
213
|
+
use_spread=False,
|
|
214
|
+
max_spikes=None,
|
|
215
|
+
cluster_alg="hdbscan",
|
|
216
|
+
**dataset_name_kwargs,
|
|
217
|
+
):
|
|
218
|
+
"""Split clusters based on per-cluster PCA and localization features
|
|
219
|
+
|
|
220
|
+
Fits PCA to relocated waveforms within a cluster, combines these with
|
|
221
|
+
localization features (x, registered z, and max amplitude), and runs
|
|
222
|
+
HDBSCAN to refine the labeling.
|
|
223
|
+
|
|
224
|
+
Arguments
|
|
225
|
+
---------
|
|
226
|
+
peeling_hdf5_filename : str or Path
|
|
227
|
+
Path to the HDF5 file output by a peeling step
|
|
228
|
+
peeling_featurization_pt : str or Path
|
|
229
|
+
Path to the .pt file where a featurization WaveformPipeline was
|
|
230
|
+
saved during peeling. Required if relocated=True.
|
|
231
|
+
motion_est : dredge.motion_util.MotionEstimate, optional
|
|
232
|
+
If supplied, registered depth positions will be used for clustering
|
|
233
|
+
and relocation
|
|
234
|
+
use_localization_features : bool
|
|
235
|
+
If false, only PCA features are used for clustering
|
|
236
|
+
n_pca_features : int
|
|
237
|
+
If 0, no PCA is used
|
|
238
|
+
relocated : bool
|
|
239
|
+
Whether to relocate waveforms according to the motion estimate.
|
|
240
|
+
Only used if motion_est is not None
|
|
241
|
+
localization_feature_scales : 3-tuple of float
|
|
242
|
+
Scales to apply to x, registered z, and log-scaled amplitude during
|
|
243
|
+
clustering to approximately sphere the clusters
|
|
244
|
+
channel_selection_radius : float
|
|
245
|
+
Spatial radius around the main channel used when selecting channels
|
|
246
|
+
used when extracting local PCA features
|
|
247
|
+
min_cluster_size, min_samples, cluster_selection_epsilon
|
|
248
|
+
HDBSCAN parameters. See their documentation for lots of info.
|
|
249
|
+
"""
|
|
250
|
+
# method parameters
|
|
251
|
+
self.channel_selection_radius = channel_selection_radius
|
|
252
|
+
self.use_localization_features = use_localization_features
|
|
253
|
+
self.n_pca_features = n_pca_features
|
|
254
|
+
self.pca_imputation = pca_imputation
|
|
255
|
+
self.pca_reassign = pca_reassign
|
|
256
|
+
self.relocated = relocated and motion_est is not None
|
|
257
|
+
self.motion_est = motion_est
|
|
258
|
+
self.localization_feature_scales = localization_feature_scales
|
|
259
|
+
self.log_c = log_c
|
|
260
|
+
self.rg = np.random.default_rng(random_state)
|
|
261
|
+
self.knn_reassign_outliers = knn_reassign_outliers
|
|
262
|
+
self.max_spikes = max_spikes
|
|
263
|
+
self.use_wfs_L2_norm = use_wfs_L2_norm
|
|
264
|
+
self.rescale_all_features = rescale_all_features
|
|
265
|
+
self.use_ptp = use_ptp
|
|
266
|
+
self.amplitude_normalized = amplitude_normalized
|
|
267
|
+
self.use_spread = use_spread
|
|
268
|
+
self.use_time_feature = use_time_feature
|
|
269
|
+
self.time_scale = time_scale
|
|
270
|
+
self.return_localization_features = return_localization_features
|
|
271
|
+
|
|
272
|
+
# hdbscan parameters
|
|
273
|
+
self.min_cluster_size = min_cluster_size
|
|
274
|
+
self.min_samples = min_samples
|
|
275
|
+
self.cluster_selection_epsilon = cluster_selection_epsilon
|
|
276
|
+
|
|
277
|
+
# DPC parameters
|
|
278
|
+
self.sigma_local = sigma_local
|
|
279
|
+
self.sigma_local_low = sigma_local_low
|
|
280
|
+
self.sigma_regional = sigma_regional
|
|
281
|
+
self.noise_density = noise_density
|
|
282
|
+
self.n_neighbors_search = n_neighbors_search
|
|
283
|
+
self.radius_search = radius_search
|
|
284
|
+
self.triage_quantile_per_cluster = triage_quantile_per_cluster
|
|
285
|
+
self.remove_clusters_smaller_than = remove_clusters_smaller_than
|
|
286
|
+
|
|
287
|
+
assert cluster_alg in ("hdbscan", "dpc")
|
|
288
|
+
self.cluster_alg = cluster_alg
|
|
289
|
+
|
|
290
|
+
# load up the required h5 datasets
|
|
291
|
+
self.initialize_from_h5(
|
|
292
|
+
peeling_hdf5_filename,
|
|
293
|
+
peeling_featurization_pt,
|
|
294
|
+
**dataset_name_kwargs,
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
def split_cluster(self, in_unit_all):
|
|
298
|
+
n_spikes = in_unit_all.size
|
|
299
|
+
subsampling = self.max_spikes and n_spikes > self.max_spikes
|
|
300
|
+
if subsampling:
|
|
301
|
+
# TODO: max_spikes could be chosen automatically
|
|
302
|
+
# based on available memory and number of spikes
|
|
303
|
+
idx_subsample = self.rg.choice(n_spikes, self.max_spikes, replace=False)
|
|
304
|
+
idx_subsample.sort()
|
|
305
|
+
in_unit = in_unit_all[idx_subsample]
|
|
306
|
+
else:
|
|
307
|
+
in_unit = in_unit_all
|
|
308
|
+
|
|
309
|
+
if n_spikes < self.min_cluster_size:
|
|
310
|
+
return SplitResult()
|
|
311
|
+
|
|
312
|
+
do_pca = self.n_pca_features > 0
|
|
313
|
+
|
|
314
|
+
if do_pca or self.relocated or self.use_wfs_L2_norm or self.pca_reassign:
|
|
315
|
+
(
|
|
316
|
+
max_registered_channel,
|
|
317
|
+
n_pitches_shift,
|
|
318
|
+
reloc_amplitudes,
|
|
319
|
+
kept,
|
|
320
|
+
) = self.get_registered_channels(in_unit)
|
|
321
|
+
if not kept.size:
|
|
322
|
+
return SplitResult()
|
|
323
|
+
else:
|
|
324
|
+
kept = np.arange(in_unit.shape[0])
|
|
325
|
+
|
|
326
|
+
features = []
|
|
327
|
+
|
|
328
|
+
if self.use_localization_features:
|
|
329
|
+
loc_features = self.localization_features[in_unit]
|
|
330
|
+
if self.relocated:
|
|
331
|
+
loc_features[kept, 2] = self.localization_feature_scales[2] * np.log(
|
|
332
|
+
self.log_c + reloc_amplitudes
|
|
333
|
+
)
|
|
334
|
+
if self.rescale_all_features:
|
|
335
|
+
mad0 = mad_sigma(loc_features[kept, 0])
|
|
336
|
+
loc_features[kept] *= mad0 / mad_sigma(loc_features[kept])
|
|
337
|
+
if not self.use_ptp:
|
|
338
|
+
loc_features = loc_features[:, :2]
|
|
339
|
+
features.append(loc_features)
|
|
340
|
+
|
|
341
|
+
if self.use_time_feature:
|
|
342
|
+
features.append(self.t_s[in_unit] * self.time_scale)
|
|
343
|
+
|
|
344
|
+
if do_pca:
|
|
345
|
+
enough_good_spikes, pca_kept, pca_embeds = self.pca_features(
|
|
346
|
+
in_unit[kept],
|
|
347
|
+
max_registered_channel,
|
|
348
|
+
n_pitches_shift,
|
|
349
|
+
amplitude_normalized=self.amplitude_normalized,
|
|
350
|
+
)
|
|
351
|
+
do_pca = enough_good_spikes
|
|
352
|
+
|
|
353
|
+
if do_pca:
|
|
354
|
+
pca_embeds_ = np.full(
|
|
355
|
+
(in_unit.size, pca_embeds.shape[1]),
|
|
356
|
+
np.nan,
|
|
357
|
+
dtype=pca_embeds.dtype,
|
|
358
|
+
)
|
|
359
|
+
pca_embeds_[kept[pca_kept]] = pca_embeds
|
|
360
|
+
pca_embeds = pca_embeds_
|
|
361
|
+
kept = kept[pca_kept]
|
|
362
|
+
# scale pc features to match localization features
|
|
363
|
+
if self.rescale_all_features:
|
|
364
|
+
mad0 = mad_sigma(loc_features[kept, 0])
|
|
365
|
+
for k in range(self.n_pca_features):
|
|
366
|
+
pca_embeds[:, k] *= mad0 / mad_sigma(pca_embeds[:, k])
|
|
367
|
+
elif self.use_localization_features:
|
|
368
|
+
pca_embeds *= mad_sigma(loc_features[kept]).mean()
|
|
369
|
+
features.append(pca_embeds)
|
|
370
|
+
|
|
371
|
+
l2_norm = None
|
|
372
|
+
if self.use_wfs_L2_norm:
|
|
373
|
+
enough_good_spikes, wf2_norm_kept, l2_norm = self.waveforms_L2norm(
|
|
374
|
+
in_unit,
|
|
375
|
+
max_registered_channel,
|
|
376
|
+
n_pitches_shift,
|
|
377
|
+
)
|
|
378
|
+
kept = kept[wf2_norm_kept]
|
|
379
|
+
|
|
380
|
+
if self.use_spread:
|
|
381
|
+
spread = self.spread_feature(in_unit)
|
|
382
|
+
if self.rescale_all_features:
|
|
383
|
+
mad0 = mad_sigma(loc_features[kept, 0])
|
|
384
|
+
spread *= mad0 / mad_sigma(spread[kept])
|
|
385
|
+
elif self.use_localization_features:
|
|
386
|
+
spread *= mad_sigma(loc_features[kept]).mean()
|
|
387
|
+
features.append(spread)
|
|
388
|
+
|
|
389
|
+
if not features:
|
|
390
|
+
return SplitResult()
|
|
391
|
+
features = np.column_stack([f[kept] for f in features])
|
|
392
|
+
|
|
393
|
+
if self.cluster_alg == "hdbscan" and features.shape[0] > self.min_cluster_size:
|
|
394
|
+
clust = HDBSCAN(
|
|
395
|
+
min_cluster_size=self.min_cluster_size,
|
|
396
|
+
min_samples=self.min_samples,
|
|
397
|
+
cluster_selection_epsilon=self.cluster_selection_epsilon,
|
|
398
|
+
core_dist_n_jobs=1, # let's just use our parallelism
|
|
399
|
+
prediction_data=self.knn_reassign_outliers,
|
|
400
|
+
)
|
|
401
|
+
clust_labels = clust.fit_predict(features)
|
|
402
|
+
new_ids = np.unique(clust_labels)
|
|
403
|
+
new_ids = new_ids[new_ids >= 0]
|
|
404
|
+
is_split = new_ids.size > 1
|
|
405
|
+
elif (
|
|
406
|
+
self.cluster_alg == "dpc"
|
|
407
|
+
and features.shape[0] > self.remove_clusters_smaller_than
|
|
408
|
+
):
|
|
409
|
+
clust_labels = density.density_peaks_clustering(
|
|
410
|
+
features,
|
|
411
|
+
l2_norm=l2_norm,
|
|
412
|
+
sigma_local=self.sigma_local,
|
|
413
|
+
sigma_local_low=self.sigma_local_low,
|
|
414
|
+
sigma_regional=self.sigma_regional,
|
|
415
|
+
noise_density=self.noise_density,
|
|
416
|
+
n_neighbors_search=self.n_neighbors_search,
|
|
417
|
+
radius_search=self.radius_search,
|
|
418
|
+
triage_quantile_per_cluster=self.triage_quantile_per_cluster,
|
|
419
|
+
remove_clusters_smaller_than=self.remove_clusters_smaller_than,
|
|
420
|
+
)
|
|
421
|
+
new_ids = np.unique(clust_labels)
|
|
422
|
+
new_ids = new_ids[new_ids >= 0]
|
|
423
|
+
is_split = new_ids.size > 1
|
|
424
|
+
else:
|
|
425
|
+
is_split = False
|
|
426
|
+
|
|
427
|
+
if is_split and self.knn_reassign_outliers:
|
|
428
|
+
clust_labels = cluster_util.knn_reassign_outliers(clust_labels, features)
|
|
429
|
+
|
|
430
|
+
new_labels = None
|
|
431
|
+
if is_split:
|
|
432
|
+
new_labels = np.full(n_spikes, -1)
|
|
433
|
+
if subsampling:
|
|
434
|
+
new_labels[idx_subsample[kept]] = clust_labels
|
|
435
|
+
else:
|
|
436
|
+
new_labels[kept] = clust_labels
|
|
437
|
+
|
|
438
|
+
if is_split and self.pca_reassign:
|
|
439
|
+
if subsampling:
|
|
440
|
+
new_labels[idx_subsample] = self.tpca_reassignment(
|
|
441
|
+
new_labels[idx_subsample], in_unit, n_pitches_shift
|
|
442
|
+
)
|
|
443
|
+
else:
|
|
444
|
+
new_labels = self.tpca_reassignment(
|
|
445
|
+
new_labels, in_unit, n_pitches_shift
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
if self.return_localization_features and self.use_localization_features:
|
|
449
|
+
return SplitResult(
|
|
450
|
+
is_split=is_split,
|
|
451
|
+
in_unit=in_unit_all,
|
|
452
|
+
new_labels=new_labels,
|
|
453
|
+
x=loc_features[:, 0],
|
|
454
|
+
z_reg=loc_features[:, 1],
|
|
455
|
+
)
|
|
456
|
+
return SplitResult(
|
|
457
|
+
is_split=is_split, in_unit=in_unit_all, new_labels=new_labels
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
def get_registered_channels(self, in_unit, n_samples=1000):
|
|
461
|
+
n_pitches_shift = drift_util.get_spike_pitch_shifts(
|
|
462
|
+
self.z[in_unit],
|
|
463
|
+
geom=self.geom,
|
|
464
|
+
registered_depths_um=self.z_reg[in_unit],
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
amp_samples = slice(None)
|
|
468
|
+
if in_unit.size > n_samples:
|
|
469
|
+
amp_samples = self.rg.choice(in_unit.size, size=n_samples, replace=False)
|
|
470
|
+
amp_samples.sort()
|
|
471
|
+
amp_vecs = batched_h5_read(self.amplitude_vectors, in_unit)
|
|
472
|
+
amplitude_template = drift_util.registered_average(
|
|
473
|
+
amp_vecs[amp_samples],
|
|
474
|
+
n_pitches_shift[amp_samples],
|
|
475
|
+
self.geom,
|
|
476
|
+
self.registered_geom,
|
|
477
|
+
main_channels=self.channels[in_unit[amp_samples]],
|
|
478
|
+
channel_index=self.channel_index,
|
|
479
|
+
)
|
|
480
|
+
max_registered_channel = amplitude_template.argmax()
|
|
481
|
+
|
|
482
|
+
if self.relocated:
|
|
483
|
+
targ_chans = self.registered_channel_index[max_registered_channel]
|
|
484
|
+
targ_chans = targ_chans[targ_chans < len(self.registered_geom)]
|
|
485
|
+
reloc_amp_vecs = relocate.relocated_waveforms_on_static_channels(
|
|
486
|
+
amp_vecs,
|
|
487
|
+
main_channels=self.channels[in_unit],
|
|
488
|
+
channel_index=self.channel_index,
|
|
489
|
+
xyza_from=self.xyza[in_unit],
|
|
490
|
+
target_channels=targ_chans,
|
|
491
|
+
z_to=self.z_reg[in_unit],
|
|
492
|
+
geom=self.geom,
|
|
493
|
+
registered_geom=self.registered_geom,
|
|
494
|
+
match_distance=self.match_distance,
|
|
495
|
+
)
|
|
496
|
+
kept = np.flatnonzero(~np.isnan(reloc_amp_vecs).all(axis=1))
|
|
497
|
+
reloc_amplitudes = np.nanmax(reloc_amp_vecs[kept], axis=1)
|
|
498
|
+
else:
|
|
499
|
+
reloc_amplitudes = None
|
|
500
|
+
kept = np.arange(in_unit.size)
|
|
501
|
+
|
|
502
|
+
return max_registered_channel, n_pitches_shift, reloc_amplitudes, kept
|
|
503
|
+
|
|
504
|
+
def spread_feature(
|
|
505
|
+
self,
|
|
506
|
+
in_unit,
|
|
507
|
+
max_value_dist=70,
|
|
508
|
+
):
|
|
509
|
+
spread = np.zeros(in_unit.shape[0])
|
|
510
|
+
amp_vecs = batched_h5_read(self.amplitude_vectors, in_unit)
|
|
511
|
+
main_channels = self.channels[in_unit]
|
|
512
|
+
|
|
513
|
+
channel_distances_index = np.sqrt(
|
|
514
|
+
(
|
|
515
|
+
(
|
|
516
|
+
np.pad(
|
|
517
|
+
self.geom,
|
|
518
|
+
[[0, 1], [0, 0]],
|
|
519
|
+
mode="constant",
|
|
520
|
+
constant_values=np.nan,
|
|
521
|
+
)[self.channel_index]
|
|
522
|
+
- self.geom[:, None]
|
|
523
|
+
)
|
|
524
|
+
** 2
|
|
525
|
+
).sum(2)
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
for k in range(in_unit.shape[0]):
|
|
529
|
+
max_chan = main_channels[k]
|
|
530
|
+
channels_effective = np.flatnonzero(self.channel_index[max_chan] < 384)
|
|
531
|
+
channels_effective = channels_effective[
|
|
532
|
+
channel_distances_index[max_chan][channels_effective] < max_value_dist
|
|
533
|
+
]
|
|
534
|
+
spread[k] = np.nansum(
|
|
535
|
+
amp_vecs[k, channels_effective]
|
|
536
|
+
* channel_distances_index[max_chan][channels_effective]
|
|
537
|
+
) / np.nansum(amp_vecs[k, channels_effective])
|
|
538
|
+
|
|
539
|
+
return spread
|
|
540
|
+
|
|
541
|
+
def waveforms_L2norm(
|
|
542
|
+
self,
|
|
543
|
+
in_unit,
|
|
544
|
+
max_registered_channel,
|
|
545
|
+
n_pitches_shift,
|
|
546
|
+
batch_size=1_000,
|
|
547
|
+
):
|
|
548
|
+
"""Compute relocated waveforms on a drift-invariant channel set, for computing L2 distance"""
|
|
549
|
+
# figure out which set of channels to use
|
|
550
|
+
# we use the stored amplitudes to do this rather than computing a
|
|
551
|
+
# template, which can be expensive
|
|
552
|
+
# max_batch_size set to avoid memory errors
|
|
553
|
+
pca_channels = self.registered_channel_index[max_registered_channel]
|
|
554
|
+
pca_channels = pca_channels[pca_channels < len(self.registered_geom)]
|
|
555
|
+
|
|
556
|
+
# load waveform embeddings and invert TPCA if we are relocating
|
|
557
|
+
waveforms = None # will allocate output array when we know its size
|
|
558
|
+
for bs in range(0, in_unit.size, batch_size):
|
|
559
|
+
be = min(in_unit.size, bs + batch_size)
|
|
560
|
+
|
|
561
|
+
batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
|
|
562
|
+
n_batch, rank, c = batch.shape
|
|
563
|
+
|
|
564
|
+
# invert TPCA for relocation
|
|
565
|
+
if self.relocated:
|
|
566
|
+
batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
|
|
567
|
+
batch = self.tpca.inverse_transform(batch)
|
|
568
|
+
t = batch.shape[1]
|
|
569
|
+
batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
|
|
570
|
+
|
|
571
|
+
# relocate or just restrict to channel subset
|
|
572
|
+
if self.relocated:
|
|
573
|
+
batch = relocate.relocated_waveforms_on_static_channels(
|
|
574
|
+
batch,
|
|
575
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
576
|
+
channel_index=self.channel_index,
|
|
577
|
+
target_channels=pca_channels,
|
|
578
|
+
xyza_from=self.xyza[in_unit[bs:be]],
|
|
579
|
+
z_to=self.z_reg[in_unit[bs:be]],
|
|
580
|
+
geom=self.geom,
|
|
581
|
+
registered_geom=self.registered_geom,
|
|
582
|
+
match_distance=self.match_distance,
|
|
583
|
+
)
|
|
584
|
+
else:
|
|
585
|
+
batch = drift_util.get_waveforms_on_static_channels(
|
|
586
|
+
batch,
|
|
587
|
+
self.geom,
|
|
588
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
589
|
+
channel_index=self.channel_index,
|
|
590
|
+
target_channels=pca_channels,
|
|
591
|
+
n_pitches_shift=n_pitches_shift[bs:be],
|
|
592
|
+
registered_geom=self.registered_geom,
|
|
593
|
+
match_distance=self.match_distance,
|
|
594
|
+
)
|
|
595
|
+
t = batch.shape[1]
|
|
596
|
+
|
|
597
|
+
if waveforms is None:
|
|
598
|
+
waveforms = np.empty(
|
|
599
|
+
(in_unit.size, t * pca_channels.size), dtype=batch.dtype
|
|
600
|
+
)
|
|
601
|
+
waveforms[bs:be] = batch.reshape(n_batch, -1)
|
|
602
|
+
|
|
603
|
+
# remove channels which are entirely nan
|
|
604
|
+
not_entirely_nan_channels = np.flatnonzero(np.isfinite(waveforms).any(axis=0))
|
|
605
|
+
if not_entirely_nan_channels.size < waveforms.shape[1]:
|
|
606
|
+
waveforms = waveforms[:, not_entirely_nan_channels]
|
|
607
|
+
|
|
608
|
+
# figure out which waveforms overlap completely with the remaining channels
|
|
609
|
+
no_nan = np.flatnonzero(np.isfinite(waveforms).all(axis=1))
|
|
610
|
+
if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
|
|
611
|
+
return False, no_nan, None
|
|
612
|
+
|
|
613
|
+
else:
|
|
614
|
+
size_wf = waveforms.shape[1]
|
|
615
|
+
waveforms_l2_norm = cdist(waveforms[no_nan], waveforms[no_nan]) / size_wf
|
|
616
|
+
return True, no_nan, waveforms_l2_norm
|
|
617
|
+
|
|
618
|
+
def pca_features(
|
|
619
|
+
self,
|
|
620
|
+
in_unit,
|
|
621
|
+
max_registered_channel,
|
|
622
|
+
n_pitches_shift,
|
|
623
|
+
batch_size=128,
|
|
624
|
+
max_samples_pca=5_000,
|
|
625
|
+
amplitude_normalized=False,
|
|
626
|
+
):
|
|
627
|
+
"""Compute relocated PCA features on a drift-invariant channel set
|
|
628
|
+
|
|
629
|
+
Returns
|
|
630
|
+
-------
|
|
631
|
+
enough_good_spikes : bool
|
|
632
|
+
kept : integer index array, size < in_unit.size, indices into in_unit
|
|
633
|
+
pca_embeds : array of shape (kept.size, rank)
|
|
634
|
+
|
|
635
|
+
"""
|
|
636
|
+
# figure out which set of channels to use
|
|
637
|
+
# we use the stored amplitudes to do this rather than computing a
|
|
638
|
+
# template, which can be expensive
|
|
639
|
+
# max_batch_size set to avoid memory errors
|
|
640
|
+
pca_channels = self.registered_channel_index[max_registered_channel]
|
|
641
|
+
pca_channels = pca_channels[pca_channels < len(self.registered_geom)]
|
|
642
|
+
|
|
643
|
+
# fast path for small data case
|
|
644
|
+
if in_unit.size < max_samples_pca:
|
|
645
|
+
waveforms = batched_h5_read(self.tpca_features, in_unit)
|
|
646
|
+
n, rank, c = waveforms.shape
|
|
647
|
+
|
|
648
|
+
# invert TPCA for relocation
|
|
649
|
+
if self.relocated:
|
|
650
|
+
waveforms = waveforms.transpose(0, 2, 1).reshape(n * c, rank)
|
|
651
|
+
waveforms = self.tpca.inverse_transform(waveforms)
|
|
652
|
+
t = waveforms.shape[1]
|
|
653
|
+
waveforms = waveforms.reshape(n, c, t).transpose(0, 2, 1)
|
|
654
|
+
|
|
655
|
+
# relocate or just restrict to channel subset
|
|
656
|
+
if self.relocated:
|
|
657
|
+
waveforms = relocate.relocated_waveforms_on_static_channels(
|
|
658
|
+
waveforms,
|
|
659
|
+
main_channels=self.channels[in_unit],
|
|
660
|
+
channel_index=self.channel_index,
|
|
661
|
+
target_channels=pca_channels,
|
|
662
|
+
xyza_from=self.xyza[in_unit],
|
|
663
|
+
z_to=self.z_reg[in_unit],
|
|
664
|
+
geom=self.geom,
|
|
665
|
+
registered_geom=self.registered_geom,
|
|
666
|
+
match_distance=self.match_distance,
|
|
667
|
+
)
|
|
668
|
+
else:
|
|
669
|
+
waveforms = drift_util.get_waveforms_on_static_channels(
|
|
670
|
+
waveforms,
|
|
671
|
+
self.geom,
|
|
672
|
+
main_channels=self.channels[in_unit],
|
|
673
|
+
channel_index=self.channel_index,
|
|
674
|
+
target_channels=pca_channels,
|
|
675
|
+
n_pitches_shift=n_pitches_shift,
|
|
676
|
+
registered_geom=self.registered_geom,
|
|
677
|
+
match_distance=self.match_distance,
|
|
678
|
+
)
|
|
679
|
+
t = waveforms.shape[1]
|
|
680
|
+
|
|
681
|
+
# remove channels which are entirely nan
|
|
682
|
+
not_entirely_nan_channels = np.flatnonzero(
|
|
683
|
+
np.isfinite(waveforms[:, 0, :]).any(axis=0)
|
|
684
|
+
)
|
|
685
|
+
if not not_entirely_nan_channels.size:
|
|
686
|
+
return False, None, None
|
|
687
|
+
if not_entirely_nan_channels.size < waveforms.shape[2]:
|
|
688
|
+
waveforms = waveforms[:, :, not_entirely_nan_channels]
|
|
689
|
+
|
|
690
|
+
# figure out which waveforms overlap completely with the remaining channels
|
|
691
|
+
no_nan = np.flatnonzero(np.isfinite(waveforms[:, 0, :]).all(axis=1))
|
|
692
|
+
if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
|
|
693
|
+
return False, no_nan, None
|
|
694
|
+
|
|
695
|
+
# fit the per-cluster small rank PCA
|
|
696
|
+
pca = PCA(
|
|
697
|
+
self.n_pca_features,
|
|
698
|
+
random_state=np.random.RandomState(seed=self.rg.bit_generator),
|
|
699
|
+
whiten=True,
|
|
700
|
+
)
|
|
701
|
+
if self.pca_imputation is None:
|
|
702
|
+
waveforms = waveforms[no_nan]
|
|
703
|
+
waveforms = waveforms.reshape(waveforms.shape[0], -1)
|
|
704
|
+
if self.amplitude_normalized:
|
|
705
|
+
waveforms /= self.amplitudes[in_unit[no_nan]][:, None]
|
|
706
|
+
pca_projs = pca.fit_transform(waveforms)
|
|
707
|
+
return True, no_nan, pca_projs
|
|
708
|
+
elif self.pca_imputation in ("mean", "iterative"):
|
|
709
|
+
wfs = waveforms.reshape(waveforms.shape[0], -1)
|
|
710
|
+
if self.amplitude_normalized:
|
|
711
|
+
wfs /= self.amplitudes[in_unit][:, None]
|
|
712
|
+
pca.fit(wfs[no_nan])
|
|
713
|
+
if self.pca_imputation == "iterative":
|
|
714
|
+
wfs = np.where(np.isfinite(wfs), wfs, pca.mean_[None])
|
|
715
|
+
isnan = np.setdiff1d(np.arange(wfs.shape[0]), no_nan)
|
|
716
|
+
for i in range(10):
|
|
717
|
+
emb = pca.fit_transform(wfs)
|
|
718
|
+
wfs[isnan] = pca.inverse_transform(emb[isnan])
|
|
719
|
+
pca_projs = pca.transform(wfs)
|
|
720
|
+
else:
|
|
721
|
+
pca_projs = pca.transform(
|
|
722
|
+
np.where(np.isfinite(wfs), wfs, pca.mean_[None])
|
|
723
|
+
)
|
|
724
|
+
return True, slice(None), pca_projs
|
|
725
|
+
else:
|
|
726
|
+
assert False
|
|
727
|
+
|
|
728
|
+
# otherwise, we have bigger data. in that case, first fit the PCA
|
|
729
|
+
fit_choices = self.rg.choice(in_unit.size, size=max_samples_pca, replace=False)
|
|
730
|
+
fit_choices.sort()
|
|
731
|
+
fit_waveforms = batched_h5_read(self.tpca_features, in_unit[fit_choices])
|
|
732
|
+
n, rank, c = fit_waveforms.shape
|
|
733
|
+
|
|
734
|
+
# invert TPCA for relocation
|
|
735
|
+
if self.relocated:
|
|
736
|
+
fit_waveforms = fit_waveforms.transpose(0, 2, 1).reshape(n * c, rank)
|
|
737
|
+
fit_waveforms = self.tpca.inverse_transform(fit_waveforms)
|
|
738
|
+
t = fit_waveforms.shape[1]
|
|
739
|
+
fit_waveforms = fit_waveforms.reshape(n, c, t).transpose(0, 2, 1)
|
|
740
|
+
|
|
741
|
+
# relocate or just restrict to channel subset
|
|
742
|
+
if self.relocated:
|
|
743
|
+
fit_waveforms = relocate.relocated_waveforms_on_static_channels(
|
|
744
|
+
fit_waveforms,
|
|
745
|
+
main_channels=self.channels[in_unit[fit_choices]],
|
|
746
|
+
channel_index=self.channel_index,
|
|
747
|
+
target_channels=pca_channels,
|
|
748
|
+
xyza_from=self.xyza[in_unit[fit_choices]],
|
|
749
|
+
z_to=self.z_reg[in_unit[fit_choices]],
|
|
750
|
+
geom=self.geom,
|
|
751
|
+
registered_geom=self.registered_geom,
|
|
752
|
+
match_distance=self.match_distance,
|
|
753
|
+
)
|
|
754
|
+
else:
|
|
755
|
+
fit_waveforms = drift_util.get_waveforms_on_static_channels(
|
|
756
|
+
fit_waveforms,
|
|
757
|
+
self.geom,
|
|
758
|
+
main_channels=self.channels[in_unit[fit_choices]],
|
|
759
|
+
channel_index=self.channel_index,
|
|
760
|
+
target_channels=pca_channels,
|
|
761
|
+
n_pitches_shift=n_pitches_shift[fit_choices],
|
|
762
|
+
registered_geom=self.registered_geom,
|
|
763
|
+
match_distance=self.match_distance,
|
|
764
|
+
)
|
|
765
|
+
t = fit_waveforms.shape[1]
|
|
766
|
+
|
|
767
|
+
# remove channels which are entirely nan
|
|
768
|
+
not_entirely_nan_channels = np.flatnonzero(
|
|
769
|
+
np.isfinite(fit_waveforms[:, 0, :]).any(axis=0)
|
|
770
|
+
)
|
|
771
|
+
if not not_entirely_nan_channels.size:
|
|
772
|
+
print("All empty chans")
|
|
773
|
+
return False, None, None
|
|
774
|
+
|
|
775
|
+
subset_chans = not_entirely_nan_channels.size < fit_waveforms.shape[2]
|
|
776
|
+
if subset_chans:
|
|
777
|
+
fit_waveforms = fit_waveforms[:, :, not_entirely_nan_channels]
|
|
778
|
+
|
|
779
|
+
# figure out which fit_waveforms overlap completely with the remaining channels
|
|
780
|
+
no_nan = np.flatnonzero(np.isfinite(fit_waveforms[:, 0, :]).all(axis=1))
|
|
781
|
+
if no_nan.size < max(self.min_cluster_size, self.n_pca_features):
|
|
782
|
+
return False, no_nan, None
|
|
783
|
+
fit_waveforms = fit_waveforms.reshape(fit_waveforms.shape[0], -1)
|
|
784
|
+
if self.amplitude_normalized:
|
|
785
|
+
amp_ix = in_unit[fit_choices]
|
|
786
|
+
fit_waveforms /= self.amplitudes[amp_ix][:, None]
|
|
787
|
+
|
|
788
|
+
# fit the per-cluster small rank PCA
|
|
789
|
+
pca = PCA(
|
|
790
|
+
self.n_pca_features,
|
|
791
|
+
random_state=np.random.RandomState(seed=self.rg.bit_generator),
|
|
792
|
+
whiten=True,
|
|
793
|
+
)
|
|
794
|
+
pca.fit(fit_waveforms[no_nan])
|
|
795
|
+
if self.pca_imputation == "iterative":
|
|
796
|
+
fit_waveforms = np.where(
|
|
797
|
+
np.isfinite(fit_waveforms), fit_waveforms, pca.mean_[None]
|
|
798
|
+
)
|
|
799
|
+
isnan = np.setdiff1d(np.arange(fit_waveforms.shape[0]), no_nan)
|
|
800
|
+
for i in range(10):
|
|
801
|
+
emb = pca.fit_transform(fit_waveforms)
|
|
802
|
+
fit_waveforms[isnan] = pca.inverse_transform(emb[isnan])
|
|
803
|
+
|
|
804
|
+
# now, we have the PCA. let's embed our data in batches.
|
|
805
|
+
pca_projs = np.full(
|
|
806
|
+
(in_unit.size, self.n_pca_features),
|
|
807
|
+
np.nan,
|
|
808
|
+
dtype=fit_waveforms.dtype,
|
|
809
|
+
)
|
|
810
|
+
all_kept = []
|
|
811
|
+
|
|
812
|
+
# load waveform embeddings and invert TPCA if we are relocating
|
|
813
|
+
for bs in range(0, in_unit.size, batch_size):
|
|
814
|
+
be = min(in_unit.size, bs + batch_size)
|
|
815
|
+
|
|
816
|
+
batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
|
|
817
|
+
n_batch, rank, c = batch.shape
|
|
818
|
+
|
|
819
|
+
# invert TPCA for relocation
|
|
820
|
+
if self.relocated:
|
|
821
|
+
batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
|
|
822
|
+
batch = self.tpca.inverse_transform(batch)
|
|
823
|
+
t = batch.shape[1]
|
|
824
|
+
batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
|
|
825
|
+
|
|
826
|
+
# relocate or just restrict to channel subset
|
|
827
|
+
if self.relocated:
|
|
828
|
+
batch = relocate.relocated_waveforms_on_static_channels(
|
|
829
|
+
batch,
|
|
830
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
831
|
+
channel_index=self.channel_index,
|
|
832
|
+
target_channels=pca_channels,
|
|
833
|
+
xyza_from=self.xyza[in_unit[bs:be]],
|
|
834
|
+
z_to=self.z_reg[in_unit[bs:be]],
|
|
835
|
+
geom=self.geom,
|
|
836
|
+
registered_geom=self.registered_geom,
|
|
837
|
+
match_distance=self.match_distance,
|
|
838
|
+
)
|
|
839
|
+
else:
|
|
840
|
+
batch = drift_util.get_waveforms_on_static_channels(
|
|
841
|
+
batch,
|
|
842
|
+
self.geom,
|
|
843
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
844
|
+
channel_index=self.channel_index,
|
|
845
|
+
target_channels=pca_channels,
|
|
846
|
+
n_pitches_shift=n_pitches_shift[bs:be],
|
|
847
|
+
registered_geom=self.registered_geom,
|
|
848
|
+
match_distance=self.match_distance,
|
|
849
|
+
)
|
|
850
|
+
t = batch.shape[1]
|
|
851
|
+
|
|
852
|
+
if subset_chans:
|
|
853
|
+
batch = batch[:, :, not_entirely_nan_channels]
|
|
854
|
+
|
|
855
|
+
if self.pca_imputation is None:
|
|
856
|
+
no_nan = np.flatnonzero(np.isfinite(batch[:, 0, :]).all(axis=1))
|
|
857
|
+
if not no_nan.size:
|
|
858
|
+
continue
|
|
859
|
+
all_kept.append(bs + no_nan)
|
|
860
|
+
batch = batch[no_nan].reshape(no_nan.size, -1)
|
|
861
|
+
if self.amplitude_normalized:
|
|
862
|
+
amp_ix = in_unit[bs:be][no_nan]
|
|
863
|
+
batch /= self.amplitudes[amp_ix][:, None]
|
|
864
|
+
pca_projs[bs + no_nan] = pca.transform(batch)
|
|
865
|
+
elif self.pca_imputation in ("mean", "iterative"):
|
|
866
|
+
batch = batch.reshape(batch.shape[0], -1)
|
|
867
|
+
batch /= self.amplitudes[in_unit[bs:be]][:, None]
|
|
868
|
+
pca_projs[bs:be] = pca.transform(
|
|
869
|
+
np.where(np.isfinite(batch), batch, pca.mean_[None])
|
|
870
|
+
)
|
|
871
|
+
else:
|
|
872
|
+
assert False
|
|
873
|
+
if self.pca_imputation is None:
|
|
874
|
+
kept = np.concatenate(all_kept)
|
|
875
|
+
elif self.pca_imputation in ("mean", "iterative"):
|
|
876
|
+
kept = slice(None)
|
|
877
|
+
else:
|
|
878
|
+
assert False
|
|
879
|
+
return True, kept, pca_projs[kept]
|
|
880
|
+
|
|
881
|
+
def tpca_reassignment(
|
|
882
|
+
self,
|
|
883
|
+
clust_labels,
|
|
884
|
+
in_unit,
|
|
885
|
+
n_pitches_shift,
|
|
886
|
+
centroid_samples=500,
|
|
887
|
+
batch_size=128,
|
|
888
|
+
):
|
|
889
|
+
unit_ids = np.unique(clust_labels)
|
|
890
|
+
unit_ids = unit_ids[unit_ids >= 0]
|
|
891
|
+
n_units = unit_ids.size
|
|
892
|
+
|
|
893
|
+
# compute centroids in TPCA space
|
|
894
|
+
centroids = []
|
|
895
|
+
for split_label in unit_ids:
|
|
896
|
+
in_split = np.flatnonzero(clust_labels == split_label)
|
|
897
|
+
if in_split.size > centroid_samples:
|
|
898
|
+
in_split = self.rg.choice(
|
|
899
|
+
in_split, size=centroid_samples, replace=False
|
|
900
|
+
)
|
|
901
|
+
in_split.sort()
|
|
902
|
+
|
|
903
|
+
ix = in_unit[in_split]
|
|
904
|
+
split_feats = batched_h5_read(self.tpca_features, ix)
|
|
905
|
+
n_batch, rank, c = split_feats.shape
|
|
906
|
+
|
|
907
|
+
# invert TPCA for relocation
|
|
908
|
+
if self.relocated:
|
|
909
|
+
split_feats = split_feats.transpose(0, 2, 1).reshape(n_batch * c, rank)
|
|
910
|
+
split_feats = self.tpca.inverse_transform(split_feats)
|
|
911
|
+
t = split_feats.shape[1]
|
|
912
|
+
split_feats = split_feats.reshape(n_batch, c, t).transpose(0, 2, 1)
|
|
913
|
+
|
|
914
|
+
# relocate or just restrict to channel subset
|
|
915
|
+
if self.relocated:
|
|
916
|
+
split_feats = relocate.relocated_waveforms_on_static_channels(
|
|
917
|
+
split_feats,
|
|
918
|
+
target_channels=slice(None),
|
|
919
|
+
main_channels=self.channels[ix],
|
|
920
|
+
channel_index=self.channel_index,
|
|
921
|
+
xyza_from=self.xyza[ix],
|
|
922
|
+
z_to=self.z_reg[ix],
|
|
923
|
+
geom=self.geom,
|
|
924
|
+
registered_geom=self.registered_geom,
|
|
925
|
+
match_distance=self.match_distance,
|
|
926
|
+
)
|
|
927
|
+
else:
|
|
928
|
+
split_feats = drift_util.get_waveforms_on_static_channels(
|
|
929
|
+
split_feats,
|
|
930
|
+
self.geom,
|
|
931
|
+
main_channels=self.channels[ix],
|
|
932
|
+
channel_index=self.channel_index,
|
|
933
|
+
n_pitches_shift=n_pitches_shift[in_split],
|
|
934
|
+
registered_geom=self.registered_geom,
|
|
935
|
+
match_distance=self.match_distance,
|
|
936
|
+
)
|
|
937
|
+
centroids.append(np.nan_to_num(np.nanmean(split_feats, axis=0)))
|
|
938
|
+
centroids = np.array(centroids)
|
|
939
|
+
chans = np.flatnonzero(np.isfinite(centroids[:, 0, :]).any(0))
|
|
940
|
+
centroids = centroids[:, :, chans]
|
|
941
|
+
|
|
942
|
+
# load waveform embeddings and invert TPCA if we are relocating
|
|
943
|
+
new_labels = np.full_like(clust_labels, -1)
|
|
944
|
+
assert clust_labels.shape == in_unit.shape
|
|
945
|
+
for bs in range(0, in_unit.size, batch_size):
|
|
946
|
+
be = min(in_unit.size, bs + batch_size)
|
|
947
|
+
|
|
948
|
+
batch = batched_h5_read(self.tpca_features, in_unit[bs:be])
|
|
949
|
+
n_batch, rank, c = batch.shape
|
|
950
|
+
|
|
951
|
+
# invert TPCA for relocation
|
|
952
|
+
if self.relocated:
|
|
953
|
+
batch = batch.transpose(0, 2, 1).reshape(n_batch * c, rank)
|
|
954
|
+
batch = self.tpca.inverse_transform(batch)
|
|
955
|
+
t = batch.shape[1]
|
|
956
|
+
batch = batch.reshape(n_batch, c, t).transpose(0, 2, 1)
|
|
957
|
+
|
|
958
|
+
# relocate or just restrict to channel subset
|
|
959
|
+
if self.relocated:
|
|
960
|
+
batch = relocate.relocated_waveforms_on_static_channels(
|
|
961
|
+
batch,
|
|
962
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
963
|
+
channel_index=self.channel_index,
|
|
964
|
+
target_channels=chans,
|
|
965
|
+
xyza_from=self.xyza[in_unit[bs:be]],
|
|
966
|
+
z_to=self.z_reg[in_unit[bs:be]],
|
|
967
|
+
geom=self.geom,
|
|
968
|
+
registered_geom=self.registered_geom,
|
|
969
|
+
match_distance=self.match_distance,
|
|
970
|
+
)
|
|
971
|
+
else:
|
|
972
|
+
batch = drift_util.get_waveforms_on_static_channels(
|
|
973
|
+
batch,
|
|
974
|
+
self.geom,
|
|
975
|
+
main_channels=self.channels[in_unit[bs:be]],
|
|
976
|
+
channel_index=self.channel_index,
|
|
977
|
+
target_channels=chans,
|
|
978
|
+
n_pitches_shift=n_pitches_shift[bs:be],
|
|
979
|
+
registered_geom=self.registered_geom,
|
|
980
|
+
match_distance=self.match_distance,
|
|
981
|
+
)
|
|
982
|
+
|
|
983
|
+
d = nan_sqeuclidean_distances(
|
|
984
|
+
batch.reshape(batch.shape[0], -1), centroids.reshape(n_units, -1)
|
|
985
|
+
)
|
|
986
|
+
# d = batch.reshape(batch.shape[0], -1)[:, None] - centroids.reshape(n_units, -1)[None, :]
|
|
987
|
+
# np.nan_to_num(d, copy=False)
|
|
988
|
+
# d *= d
|
|
989
|
+
# d = d.sum(2)
|
|
990
|
+
new_labels[bs:be] = d.argmin(1)
|
|
991
|
+
# for j, i in enumerate(range(bs, be)):
|
|
992
|
+
# wf = batch[j]
|
|
993
|
+
# mask = np.isfinite(wf[0])
|
|
994
|
+
# wf = wf[:, mask].ravel()[None]
|
|
995
|
+
# c = centroids[:, :, mask].reshape(n_units, -1)
|
|
996
|
+
# new_labels[i] = cdist(wf, c, metric="sqeuclidean").ravel().argmin()
|
|
997
|
+
|
|
998
|
+
return new_labels
|
|
999
|
+
|
|
1000
|
+
def initialize_from_h5(
|
|
1001
|
+
self,
|
|
1002
|
+
peeling_hdf5_filename,
|
|
1003
|
+
peeling_featurization_pt,
|
|
1004
|
+
tpca_features_dataset_name="collisioncleaned_tpca_features",
|
|
1005
|
+
localizations_dataset_name="point_source_localizations",
|
|
1006
|
+
amplitudes_dataset_name="denoised_ptp_amplitudes",
|
|
1007
|
+
amplitude_vectors_dataset_name="denoised_ptp_amplitude_vectors",
|
|
1008
|
+
):
|
|
1009
|
+
peeling_hdf5_filename = Path(peeling_hdf5_filename)
|
|
1010
|
+
h5 = h5py.File(peeling_hdf5_filename, "r", libver="latest", locking=False)
|
|
1011
|
+
self.geom = h5["geom"][:]
|
|
1012
|
+
self.channel_index = h5["channel_index"][:]
|
|
1013
|
+
self.channels = h5["channels"][:]
|
|
1014
|
+
self.match_distance = pdist(self.geom).min() / 2
|
|
1015
|
+
|
|
1016
|
+
if self.use_localization_features or self.relocated or self.use_time_feature:
|
|
1017
|
+
self.t_s = h5["times_seconds"][:]
|
|
1018
|
+
|
|
1019
|
+
if self.use_localization_features or self.relocated:
|
|
1020
|
+
self.xyza = h5[localizations_dataset_name][:]
|
|
1021
|
+
# registered spike positions (=originals if not relocated)
|
|
1022
|
+
self.z_reg = self.z = self.xyza[:, 2]
|
|
1023
|
+
|
|
1024
|
+
self.amplitudes = h5[amplitudes_dataset_name][:]
|
|
1025
|
+
self.registered_geom = self.geom
|
|
1026
|
+
self.registered_channel_index = self.channel_index
|
|
1027
|
+
|
|
1028
|
+
if self.motion_est is not None:
|
|
1029
|
+
self.z_reg = self.motion_est.correct_s(self.t_s, self.z)
|
|
1030
|
+
if self.relocated or self.n_pca_features or self.use_wfs_L2_norm:
|
|
1031
|
+
self.registered_geom = drift_util.registered_geometry(
|
|
1032
|
+
self.geom, self.motion_est
|
|
1033
|
+
)
|
|
1034
|
+
self.registered_channel_index = waveform_util.make_channel_index(
|
|
1035
|
+
self.registered_geom, self.channel_selection_radius
|
|
1036
|
+
)
|
|
1037
|
+
|
|
1038
|
+
if (
|
|
1039
|
+
(self.use_localization_features and self.relocated)
|
|
1040
|
+
or self.n_pca_features
|
|
1041
|
+
or self.use_ptp
|
|
1042
|
+
or self.use_wfs_L2_norm
|
|
1043
|
+
):
|
|
1044
|
+
self.amplitude_vectors = h5[amplitude_vectors_dataset_name]
|
|
1045
|
+
|
|
1046
|
+
if self.use_localization_features:
|
|
1047
|
+
self.localization_features = np.c_[
|
|
1048
|
+
self.xyza[:, 0],
|
|
1049
|
+
self.z_reg,
|
|
1050
|
+
np.log(self.log_c + self.amplitudes),
|
|
1051
|
+
]
|
|
1052
|
+
self.localization_features *= self.localization_feature_scales
|
|
1053
|
+
|
|
1054
|
+
if self.n_pca_features or self.use_wfs_L2_norm:
|
|
1055
|
+
# don't load these one into memory, since it's a bit heavier
|
|
1056
|
+
self.tpca_features = h5[tpca_features_dataset_name]
|
|
1057
|
+
|
|
1058
|
+
if peeling_featurization_pt is None:
|
|
1059
|
+
mdir = peeling_hdf5_filename.parent / f"{peeling_hdf5_filename.stem}_models"
|
|
1060
|
+
peeling_featurization_pt = mdir / "featurization_pipeline.pt"
|
|
1061
|
+
assert peeling_featurization_pt.exists()
|
|
1062
|
+
|
|
1063
|
+
if (self.n_pca_features or self.use_wfs_L2_norm) and self.relocated:
|
|
1064
|
+
# load up featurization pipeline for tpca inversion
|
|
1065
|
+
assert peeling_featurization_pt is not None
|
|
1066
|
+
feature_pipeline = get_featurization_pipeline(
|
|
1067
|
+
peeling_hdf5_filename, featurization_pipeline_pt
|
|
1068
|
+
)
|
|
1069
|
+
tpca_feature = [
|
|
1070
|
+
f
|
|
1071
|
+
for f in feature_pipeline.transformers
|
|
1072
|
+
if f.name == tpca_features_dataset_name
|
|
1073
|
+
]
|
|
1074
|
+
assert len(tpca_feature) == 1
|
|
1075
|
+
self.tpca = tpca_feature[0].to_sklearn()
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
class ChunkForwardBackwardFeatureSplit(FeatureSplit):
|
|
1079
|
+
def __init__(
|
|
1080
|
+
self,
|
|
1081
|
+
peeling_hdf5_filename,
|
|
1082
|
+
recording=None,
|
|
1083
|
+
chunk_size_s=300.0,
|
|
1084
|
+
**split_strategy_kwargs,
|
|
1085
|
+
):
|
|
1086
|
+
"""Split clusters based on per-cluster PCA and localization features
|
|
1087
|
+
|
|
1088
|
+
Fits PCA to relocated waveforms within a cluster, combines these with
|
|
1089
|
+
localization features (x, registered z, and max amplitude), and runs
|
|
1090
|
+
HDBSCAN to refine the labeling.
|
|
1091
|
+
|
|
1092
|
+
Arguments
|
|
1093
|
+
---------
|
|
1094
|
+
peeling_hdf5_filename : str or Path
|
|
1095
|
+
Path to the HDF5 file output by a peeling step
|
|
1096
|
+
peeling_featurization_pt : str or Path
|
|
1097
|
+
Path to the .pt file where a featurization WaveformPipeline was
|
|
1098
|
+
saved during peeling. Required if relocated=True.
|
|
1099
|
+
motion_est : dredge.motion_util.MotionEstimate, optional
|
|
1100
|
+
If supplied, registered depth positions will be used for clustering
|
|
1101
|
+
and relocation
|
|
1102
|
+
use_localization_features : bool
|
|
1103
|
+
If false, only PCA features are used for clustering
|
|
1104
|
+
n_pca_features : int
|
|
1105
|
+
If 0, no PCA is used
|
|
1106
|
+
relocated : bool
|
|
1107
|
+
Whether to relocate waveforms according to the motion estimate.
|
|
1108
|
+
Only used if motion_est is not None
|
|
1109
|
+
localization_feature_scales : 3-tuple of float
|
|
1110
|
+
Scales to apply to x, registered z, and log-scaled amplitude during
|
|
1111
|
+
clustering to approximately sphere the clusters
|
|
1112
|
+
channel_selection_radius : float
|
|
1113
|
+
Spatial radius around the main channel used when selecting channels
|
|
1114
|
+
used when extracting local PCA features
|
|
1115
|
+
min_cluster_size, min_samples, cluster_selection_epsilon
|
|
1116
|
+
HDBSCAN parameters. See their documentation for lots of info.
|
|
1117
|
+
"""
|
|
1118
|
+
split_strategy_kwargs["return_localization_features"] = True
|
|
1119
|
+
super().__init__(
|
|
1120
|
+
peeling_hdf5_filename, recording=recording, **split_strategy_kwargs
|
|
1121
|
+
)
|
|
1122
|
+
|
|
1123
|
+
# Check for ensembling
|
|
1124
|
+
self.chunk_size_s = chunk_size_s
|
|
1125
|
+
assert (
|
|
1126
|
+
self.use_localization_features
|
|
1127
|
+
), "Need to use loc features for ensembling over chunks"
|
|
1128
|
+
assert (
|
|
1129
|
+
self.recording is not None
|
|
1130
|
+
), "Need to input recording for ensembling over chunks"
|
|
1131
|
+
assert (
|
|
1132
|
+
self.chunk_size_s is not None
|
|
1133
|
+
), "Need to input chunk size for ensembling over chunks"
|
|
1134
|
+
|
|
1135
|
+
def split_cluster(self, in_unit):
|
|
1136
|
+
chunk_samples = self.recording.sampling_frequency * self.chunk_size_s
|
|
1137
|
+
|
|
1138
|
+
n_chunks = self.recording.get_num_samples() / chunk_samples
|
|
1139
|
+
# we'll count the remainder as a chunk if it's at least 2/3 of one
|
|
1140
|
+
n_chunks = np.floor(n_chunks) + (n_chunks - np.floor(n_chunks) > 0.66)
|
|
1141
|
+
n_chunks = int(max(1, n_chunks))
|
|
1142
|
+
|
|
1143
|
+
# evenly divide the recording into chunks
|
|
1144
|
+
assert self.recording.get_num_segments() == 1
|
|
1145
|
+
start_time_s, end_time_s = self.recording._recording_segments[
|
|
1146
|
+
0
|
|
1147
|
+
].sample_index_to_time(np.array([0, self.recording.get_num_samples() - 1]))
|
|
1148
|
+
chunk_times_s = np.linspace(start_time_s, end_time_s, num=n_chunks + 1)
|
|
1149
|
+
chunk_time_ranges_s = list(zip(chunk_times_s[:-1], chunk_times_s[1:]))
|
|
1150
|
+
|
|
1151
|
+
chunks_indices_unit = [
|
|
1152
|
+
np.flatnonzero(
|
|
1153
|
+
np.logical_and(
|
|
1154
|
+
self.t_s[in_unit] >= chunk_range[0],
|
|
1155
|
+
self.t_s[in_unit] < chunk_range[1],
|
|
1156
|
+
)
|
|
1157
|
+
)
|
|
1158
|
+
for chunk_range in chunk_time_ranges_s
|
|
1159
|
+
]
|
|
1160
|
+
|
|
1161
|
+
# cluster each chunk. can be parallelized in the future.
|
|
1162
|
+
splits = [
|
|
1163
|
+
super().split_cluster(in_unit[idx_unit_chunk])
|
|
1164
|
+
for idx_unit_chunk in chunks_indices_unit
|
|
1165
|
+
]
|
|
1166
|
+
|
|
1167
|
+
new_labels_chunks = []
|
|
1168
|
+
for k in range(len(splits)):
|
|
1169
|
+
new_labels = np.full(len(in_unit), -1)
|
|
1170
|
+
if len(chunks_indices_unit[k]) and splits[k].new_labels is not None:
|
|
1171
|
+
new_labels[chunks_indices_unit[k]] = splits[k].new_labels
|
|
1172
|
+
elif len(chunks_indices_unit[k]) and splits[k].new_labels is None:
|
|
1173
|
+
# moved the 0 logic from split_cluster to here
|
|
1174
|
+
new_labels[chunks_indices_unit[k]] = 0
|
|
1175
|
+
new_labels_chunks.append(new_labels)
|
|
1176
|
+
|
|
1177
|
+
split_sortings = [
|
|
1178
|
+
DARTsortSorting(
|
|
1179
|
+
times_samples=self.t_s[
|
|
1180
|
+
in_unit
|
|
1181
|
+
], # Needed for initialization but not for anything else
|
|
1182
|
+
channels=self.channels[in_unit],
|
|
1183
|
+
labels=new_labels,
|
|
1184
|
+
ephemeral_features={
|
|
1185
|
+
"point_source_localizations": self.xyza[in_unit],
|
|
1186
|
+
"denoised_ptp_amplitudes": self.amplitudes[in_unit],
|
|
1187
|
+
"times_seconds": self.t_s[in_unit],
|
|
1188
|
+
},
|
|
1189
|
+
)
|
|
1190
|
+
for new_labels in new_labels_chunks
|
|
1191
|
+
]
|
|
1192
|
+
|
|
1193
|
+
labels_ensembled = forward_backward(
|
|
1194
|
+
self.recording,
|
|
1195
|
+
chunk_time_ranges_s,
|
|
1196
|
+
split_sortings,
|
|
1197
|
+
log_c=self.log_c,
|
|
1198
|
+
feature_scales=self.localization_feature_scales,
|
|
1199
|
+
adaptive_feature_scales=self.rescale_all_features,
|
|
1200
|
+
motion_est=self.motion_est,
|
|
1201
|
+
verbose=False,
|
|
1202
|
+
)
|
|
1203
|
+
|
|
1204
|
+
is_split = np.setdiff1d(np.unique(labels_ensembled), [-1]).size > 1
|
|
1205
|
+
|
|
1206
|
+
return SplitResult(
|
|
1207
|
+
is_split=is_split,
|
|
1208
|
+
in_unit=in_unit,
|
|
1209
|
+
new_labels=labels_ensembled,
|
|
1210
|
+
x=self.localization_features[in_unit, 0],
|
|
1211
|
+
z_reg=self.localization_features[in_unit, 2],
|
|
1212
|
+
)
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
class NullSplit(SplitStrategy):
|
|
1216
|
+
def __init__(self, *args, **kwargs):
|
|
1217
|
+
pass
|
|
1218
|
+
|
|
1219
|
+
def split_cluster(self, in_unit):
|
|
1220
|
+
return SplitResult(in_unit=in_unit)
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
# this is to help split_clusters take a string argument
|
|
1224
|
+
all_split_strategies = [
|
|
1225
|
+
FeatureSplit,
|
|
1226
|
+
ChunkForwardBackwardFeatureSplit,
|
|
1227
|
+
NullSplit,
|
|
1228
|
+
]
|
|
1229
|
+
split_strategies_by_class_name = {cls.__name__: cls for cls in all_split_strategies}
|
|
1230
|
+
|
|
1231
|
+
# -- parallelism widgets
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
class SplitJobContext:
|
|
1235
|
+
def __init__(self, split_strategy):
|
|
1236
|
+
self.split_strategy = split_strategy
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
_split_job_context = None
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
def _split_job_init(
|
|
1243
|
+
split_strategy_class_name, peeling_hdf5_filename, split_strategy_kwargs
|
|
1244
|
+
):
|
|
1245
|
+
global _split_job_context
|
|
1246
|
+
SplitStrategy = split_strategies_by_class_name[split_strategy_class_name]
|
|
1247
|
+
if split_strategy_kwargs is None:
|
|
1248
|
+
split_strategy_kwargs = {}
|
|
1249
|
+
split_strategy_kwargs["peeling_hdf5_filename"] = peeling_hdf5_filename
|
|
1250
|
+
_split_job_context = SplitJobContext(SplitStrategy(**split_strategy_kwargs))
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def _split_job(in_unit):
|
|
1254
|
+
return _split_job_context.split_strategy.split_cluster(in_unit)
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
# other helpers
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
def mad_sigma(x, axis=0, keepdims=False, gauss_correct=True):
|
|
1261
|
+
med = np.median(x, axis=axis, keepdims=True)
|
|
1262
|
+
mad = np.median(np.abs(x - med), axis=axis, keepdims=keepdims)
|
|
1263
|
+
if gauss_correct:
|
|
1264
|
+
mad *= 1.4826
|
|
1265
|
+
return mad
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
def nan_sqeuclidean_distances(X, Y, copy_X=False, copy_Y=True):
|
|
1269
|
+
missing_X = np.isnan(X)
|
|
1270
|
+
missing_Y = np.isnan(Y)
|
|
1271
|
+
if copy_X:
|
|
1272
|
+
X = X.copy()
|
|
1273
|
+
if copy_Y:
|
|
1274
|
+
Y = Y.copy()
|
|
1275
|
+
X[missing_X] = 0
|
|
1276
|
+
Y[missing_Y] = 0
|
|
1277
|
+
distances = cdist(X, Y, metric="sqeuclidean")
|
|
1278
|
+
|
|
1279
|
+
# Adjust distances for missing values
|
|
1280
|
+
X *= X
|
|
1281
|
+
Y *= Y
|
|
1282
|
+
distances -= np.dot(X, missing_Y.T)
|
|
1283
|
+
distances -= np.dot(missing_X, Y.T)
|
|
1284
|
+
|
|
1285
|
+
return distances
|