DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
from tempfile import tempdir
|
|
2
|
+
from typing import Optional
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from scipy.cluster.hierarchy import fcluster, linkage
|
|
7
|
+
from scipy.sparse import coo_array
|
|
8
|
+
from scipy.sparse.csgraph import maximum_bipartite_matching
|
|
9
|
+
from tqdm.auto import tqdm
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
from ..util.internal_config import (
|
|
13
|
+
TemplateConfig,
|
|
14
|
+
TemplateMergeConfig,
|
|
15
|
+
ComputationConfig,
|
|
16
|
+
)
|
|
17
|
+
from ..util.logging_util import get_logger
|
|
18
|
+
from ..templates import TemplateData, template_util
|
|
19
|
+
from ..peel.matching_util.pairwise_util import (
|
|
20
|
+
construct_shift_indices,
|
|
21
|
+
iterate_compressed_pairwise_convolutions,
|
|
22
|
+
)
|
|
23
|
+
from ..util.data_util import DARTsortSorting, combine_sortings
|
|
24
|
+
from ..util import job_util
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
logger = get_logger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def merge_templates(
|
|
31
|
+
sorting: DARTsortSorting,
|
|
32
|
+
template_data: TemplateData,
|
|
33
|
+
max_shift_samples=40,
|
|
34
|
+
superres_linkage=np.max,
|
|
35
|
+
linkage="complete",
|
|
36
|
+
distance_kind="rms",
|
|
37
|
+
sym_function=np.minimum,
|
|
38
|
+
merge_distance_threshold=0.25,
|
|
39
|
+
temporal_upsampling_factor=1,
|
|
40
|
+
amplitude_scaling_variance=0.001,
|
|
41
|
+
amplitude_scaling_boundary=0.1,
|
|
42
|
+
svd_compression_rank=20,
|
|
43
|
+
min_channel_amplitude=0.0,
|
|
44
|
+
spatial_radius_a=None,
|
|
45
|
+
min_spatial_cosine=0.0,
|
|
46
|
+
conv_batch_size=128,
|
|
47
|
+
units_batch_size=8,
|
|
48
|
+
computation_cfg=None,
|
|
49
|
+
show_progress=True,
|
|
50
|
+
):
|
|
51
|
+
"""Template distance based merge
|
|
52
|
+
|
|
53
|
+
Pass in a sorting, recording and template config to make templates,
|
|
54
|
+
and this will merge them (with superres). Or, if you have templates
|
|
55
|
+
already, pass them into template_data and we can skip the template
|
|
56
|
+
construction.
|
|
57
|
+
|
|
58
|
+
Arguments
|
|
59
|
+
---------
|
|
60
|
+
max_shift_samples
|
|
61
|
+
Max offset during matching
|
|
62
|
+
superres_linkage
|
|
63
|
+
How to combine distances between two units' superres templates
|
|
64
|
+
By default, it's the max.
|
|
65
|
+
amplitude_scaling_*
|
|
66
|
+
Optionally allow scaling during matching
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
A new DARTsortSorting
|
|
71
|
+
"""
|
|
72
|
+
computation_cfg = job_util.ensure_computation_config(computation_cfg)
|
|
73
|
+
dist_matrix_kwargs = dict(
|
|
74
|
+
superres_linkage=superres_linkage,
|
|
75
|
+
sym_function=sym_function,
|
|
76
|
+
max_shift_samples=max_shift_samples,
|
|
77
|
+
temporal_upsampling_factor=temporal_upsampling_factor,
|
|
78
|
+
amplitude_scaling_variance=amplitude_scaling_variance,
|
|
79
|
+
amplitude_scaling_boundary=amplitude_scaling_boundary,
|
|
80
|
+
svd_compression_rank=svd_compression_rank,
|
|
81
|
+
min_channel_amplitude=min_channel_amplitude,
|
|
82
|
+
min_spatial_cosine=min_spatial_cosine,
|
|
83
|
+
conv_batch_size=conv_batch_size,
|
|
84
|
+
units_batch_size=units_batch_size,
|
|
85
|
+
distance_kind=distance_kind,
|
|
86
|
+
spatial_radius_a=spatial_radius_a,
|
|
87
|
+
)
|
|
88
|
+
units, dists, shifts, template_snrs = calculate_merge_distances(
|
|
89
|
+
template_data,
|
|
90
|
+
device=computation_cfg.actual_device(),
|
|
91
|
+
n_jobs=computation_cfg.actual_n_jobs(),
|
|
92
|
+
show_progress=show_progress,
|
|
93
|
+
**dist_matrix_kwargs, # type: ignore
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# now run hierarchical clustering
|
|
97
|
+
merged_sorting, new_unit_ids = recluster(
|
|
98
|
+
sorting,
|
|
99
|
+
units,
|
|
100
|
+
dists,
|
|
101
|
+
shifts,
|
|
102
|
+
template_snrs,
|
|
103
|
+
merge_distance_threshold=merge_distance_threshold,
|
|
104
|
+
link=linkage,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
return dict(sorting=merged_sorting, new_unit_ids=new_unit_ids)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def get_merge_distances(
|
|
111
|
+
template_data: TemplateData,
|
|
112
|
+
template_merge_cfg: TemplateMergeConfig,
|
|
113
|
+
computation_cfg: ComputationConfig | None = None,
|
|
114
|
+
cooccurrence_mask=None,
|
|
115
|
+
show_progress=True,
|
|
116
|
+
conv_batch_size=128,
|
|
117
|
+
units_batch_size=8,
|
|
118
|
+
sampling_frequency: float = 30000.0,
|
|
119
|
+
):
|
|
120
|
+
computation_cfg = job_util.ensure_computation_config(computation_cfg)
|
|
121
|
+
shift_samples = int(template_merge_cfg.max_shift_ms * (sampling_frequency / 1000))
|
|
122
|
+
units, dists, shifts, template_snrs = calculate_merge_distances(
|
|
123
|
+
template_data=template_data,
|
|
124
|
+
max_shift_samples=shift_samples,
|
|
125
|
+
temporal_upsampling_factor=template_merge_cfg.temporal_upsampling_factor,
|
|
126
|
+
amplitude_scaling_variance=template_merge_cfg.amplitude_scaling_variance,
|
|
127
|
+
amplitude_scaling_boundary=template_merge_cfg.amplitude_scaling_boundary,
|
|
128
|
+
svd_compression_rank=template_merge_cfg.svd_compression_rank,
|
|
129
|
+
min_spatial_cosine=template_merge_cfg.min_spatial_cosine,
|
|
130
|
+
cooccurrence_mask=cooccurrence_mask,
|
|
131
|
+
conv_batch_size=conv_batch_size,
|
|
132
|
+
units_batch_size=units_batch_size,
|
|
133
|
+
device=computation_cfg.actual_device(),
|
|
134
|
+
n_jobs=computation_cfg.actual_n_jobs(),
|
|
135
|
+
show_progress=show_progress,
|
|
136
|
+
distance_kind=template_merge_cfg.distance_kind,
|
|
137
|
+
)
|
|
138
|
+
return units, dists, shifts, template_snrs
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def calculate_merge_distances(
|
|
142
|
+
template_data,
|
|
143
|
+
superres_linkage=np.max,
|
|
144
|
+
sym_function=np.minimum,
|
|
145
|
+
max_shift_samples=40,
|
|
146
|
+
temporal_upsampling_factor=1,
|
|
147
|
+
amplitude_scaling_variance=0.001,
|
|
148
|
+
amplitude_scaling_boundary=0.1,
|
|
149
|
+
svd_compression_rank=20,
|
|
150
|
+
min_channel_amplitude=0.0,
|
|
151
|
+
min_spatial_cosine=0.0,
|
|
152
|
+
spatial_radius_a=None,
|
|
153
|
+
cooccurrence_mask=None,
|
|
154
|
+
conv_batch_size=128,
|
|
155
|
+
units_batch_size=8,
|
|
156
|
+
device=None,
|
|
157
|
+
n_jobs=0,
|
|
158
|
+
show_progress=True,
|
|
159
|
+
distance_kind="rms",
|
|
160
|
+
):
|
|
161
|
+
# allocate distance + shift matrices. shifts[i,j] is trough[j]-trough[i].
|
|
162
|
+
n_templates = template_data.templates.shape[0]
|
|
163
|
+
sup_dists = np.full((n_templates, n_templates), np.inf)
|
|
164
|
+
sup_shifts = np.zeros((n_templates, n_templates), dtype=int)
|
|
165
|
+
|
|
166
|
+
# build distance matrix
|
|
167
|
+
dec_res_iter = get_deconv_resid_decrease_iter(
|
|
168
|
+
template_data,
|
|
169
|
+
max_shift_samples=max_shift_samples,
|
|
170
|
+
temporal_upsampling_factor=temporal_upsampling_factor,
|
|
171
|
+
amplitude_scaling_variance=amplitude_scaling_variance,
|
|
172
|
+
amplitude_scaling_boundary=amplitude_scaling_boundary,
|
|
173
|
+
svd_compression_rank=svd_compression_rank,
|
|
174
|
+
min_channel_amplitude=min_channel_amplitude,
|
|
175
|
+
min_spatial_cosine=min_spatial_cosine,
|
|
176
|
+
cooccurrence_mask=cooccurrence_mask,
|
|
177
|
+
conv_batch_size=conv_batch_size,
|
|
178
|
+
units_batch_size=units_batch_size,
|
|
179
|
+
spatial_radius_a=spatial_radius_a,
|
|
180
|
+
device=device,
|
|
181
|
+
n_jobs=n_jobs,
|
|
182
|
+
show_progress=show_progress,
|
|
183
|
+
distance_kind=distance_kind,
|
|
184
|
+
)
|
|
185
|
+
for res in dec_res_iter:
|
|
186
|
+
if res is None:
|
|
187
|
+
# all pairs in chunk were ignored for one reason or another
|
|
188
|
+
continue
|
|
189
|
+
|
|
190
|
+
tixa = res.template_indices_a
|
|
191
|
+
tixb = res.template_indices_b
|
|
192
|
+
sup_dists[tixa, tixb] = res.deconv_resid_decreases / res.template_a_norms # type: ignore
|
|
193
|
+
sup_shifts[tixa, tixb] = res.shifts # type: ignore
|
|
194
|
+
|
|
195
|
+
# apply linkage to reduce across superres templates
|
|
196
|
+
units = np.unique(template_data.unit_ids)
|
|
197
|
+
if units.size < n_templates:
|
|
198
|
+
dists = np.full((units.size, units.size), np.inf)
|
|
199
|
+
shifts = np.zeros((units.size, units.size), dtype=int)
|
|
200
|
+
for ia, ua in enumerate(units):
|
|
201
|
+
in_ua = np.flatnonzero(template_data.unit_ids == ua)
|
|
202
|
+
for ib, ub in enumerate(units):
|
|
203
|
+
in_ub = np.flatnonzero(template_data.unit_ids == ub)
|
|
204
|
+
in_pair = (in_ua[:, None], in_ub[None, :])
|
|
205
|
+
dists[ia, ib] = superres_linkage(sup_dists[in_pair])
|
|
206
|
+
shifts[ia, ib] = np.median(sup_shifts[in_pair])
|
|
207
|
+
coarse_td = template_data.coarsen(with_locs=False)
|
|
208
|
+
template_snrs = np.ptp(coarse_td.templates, 1).max(1) / coarse_td.spike_counts
|
|
209
|
+
else:
|
|
210
|
+
dists = sup_dists
|
|
211
|
+
shifts = sup_shifts
|
|
212
|
+
template_snrs = (
|
|
213
|
+
np.ptp(template_data.templates, 1).max(1) / template_data.spike_counts
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
dists = sym_function(dists, dists.T)
|
|
217
|
+
np.fill_diagonal(dists, 0.0) # sometimes numerical 0 is -1e-6.
|
|
218
|
+
min_dist = dists.min()
|
|
219
|
+
if min_dist < -1e-3:
|
|
220
|
+
warnings.warn(f"Alarmingly negative min distance {min_dist}.")
|
|
221
|
+
dists = np.maximum(dists, 0.0, out=dists)
|
|
222
|
+
|
|
223
|
+
return units, dists, shifts, template_snrs
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def cross_match_distance_matrix(
|
|
227
|
+
template_data_a,
|
|
228
|
+
template_data_b,
|
|
229
|
+
superres_linkage=np.max,
|
|
230
|
+
sym_function=np.minimum,
|
|
231
|
+
max_shift_samples=40,
|
|
232
|
+
temporal_upsampling_factor=1,
|
|
233
|
+
amplitude_scaling_variance=0.001,
|
|
234
|
+
amplitude_scaling_boundary=0.1,
|
|
235
|
+
svd_compression_rank=20,
|
|
236
|
+
min_channel_amplitude=0.0,
|
|
237
|
+
min_spatial_cosine=0.0,
|
|
238
|
+
spatial_radius_a=None,
|
|
239
|
+
conv_batch_size=128,
|
|
240
|
+
units_batch_size=8,
|
|
241
|
+
distance_kind="rms",
|
|
242
|
+
device=None,
|
|
243
|
+
n_jobs=0,
|
|
244
|
+
show_progress=True,
|
|
245
|
+
):
|
|
246
|
+
template_data, cross_mask, ids_a, ids_b = combine_templates(
|
|
247
|
+
template_data_a, template_data_b
|
|
248
|
+
)
|
|
249
|
+
units, dists, shifts, template_snrs = calculate_merge_distances(
|
|
250
|
+
template_data,
|
|
251
|
+
superres_linkage=superres_linkage,
|
|
252
|
+
sym_function=sym_function,
|
|
253
|
+
max_shift_samples=max_shift_samples,
|
|
254
|
+
temporal_upsampling_factor=temporal_upsampling_factor,
|
|
255
|
+
amplitude_scaling_variance=amplitude_scaling_variance,
|
|
256
|
+
amplitude_scaling_boundary=amplitude_scaling_boundary,
|
|
257
|
+
svd_compression_rank=svd_compression_rank,
|
|
258
|
+
min_channel_amplitude=min_channel_amplitude,
|
|
259
|
+
min_spatial_cosine=min_spatial_cosine,
|
|
260
|
+
spatial_radius_a=spatial_radius_a,
|
|
261
|
+
cooccurrence_mask=cross_mask,
|
|
262
|
+
conv_batch_size=conv_batch_size,
|
|
263
|
+
units_batch_size=units_batch_size,
|
|
264
|
+
distance_kind=distance_kind,
|
|
265
|
+
device=device,
|
|
266
|
+
n_jobs=n_jobs,
|
|
267
|
+
show_progress=show_progress,
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# symmetrize the merge distance matrix, which right now is block off-diagonal
|
|
271
|
+
# (with infs on main diag blocks)
|
|
272
|
+
a_mask = np.flatnonzero(np.isin(units, ids_a))
|
|
273
|
+
b_mask = np.flatnonzero(np.isin(units, ids_b))
|
|
274
|
+
Dab = dists[a_mask[:, None], b_mask[None, :]]
|
|
275
|
+
Dba = dists[b_mask[:, None], a_mask[None, :]]
|
|
276
|
+
Dstack = np.stack((Dab, Dba.T))
|
|
277
|
+
shifts_ab = shifts[a_mask[:, None], b_mask[None, :]]
|
|
278
|
+
shifts_ba = shifts[b_mask[:, None], a_mask[None, :]]
|
|
279
|
+
shifts_stack = np.stack((shifts_ab, -shifts_ba.T))
|
|
280
|
+
|
|
281
|
+
# choices[ia, ib] = 0 if Dab[ia, ib] < Dba[ib, ia], else 1
|
|
282
|
+
# but what does that mean? if 0, it means that the upsampled model
|
|
283
|
+
# of B was better able to match A than the other way around. it's not
|
|
284
|
+
# that important, at the end of the day. still, we keep track of it
|
|
285
|
+
# so that we can pick the shift and move on with our lives.
|
|
286
|
+
choices = np.argmin(Dstack, axis=0)
|
|
287
|
+
dists = Dstack[
|
|
288
|
+
choices, np.arange(choices.shape[0])[:, None], np.arange(choices.shape[1])[None]
|
|
289
|
+
]
|
|
290
|
+
shifts = shifts_stack[
|
|
291
|
+
choices, np.arange(choices.shape[0])[:, None], np.arange(choices.shape[1])[None]
|
|
292
|
+
]
|
|
293
|
+
|
|
294
|
+
# handle duplicates and missing
|
|
295
|
+
a_inds = np.searchsorted(units[a_mask], ids_a, side="right") - 1
|
|
296
|
+
a_kept = units[a_mask][a_inds] == ids_a
|
|
297
|
+
b_inds = np.searchsorted(units[b_mask], ids_b, side="right") - 1
|
|
298
|
+
b_kept = units[b_mask][b_inds] == ids_b
|
|
299
|
+
dists = dists[a_inds[a_kept][:, None], b_inds[b_kept][None, :]]
|
|
300
|
+
|
|
301
|
+
snrs_a = template_snrs[a_mask][a_inds[a_kept]]
|
|
302
|
+
snrs_b = template_snrs[b_mask][b_inds[b_kept]]
|
|
303
|
+
|
|
304
|
+
return (
|
|
305
|
+
dists,
|
|
306
|
+
shifts,
|
|
307
|
+
snrs_a,
|
|
308
|
+
snrs_b,
|
|
309
|
+
a_kept,
|
|
310
|
+
b_kept,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def recluster(
|
|
315
|
+
sorting,
|
|
316
|
+
units,
|
|
317
|
+
dists,
|
|
318
|
+
shifts,
|
|
319
|
+
template_snrs,
|
|
320
|
+
merge_distance_threshold=0.25,
|
|
321
|
+
link="complete",
|
|
322
|
+
):
|
|
323
|
+
# upper triangle not including diagonal, aka condensed distance matrix in scipy
|
|
324
|
+
pdist = dists[np.triu_indices(dists.shape[0], k=1)]
|
|
325
|
+
# scipy hierarchical clustering only supports finite values, so let's just
|
|
326
|
+
# drop in a huge value here
|
|
327
|
+
finite = np.isfinite(pdist)
|
|
328
|
+
if not finite.any():
|
|
329
|
+
return sorting, np.arange(dists.shape[0])
|
|
330
|
+
assert units.shape[0] == dists.shape[0] == dists.shape[1]
|
|
331
|
+
|
|
332
|
+
pdist[~finite] = 1_000_000 + pdist[finite].max()
|
|
333
|
+
Z = linkage(pdist, method=link)
|
|
334
|
+
# extract flat clustering using our max dist threshold
|
|
335
|
+
new_labels = fcluster(Z, merge_distance_threshold, criterion="distance")
|
|
336
|
+
assert new_labels.min() == 1 # start at 1 for some reason
|
|
337
|
+
new_labels -= 1
|
|
338
|
+
|
|
339
|
+
# update labels
|
|
340
|
+
labels_updated = np.full_like(sorting.labels, -1)
|
|
341
|
+
kept = np.flatnonzero(np.isin(sorting.labels, units))
|
|
342
|
+
labels_updated[kept] = new_labels[sorting.labels[kept]]
|
|
343
|
+
|
|
344
|
+
# update times according to shifts
|
|
345
|
+
times_updated = sorting.times_samples.copy()
|
|
346
|
+
|
|
347
|
+
# find original labels in each cluster
|
|
348
|
+
clust_inverse = {i: [] for i in new_labels}
|
|
349
|
+
for orig_label, new_label in enumerate(new_labels):
|
|
350
|
+
clust_inverse[new_label].append(orig_label)
|
|
351
|
+
n_merges = sum(len(v) - 1 for v in clust_inverse.values())
|
|
352
|
+
logger.dartsortdebug(f"Merged {n_merges} templates.")
|
|
353
|
+
|
|
354
|
+
# align to best snr unit
|
|
355
|
+
for new_label, orig_labels in clust_inverse.items():
|
|
356
|
+
# we don't need to realign clusters which didn't change
|
|
357
|
+
if len(orig_labels) <= 1:
|
|
358
|
+
continue
|
|
359
|
+
|
|
360
|
+
orig_snrs = template_snrs[orig_labels]
|
|
361
|
+
best_orig = orig_labels[orig_snrs.argmax()]
|
|
362
|
+
for ogl in np.setdiff1d(orig_labels, [best_orig]):
|
|
363
|
+
in_orig_unit = np.flatnonzero(sorting.labels == ogl)
|
|
364
|
+
# this is like trough[best] - trough[ogl]
|
|
365
|
+
shift_og_best = shifts[ogl, best_orig]
|
|
366
|
+
# if >0, trough of og is behind trough of best.
|
|
367
|
+
# subtracting will move trough of og to the right.
|
|
368
|
+
times_updated[in_orig_unit] -= shift_og_best
|
|
369
|
+
|
|
370
|
+
new_sorting = sorting.ephemeral_replace(
|
|
371
|
+
times_samples=times_updated, labels=labels_updated
|
|
372
|
+
)
|
|
373
|
+
return new_sorting, new_labels
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def get_deconv_resid_decrease_iter(
|
|
377
|
+
template_data,
|
|
378
|
+
max_shift_samples=40,
|
|
379
|
+
temporal_upsampling_factor=8,
|
|
380
|
+
amplitude_scaling_variance=0.001,
|
|
381
|
+
amplitude_scaling_boundary=0.1,
|
|
382
|
+
svd_compression_rank=20,
|
|
383
|
+
min_channel_amplitude=0.0,
|
|
384
|
+
min_spatial_cosine=0.0,
|
|
385
|
+
cooccurrence_mask=None,
|
|
386
|
+
conv_batch_size=128,
|
|
387
|
+
units_batch_size=8,
|
|
388
|
+
spatial_radius_a=None,
|
|
389
|
+
ignore_empty_channels=True,
|
|
390
|
+
distance_kind="rms",
|
|
391
|
+
device=None,
|
|
392
|
+
n_jobs=0,
|
|
393
|
+
show_progress=True,
|
|
394
|
+
):
|
|
395
|
+
# get template aux data
|
|
396
|
+
low_rank_templates_b = template_util.svd_compress_templates(
|
|
397
|
+
template_data,
|
|
398
|
+
min_channel_amplitude=min_channel_amplitude,
|
|
399
|
+
rank=svd_compression_rank,
|
|
400
|
+
)
|
|
401
|
+
compressed_upsampled_temporal = template_util.compressed_upsampled_templates(
|
|
402
|
+
low_rank_templates_b.temporal_components,
|
|
403
|
+
ptps=np.ptp(template_data.templates, 1).max(1),
|
|
404
|
+
max_upsample=temporal_upsampling_factor,
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
# restrict spatial subset of target templates
|
|
408
|
+
template_data_a = template_data_b = template_data
|
|
409
|
+
low_rank_templates_a = low_rank_templates_b
|
|
410
|
+
if spatial_radius_a:
|
|
411
|
+
template_data_a = template_util.spatially_mask_templates(
|
|
412
|
+
template_data, spatial_radius_a
|
|
413
|
+
)
|
|
414
|
+
low_rank_templates_a = template_util.svd_compress_templates(
|
|
415
|
+
template_data_a,
|
|
416
|
+
min_channel_amplitude=min_channel_amplitude,
|
|
417
|
+
rank=svd_compression_rank,
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
# construct helper data and run pairwise convolutions
|
|
421
|
+
(
|
|
422
|
+
template_shift_index_a,
|
|
423
|
+
template_shift_index_b,
|
|
424
|
+
upsampled_shifted_template_index,
|
|
425
|
+
cooccurrence,
|
|
426
|
+
) = construct_shift_indices(
|
|
427
|
+
None,
|
|
428
|
+
None,
|
|
429
|
+
template_data,
|
|
430
|
+
compressed_upsampled_temporal,
|
|
431
|
+
motion_est=None,
|
|
432
|
+
)
|
|
433
|
+
if cooccurrence_mask is not None:
|
|
434
|
+
cooccurrence = cooccurrence & cooccurrence_mask
|
|
435
|
+
yield from iterate_compressed_pairwise_convolutions(
|
|
436
|
+
template_data_a,
|
|
437
|
+
low_rank_templates_a,
|
|
438
|
+
template_data_b,
|
|
439
|
+
low_rank_templates_b,
|
|
440
|
+
compressed_upsampled_temporal,
|
|
441
|
+
template_shift_index_a,
|
|
442
|
+
template_shift_index_b,
|
|
443
|
+
cooccurrence,
|
|
444
|
+
upsampled_shifted_template_index,
|
|
445
|
+
do_shifting=False,
|
|
446
|
+
reduce_deconv_resid_decrease=True,
|
|
447
|
+
geom=template_data.registered_geom,
|
|
448
|
+
conv_ignore_threshold=0.0,
|
|
449
|
+
min_spatial_cosine=min_spatial_cosine,
|
|
450
|
+
coarse_approx_error_threshold=0.0,
|
|
451
|
+
amplitude_scaling_variance=amplitude_scaling_variance,
|
|
452
|
+
amplitude_scaling_boundary=amplitude_scaling_boundary,
|
|
453
|
+
ignore_empty_channels=ignore_empty_channels,
|
|
454
|
+
distance_kind=distance_kind,
|
|
455
|
+
max_shift=max_shift_samples, # type: ignore
|
|
456
|
+
conv_batch_size=conv_batch_size,
|
|
457
|
+
units_batch_size=units_batch_size,
|
|
458
|
+
device=device,
|
|
459
|
+
n_jobs=n_jobs,
|
|
460
|
+
show_progress=show_progress,
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def combine_templates(template_data_a, template_data_b):
|
|
465
|
+
rgeom = template_data_a.registered_geom
|
|
466
|
+
if rgeom is not None:
|
|
467
|
+
if not np.array_equal(rgeom, template_data_b.registered_geom):
|
|
468
|
+
raise ValueError(
|
|
469
|
+
f"Template data had different registered geoms: "
|
|
470
|
+
f"{template_data_a.registered_geom=} {template_data_b.registered_geom=}"
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
ids_a = template_data_a.unit_ids
|
|
474
|
+
ids_b = template_data_b.unit_ids + ids_a.max() + 1
|
|
475
|
+
unit_ids = np.concatenate((ids_a, ids_b))
|
|
476
|
+
templates = np.row_stack((template_data_a.templates, template_data_b.templates))
|
|
477
|
+
spike_counts = np.concatenate(
|
|
478
|
+
(template_data_a.spike_counts, template_data_b.spike_counts)
|
|
479
|
+
)
|
|
480
|
+
spike_counts_by_channel = None
|
|
481
|
+
if template_data_a.spike_counts_by_channel is not None:
|
|
482
|
+
spike_counts_by_channel = np.concatenate(
|
|
483
|
+
(
|
|
484
|
+
template_data_a.spike_counts_by_channel,
|
|
485
|
+
template_data_b.spike_counts_by_channel,
|
|
486
|
+
)
|
|
487
|
+
)
|
|
488
|
+
template_data = TemplateData(
|
|
489
|
+
templates=templates,
|
|
490
|
+
unit_ids=unit_ids,
|
|
491
|
+
spike_counts=spike_counts,
|
|
492
|
+
registered_geom=rgeom,
|
|
493
|
+
spike_counts_by_channel=spike_counts_by_channel,
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
cross_mask = np.logical_and(
|
|
497
|
+
np.isin(unit_ids, ids_a)[:, None], np.isin(unit_ids, ids_b)[None]
|
|
498
|
+
)
|
|
499
|
+
cross_mask = np.logical_or(cross_mask, cross_mask.T)
|
|
500
|
+
|
|
501
|
+
return template_data, cross_mask, ids_a, ids_b
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from scipy.signal import find_peaks
|
|
3
|
+
from scipy.stats import norm
|
|
4
|
+
|
|
5
|
+
from . import density
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from isosplit import up_down_isotonic_regression, jisotonic5 # type: ignore[reportMissingImports]
|
|
9
|
+
except ImportError:
|
|
10
|
+
up_down_isotonic_regression = lambda *a, **k: None
|
|
11
|
+
jisotonic5 = None
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
# todo: replace all isosplit stuff with things based on scipy's isotonic regression.
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def fit_unimodal_right(x, f, weights=None, cut=0, hard=False):
|
|
18
|
+
"""Unimodal to the right of cut, increasing to the left. Continuity at the border."""
|
|
19
|
+
assert jisotonic5 is not None
|
|
20
|
+
if weights is None:
|
|
21
|
+
weights = np.ones_like(f)
|
|
22
|
+
|
|
23
|
+
# figure out where cut lands and what f(cut) should be
|
|
24
|
+
cuti_left = np.searchsorted(x, cut)
|
|
25
|
+
if cuti_left >= len(x) - 1:
|
|
26
|
+
# everything is to the left... fit increasing!
|
|
27
|
+
return jisotonic5.jisotonic5(f, weights=weights)[0]
|
|
28
|
+
cuti_right = np.searchsorted(x, cut, side="right") - 1
|
|
29
|
+
if cuti_right <= 0:
|
|
30
|
+
# everything is to the right... fit unimodal!
|
|
31
|
+
return up_down_isotonic_regression(f, weights=weights)
|
|
32
|
+
# else, in the middle somewhere...
|
|
33
|
+
assert cuti_left in (cuti_right, cuti_right + 1)
|
|
34
|
+
|
|
35
|
+
out = np.empty_like(f)
|
|
36
|
+
out[cuti_right:] = up_down_isotonic_regression(
|
|
37
|
+
np.ascontiguousarray(f[cuti_right:]),
|
|
38
|
+
weights=np.ascontiguousarray(weights[cuti_right:]),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# impose continuity with a large penalty
|
|
42
|
+
f_left = np.concatenate((f[:cuti_left], out[cuti_right : cuti_right + 1]))
|
|
43
|
+
w_left = np.concatenate(
|
|
44
|
+
(weights[:cuti_left], 1e6 + 10 * weights.sum(keepdims=True))
|
|
45
|
+
)
|
|
46
|
+
out[:cuti_left] = jisotonic5.jisotonic5(f_left, w_left)[0][:-1]
|
|
47
|
+
|
|
48
|
+
return out
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def fit_truncnorm_right(x, f, weights=None, cut=0, hard=False, n_iter=10):
|
|
52
|
+
"""Like above, but fits truncated normal to xs > cut with MoM."""
|
|
53
|
+
if weights is None:
|
|
54
|
+
weights = np.ones_like(f)
|
|
55
|
+
|
|
56
|
+
# figure out where cut lands and what f(cut) should be
|
|
57
|
+
cuti_left = np.searchsorted(x, cut)
|
|
58
|
+
if cuti_left >= len(x) - 2:
|
|
59
|
+
# everything is to the left... return uniform
|
|
60
|
+
return np.full_like(f, 1 / len(f))
|
|
61
|
+
cuti_right = np.searchsorted(x, cut, side="right") - 1
|
|
62
|
+
if cuti_right <= 1:
|
|
63
|
+
# everything is to the right... fit normal!
|
|
64
|
+
mean = np.average(x, weights=weights)
|
|
65
|
+
var = np.average((x - mean) ** 2, weights=weights)
|
|
66
|
+
return norm.pdf(x, loc=mean, scale=np.sqrt(var))
|
|
67
|
+
# else, in the middle somewhere...
|
|
68
|
+
assert cuti_left in (cuti_right, cuti_right + 1)
|
|
69
|
+
|
|
70
|
+
xcut = x[cuti_right:]
|
|
71
|
+
mean = np.average(xcut, weights=weights[cuti_right:])
|
|
72
|
+
var = np.average((xcut - mean) ** 2, weights=weights[cuti_right:])
|
|
73
|
+
mu = mean
|
|
74
|
+
sigma = std = np.sqrt(var)
|
|
75
|
+
for i in range(n_iter):
|
|
76
|
+
alpha = (cut - mu) / sigma
|
|
77
|
+
phi_alpha = norm.cdf(alpha)
|
|
78
|
+
Z = 1.0 - phi_alpha
|
|
79
|
+
# we have: mean ~ mu + sigma*phi(alpha)/Z
|
|
80
|
+
# and var ~ sigma^2(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
|
|
81
|
+
# implies mu = mean - sigma*phi(alpha)/Z
|
|
82
|
+
# sigma^2 = var/(1+alpha phi(alpha) /Z - (phi(alpha)/Z)^2)
|
|
83
|
+
sigma = np.sqrt(var / (1 + alpha * phi_alpha / Z - (phi_alpha / Z) ** 2))
|
|
84
|
+
mu = mean - sigma * phi_alpha / Z
|
|
85
|
+
|
|
86
|
+
return norm.pdf(x, loc=mu, scale=sigma)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def fit_bimodal_at(x, f, weights=None, cut=0):
|
|
90
|
+
from isosplit import up_down_isotonic_regression # type: ignore
|
|
91
|
+
|
|
92
|
+
if weights is None:
|
|
93
|
+
weights = np.ones_like(f)
|
|
94
|
+
which_right = x > cut
|
|
95
|
+
which_left = np.logical_not(which_right)
|
|
96
|
+
out = np.empty_like(f)
|
|
97
|
+
if which_left.any():
|
|
98
|
+
out[which_left] = up_down_isotonic_regression(
|
|
99
|
+
f[which_left], weights=weights[which_left]
|
|
100
|
+
)
|
|
101
|
+
if which_right.any():
|
|
102
|
+
out[which_right] = up_down_isotonic_regression(
|
|
103
|
+
f[which_right], weights=weights[which_right]
|
|
104
|
+
)
|
|
105
|
+
return out
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def smoothed_dipscore_at(
|
|
109
|
+
cut,
|
|
110
|
+
samples,
|
|
111
|
+
alternative="smoothed",
|
|
112
|
+
dipscore_only=False,
|
|
113
|
+
score_kind="tv",
|
|
114
|
+
cut_relmax_order=3,
|
|
115
|
+
null="isotoniconesideunnormed",
|
|
116
|
+
debug_info=None,
|
|
117
|
+
):
|
|
118
|
+
if samples.size < 3:
|
|
119
|
+
return 0.0
|
|
120
|
+
|
|
121
|
+
densities = density.get_smoothed_density(
|
|
122
|
+
samples[:, None],
|
|
123
|
+
sigma=0.5 * 1.06 * samples.std() * (samples.size**-0.2),
|
|
124
|
+
bin_size_ratio=20.0,
|
|
125
|
+
min_bin_size=0.025,
|
|
126
|
+
max_n_bins=512,
|
|
127
|
+
min_n_bins=10,
|
|
128
|
+
)
|
|
129
|
+
spacings = np.diff(samples)
|
|
130
|
+
spacings = np.concatenate(
|
|
131
|
+
(spacings[:1], 0.5 * (spacings[1:] + spacings[:-1]), spacings[-1:])
|
|
132
|
+
)
|
|
133
|
+
with np.errstate(all="raise"):
|
|
134
|
+
densities /= (densities * spacings).sum()
|
|
135
|
+
|
|
136
|
+
if cut is None:
|
|
137
|
+
# closest maxes left + right of 0
|
|
138
|
+
maxers, _ = find_peaks(densities, distance=cut_relmax_order)
|
|
139
|
+
coords = samples[maxers]
|
|
140
|
+
left_cut = right_cut = 0
|
|
141
|
+
if (coords < 0).any():
|
|
142
|
+
left_cut = coords[coords < 0].max()
|
|
143
|
+
if (coords > 0).any():
|
|
144
|
+
right_cut = coords[coords > 0].min()
|
|
145
|
+
candidates = np.logical_and(samples >= left_cut, samples <= right_cut)
|
|
146
|
+
cut = 0
|
|
147
|
+
if candidates.any():
|
|
148
|
+
cut = samples[candidates][np.argmin(densities[candidates])]
|
|
149
|
+
|
|
150
|
+
if alternative == "bimodal":
|
|
151
|
+
densities = fit_bimodal_at(samples, densities, cut=cut)
|
|
152
|
+
densities /= (densities * spacings).sum()
|
|
153
|
+
else:
|
|
154
|
+
assert alternative == "smoothed"
|
|
155
|
+
|
|
156
|
+
score = 0
|
|
157
|
+
best_dens_err = np.inf
|
|
158
|
+
best_uni = None
|
|
159
|
+
|
|
160
|
+
if score_kind == "ks":
|
|
161
|
+
empirical = np.cumsum(densities * spacings)
|
|
162
|
+
|
|
163
|
+
if null in ("isotonic", "gcm"):
|
|
164
|
+
orders = (slice(None),)
|
|
165
|
+
signs = (1,)
|
|
166
|
+
else:
|
|
167
|
+
orders = (slice(None), slice(None, None, -1))
|
|
168
|
+
signs = (1, -1)
|
|
169
|
+
|
|
170
|
+
# change to density which is best overall fit
|
|
171
|
+
for hard in (False, True):
|
|
172
|
+
for order, sign in zip(orders, signs):
|
|
173
|
+
s = np.ascontiguousarray(samples[order])
|
|
174
|
+
d = np.ascontiguousarray(densities[order])
|
|
175
|
+
if null in ("isotoniconeside", "isotoniconesideunnormed"):
|
|
176
|
+
dens = fit_unimodal_right(sign * s, d, cut=sign * cut, hard=hard)
|
|
177
|
+
elif null == "isotonic":
|
|
178
|
+
dens = up_down_isotonic_regression(d)
|
|
179
|
+
elif null == "truncnorm":
|
|
180
|
+
dens = fit_truncnorm_right(sign * s, d, cut=sign * cut, hard=hard)
|
|
181
|
+
else:
|
|
182
|
+
assert False
|
|
183
|
+
|
|
184
|
+
dens = dens[order] # type: ignore
|
|
185
|
+
if null not in ("isotonic", "isotoniconesideunnormed"):
|
|
186
|
+
dens /= (dens * spacings).sum()
|
|
187
|
+
|
|
188
|
+
dens_err = (np.abs(dens - densities) * spacings).sum()
|
|
189
|
+
if score_kind == "ks":
|
|
190
|
+
my_score = np.abs(empirical - np.cumsum(dens * spacings)).max() # type: ignore
|
|
191
|
+
elif score_kind == "tv":
|
|
192
|
+
my_score = 0.5 * dens_err
|
|
193
|
+
else:
|
|
194
|
+
assert False
|
|
195
|
+
|
|
196
|
+
# if my_score > score:
|
|
197
|
+
if dens_err < best_dens_err:
|
|
198
|
+
score = my_score
|
|
199
|
+
best_dens_err = dens_err
|
|
200
|
+
best_uni = dens
|
|
201
|
+
|
|
202
|
+
if debug_info is not None:
|
|
203
|
+
debug_info["domain"] = samples
|
|
204
|
+
debug_info["alternative_density"] = densities
|
|
205
|
+
debug_info["cut"] = cut
|
|
206
|
+
debug_info["score"] = score
|
|
207
|
+
debug_info["score_kind"] = score_kind
|
|
208
|
+
debug_info["uni_density"] = best_uni
|
|
209
|
+
debug_info["samples"] = samples
|
|
210
|
+
|
|
211
|
+
if dipscore_only:
|
|
212
|
+
return score
|
|
213
|
+
|
|
214
|
+
return score, samples, densities, best_uni, cut
|