DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
from dataclasses import replace
|
|
2
|
+
import gc
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
import h5py
|
|
6
|
+
|
|
7
|
+
from ..util.internal_config import default_refinement_cfg
|
|
8
|
+
from ..util import job_util, noise_util, data_util, spiketorch
|
|
9
|
+
from ..util.logging_util import get_logger
|
|
10
|
+
from ..util.main_util import ds_save_intermediate_labels
|
|
11
|
+
from ..transform.temporal_pca import BaseTemporalPCA
|
|
12
|
+
from ..templates import TemplateData
|
|
13
|
+
from .cluster_util import agglomerate, reorder_by_depth
|
|
14
|
+
from .split import split_clusters
|
|
15
|
+
from .merge import merge_templates
|
|
16
|
+
from .gmm.stable_features import StableSpikeDataset
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
logger = get_logger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def split_merge(
|
|
23
|
+
*,
|
|
24
|
+
recording,
|
|
25
|
+
sorting,
|
|
26
|
+
motion_est=None,
|
|
27
|
+
split_cfg,
|
|
28
|
+
merge_cfg,
|
|
29
|
+
merge_template_cfg,
|
|
30
|
+
computation_cfg=None,
|
|
31
|
+
):
|
|
32
|
+
if computation_cfg is None:
|
|
33
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
34
|
+
|
|
35
|
+
split_sorting = split_clusters(
|
|
36
|
+
sorting,
|
|
37
|
+
split_strategy=split_cfg.split_strategy,
|
|
38
|
+
recursive=split_cfg.recursive_split,
|
|
39
|
+
n_jobs=computation_cfg.actual_n_jobs(),
|
|
40
|
+
motion_est=motion_est,
|
|
41
|
+
)
|
|
42
|
+
template_data = TemplateData.from_config(
|
|
43
|
+
recording=recording,
|
|
44
|
+
sorting=sorting,
|
|
45
|
+
motion_est=motion_est,
|
|
46
|
+
template_cfg=merge_template_cfg,
|
|
47
|
+
)
|
|
48
|
+
merge_sorting = merge_templates(
|
|
49
|
+
sorting=split_sorting,
|
|
50
|
+
template_data=template_data,
|
|
51
|
+
merge_distance_threshold=merge_cfg.merge_distance_threshold,
|
|
52
|
+
min_spatial_cosine=merge_cfg.min_spatial_cosine,
|
|
53
|
+
linkage=merge_cfg.linkage,
|
|
54
|
+
computation_cfg=computation_cfg,
|
|
55
|
+
)["sorting"]
|
|
56
|
+
|
|
57
|
+
return merge_sorting
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_noise_log_priors(noise, sorting, refinement_cfg):
|
|
61
|
+
from dartsort.templates import TemplateData
|
|
62
|
+
|
|
63
|
+
if not refinement_cfg.noise_fp_correction:
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
h5_name = sorting.parent_h5_path
|
|
67
|
+
if h5_name is None:
|
|
68
|
+
return None
|
|
69
|
+
stem = h5_name.stem
|
|
70
|
+
|
|
71
|
+
if stem.startswith("matching"):
|
|
72
|
+
model_dir = h5_name.parent / f"{stem}_models"
|
|
73
|
+
templates_npz = model_dir / "template_data.npz"
|
|
74
|
+
if not templates_npz.exists():
|
|
75
|
+
raise ValueError(f"{templates_npz} is not there?")
|
|
76
|
+
|
|
77
|
+
with h5py.File(h5_name, "r", locking=False) as h5:
|
|
78
|
+
matching_labels = h5["labels"][:] # type: ignore
|
|
79
|
+
|
|
80
|
+
template_data = TemplateData.from_npz(templates_npz)
|
|
81
|
+
tpca = data_util.get_tpca(sorting)
|
|
82
|
+
assert isinstance(tpca, BaseTemporalPCA)
|
|
83
|
+
if (sl := getattr(tpca, "temporal_slice", None)) is not None:
|
|
84
|
+
temps_tpca = torch.asarray(template_data.templates[:, sl])
|
|
85
|
+
else:
|
|
86
|
+
temps_tpca = torch.asarray(template_data.templates)
|
|
87
|
+
|
|
88
|
+
n, t, c = temps_tpca.shape
|
|
89
|
+
temps_tpca = temps_tpca.permute(0, 2, 1).reshape(n * c, t)
|
|
90
|
+
temps_tpca = tpca._transform_in_probe(temps_tpca)
|
|
91
|
+
temps_tpca = temps_tpca.reshape(n, c, -1).permute(0, 2, 1)
|
|
92
|
+
|
|
93
|
+
noise_log_priors = noise.detection_prior_log_prob(temps_tpca)
|
|
94
|
+
logger.dartsortdebug(
|
|
95
|
+
f"Got log priors ranging {noise_log_priors.min()}-{noise_log_priors.max()}."
|
|
96
|
+
)
|
|
97
|
+
noise_log_priors = noise_log_priors[matching_labels]
|
|
98
|
+
|
|
99
|
+
return noise_log_priors
|
|
100
|
+
elif stem.startswith("subtract"):
|
|
101
|
+
noise_log_priors = noise.channelwise_detection_prior_log_prob()
|
|
102
|
+
logger.dartsortdebug(
|
|
103
|
+
f"Got log priors ranging {noise_log_priors.min()}-{noise_log_priors.max()}."
|
|
104
|
+
)
|
|
105
|
+
noise_log_priors = noise_log_priors[sorting.channels]
|
|
106
|
+
return noise_log_priors
|
|
107
|
+
else:
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def pc_merge(sorting, refinement_cfg, motion_est=None, computation_cfg=None):
|
|
112
|
+
assert refinement_cfg.refinement_strategy == "pcmerge"
|
|
113
|
+
if computation_cfg is None:
|
|
114
|
+
computation_cfg = job_util.get_global_computation_config()
|
|
115
|
+
if not refinement_cfg.pc_merge_threshold:
|
|
116
|
+
return sorting
|
|
117
|
+
|
|
118
|
+
# remove blank labels just in case
|
|
119
|
+
sorting = data_util.subset_sorting_by_spike_count(sorting)
|
|
120
|
+
nu0 = sorting.labels.max() + 1
|
|
121
|
+
if not nu0:
|
|
122
|
+
return sorting
|
|
123
|
+
|
|
124
|
+
# subset the sorting to count per unit
|
|
125
|
+
subset_sorting = data_util.subsample_to_max_count(
|
|
126
|
+
sorting, max_spikes=refinement_cfg.pc_merge_spikes_per_unit
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# make stable features, no need for core features though.
|
|
130
|
+
data = StableSpikeDataset.from_sorting(
|
|
131
|
+
subset_sorting,
|
|
132
|
+
motion_est=motion_est,
|
|
133
|
+
core_radius=None,
|
|
134
|
+
discard_triaged=True,
|
|
135
|
+
interp_params=refinement_cfg.interp_params.normalize(),
|
|
136
|
+
split_proportions=None,
|
|
137
|
+
split_names=("train",),
|
|
138
|
+
device=computation_cfg.actual_device(),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# average by unit
|
|
142
|
+
kept = data.kept_indices
|
|
143
|
+
n = len(kept)
|
|
144
|
+
x = data._train_extract_features.view(n, -1, data.n_channels_extract)
|
|
145
|
+
x = x[:, : refinement_cfg.pc_merge_rank]
|
|
146
|
+
labels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
|
|
147
|
+
means, counts = spiketorch.average_by_label(
|
|
148
|
+
x, labels, data._train_extract_channels, data.n_channels
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# compute distances
|
|
152
|
+
if refinement_cfg.pc_merge_metric == "cosine":
|
|
153
|
+
dists = spiketorch.cosine_distance(means)
|
|
154
|
+
elif refinement_cfg.pc_merge_metric == "maxz":
|
|
155
|
+
x = x.square_()
|
|
156
|
+
meansq, _ = spiketorch.average_by_label(
|
|
157
|
+
x, labels, data._train_extract_channels, data.n_channels
|
|
158
|
+
)
|
|
159
|
+
stddev = meansq.sub_(means.square()).sqrt_()
|
|
160
|
+
stddev = stddev.clamp_(min=torch.finfo(stddev.dtype).tiny)
|
|
161
|
+
stderr = stddev.div_(counts.sqrt()[:, None])
|
|
162
|
+
dists = spiketorch.maxz_distance(
|
|
163
|
+
means, stderr, counts, min_iou=refinement_cfg.pc_merge_min_iou
|
|
164
|
+
)
|
|
165
|
+
elif refinement_cfg.pc_merge_metric == "normeuc":
|
|
166
|
+
dists = spiketorch.weighted_normeuc_distance(
|
|
167
|
+
means, counts, min_iou=refinement_cfg.pc_merge_min_iou
|
|
168
|
+
)
|
|
169
|
+
elif refinement_cfg.pc_merge_metric == "normsup":
|
|
170
|
+
dists = spiketorch.weighted_normsup_distance(
|
|
171
|
+
means, counts, min_iou=refinement_cfg.pc_merge_min_iou
|
|
172
|
+
)
|
|
173
|
+
elif refinement_cfg.pc_merge_metric == "euclidean":
|
|
174
|
+
means = means.reshape(len(means), -1)
|
|
175
|
+
dists = torch.cdist(means, means).numpy(force=True)
|
|
176
|
+
else:
|
|
177
|
+
raise ValueError(f"Have not implemented {refinement_cfg.pc_merge_metric=}.")
|
|
178
|
+
|
|
179
|
+
# linkage
|
|
180
|
+
labels, ids = agglomerate(
|
|
181
|
+
sorting.labels,
|
|
182
|
+
dists,
|
|
183
|
+
linkage_method=refinement_cfg.pc_merge_linkage,
|
|
184
|
+
threshold=refinement_cfg.pc_merge_threshold,
|
|
185
|
+
)
|
|
186
|
+
logger.dartsortdebug(f"pc_merge: Unit count {nu0}->{ids.max() + 1}.")
|
|
187
|
+
|
|
188
|
+
sorting = sorting.ephemeral_replace(labels=labels)
|
|
189
|
+
sorting = reorder_by_depth(sorting, motion_est=motion_est)
|
|
190
|
+
return sorting
|
|
191
|
+
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Computing relocated waveforms used in clustering
|
|
2
|
+
|
|
3
|
+
These waveforms are first shifted by an integer number of
|
|
4
|
+
probe pitches to account for gross drift and then rescaled
|
|
5
|
+
on each channel according to the point source model to
|
|
6
|
+
account for sub-pitch motion.
|
|
7
|
+
|
|
8
|
+
This is in numpy, since it's part of the clustering which
|
|
9
|
+
is all just numpy, and that's why it lives here and not
|
|
10
|
+
in transform/ which has torch-based stuff for use during
|
|
11
|
+
peeling.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
from dartsort.util import drift_util
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def relocated_waveforms_on_static_channels(
|
|
19
|
+
waveforms,
|
|
20
|
+
main_channels,
|
|
21
|
+
channel_index,
|
|
22
|
+
target_channels,
|
|
23
|
+
xyza_from,
|
|
24
|
+
z_to,
|
|
25
|
+
geom,
|
|
26
|
+
amplitude_vectors=None,
|
|
27
|
+
registered_geom=None,
|
|
28
|
+
target_kdtree=None,
|
|
29
|
+
match_distance=None,
|
|
30
|
+
fill_value=np.nan,
|
|
31
|
+
):
|
|
32
|
+
"""Compute relocated waveforms"""
|
|
33
|
+
two_d = waveforms.ndim == 2
|
|
34
|
+
if two_d:
|
|
35
|
+
waveforms = waveforms[:, None, :]
|
|
36
|
+
x, y, z_from = xyza_from[:, :3].T
|
|
37
|
+
if xyza_from.shape[1] == 4:
|
|
38
|
+
alpha = xyza_from[:, 3]
|
|
39
|
+
elif xyza_from.shape[1] == 3:
|
|
40
|
+
alpha = determine_alpha(
|
|
41
|
+
amplitude_vectors, x, y, z_from, geom, channels=channel_index[main_channels]
|
|
42
|
+
)
|
|
43
|
+
else:
|
|
44
|
+
assert False
|
|
45
|
+
if registered_geom is None:
|
|
46
|
+
registered_geom = geom
|
|
47
|
+
|
|
48
|
+
# -- handle coarse drift (larger than a pitch)
|
|
49
|
+
n_pitches_shift = drift_util.get_spike_pitch_shifts(
|
|
50
|
+
z_from, geom, registered_depths_um=z_to
|
|
51
|
+
)
|
|
52
|
+
shifted_waveforms = drift_util.get_waveforms_on_static_channels(
|
|
53
|
+
waveforms,
|
|
54
|
+
main_channels=main_channels,
|
|
55
|
+
channel_index=channel_index,
|
|
56
|
+
target_channels=target_channels,
|
|
57
|
+
n_pitches_shift=n_pitches_shift,
|
|
58
|
+
geom=geom,
|
|
59
|
+
registered_geom=registered_geom,
|
|
60
|
+
target_kdtree=target_kdtree,
|
|
61
|
+
match_distance=match_distance,
|
|
62
|
+
fill_value=fill_value,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# -- handle sub-pitch drift
|
|
66
|
+
# account for the already applied coarse drift correction
|
|
67
|
+
pitch = drift_util.get_pitch(geom)
|
|
68
|
+
z_from = z_from + pitch * n_pitches_shift
|
|
69
|
+
original_amplitudes = point_source_amplitude_vectors(
|
|
70
|
+
x, y, z_from, alpha, registered_geom, channels=target_channels
|
|
71
|
+
)
|
|
72
|
+
target_amplitudes = point_source_amplitude_vectors(
|
|
73
|
+
x, y, z_to, alpha, registered_geom, channels=target_channels
|
|
74
|
+
)
|
|
75
|
+
rescaling = target_amplitudes / original_amplitudes
|
|
76
|
+
shifted_waveforms *= rescaling[:, None, :]
|
|
77
|
+
|
|
78
|
+
if two_d:
|
|
79
|
+
shifted_waveforms = shifted_waveforms[:, 0, :]
|
|
80
|
+
|
|
81
|
+
return shifted_waveforms
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def point_source_amplitude_vectors(x, y, z_abs, alpha, geom, channels=None):
|
|
85
|
+
"""Point-source amplitudes on fixed channels from varying spike positions"""
|
|
86
|
+
if channels is not None:
|
|
87
|
+
geom = geom[channels]
|
|
88
|
+
if geom.ndim == 2:
|
|
89
|
+
geom = geom[None]
|
|
90
|
+
|
|
91
|
+
dx = geom[..., 0] - x[:, None]
|
|
92
|
+
dz = geom[..., 1] - z_abs[:, None]
|
|
93
|
+
dy = y[:, None]
|
|
94
|
+
denom = np.sqrt(dy * dy + dx * dx + dz * dz)
|
|
95
|
+
denom[denom < 1e-8] = 1
|
|
96
|
+
|
|
97
|
+
return alpha[:, None] / denom
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def determine_alpha(ampvecs, x, y, z, geom, channels):
|
|
101
|
+
geom = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
|
|
102
|
+
mask = (channels < len(geom)).astype(ampvecs.dtype)
|
|
103
|
+
preds = point_source_amplitude_vectors(
|
|
104
|
+
x, y, z, np.ones_like(z), geom, channels=channels
|
|
105
|
+
)
|
|
106
|
+
alpha = np.nan_to_num(ampvecs * preds * mask).sum(axis=1) / np.nan_to_num(
|
|
107
|
+
ampvecs * ampvecs * mask
|
|
108
|
+
).sum(axis=1)
|
|
109
|
+
return alpha
|