DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
|
|
3
|
+
from . import detect, util
|
|
4
|
+
from .clustering import (
|
|
5
|
+
cluster_util,
|
|
6
|
+
clustering_strategies,
|
|
7
|
+
density,
|
|
8
|
+
get_clusterer,
|
|
9
|
+
get_clustering_features,
|
|
10
|
+
kmeans,
|
|
11
|
+
merge,
|
|
12
|
+
refinement_strategies,
|
|
13
|
+
)
|
|
14
|
+
from .config import *
|
|
15
|
+
from .evaluate import config_grid, hybrid_util, simkit, simlib
|
|
16
|
+
from .evaluate.analysis import DARTsortAnalysis
|
|
17
|
+
from .evaluate.comparison import DARTsortGroundTruthComparison, DARTsortGTVersus
|
|
18
|
+
from .localize.localize_util import (
|
|
19
|
+
localize_amplitude_vectors,
|
|
20
|
+
localize_hdf5,
|
|
21
|
+
localize_waveforms,
|
|
22
|
+
)
|
|
23
|
+
from .main import (
|
|
24
|
+
ObjectiveUpdateTemplateMatchingPeeler,
|
|
25
|
+
SubtractionPeeler,
|
|
26
|
+
check_recording,
|
|
27
|
+
cluster,
|
|
28
|
+
dartsort,
|
|
29
|
+
estimate_motion,
|
|
30
|
+
grab,
|
|
31
|
+
logger,
|
|
32
|
+
match,
|
|
33
|
+
run_peeler,
|
|
34
|
+
subtract,
|
|
35
|
+
threshold,
|
|
36
|
+
universal_match,
|
|
37
|
+
)
|
|
38
|
+
from .peel.grab import GrabAndFeaturize
|
|
39
|
+
from .templates import (
|
|
40
|
+
TemplateData,
|
|
41
|
+
estimate_template_library,
|
|
42
|
+
postprocess_util,
|
|
43
|
+
realign,
|
|
44
|
+
)
|
|
45
|
+
from .transform import WaveformPipeline
|
|
46
|
+
from .util import data_util, logging_util, noise_util, spiketorch, waveform_util
|
|
47
|
+
from .util.data_util import (
|
|
48
|
+
DARTsortSorting,
|
|
49
|
+
get_featurization_pipeline,
|
|
50
|
+
load_h5,
|
|
51
|
+
get_tpca,
|
|
52
|
+
load_stored_tsvd,
|
|
53
|
+
check_recording,
|
|
54
|
+
)
|
|
55
|
+
from .util.drift_util import registered_geometry
|
|
56
|
+
from .util.internal_config import *
|
|
57
|
+
from .util.logging_util import (
|
|
58
|
+
DARTSORTDEBUG,
|
|
59
|
+
DARTSORTVERBOSE,
|
|
60
|
+
DARTsortLogger,
|
|
61
|
+
get_logger,
|
|
62
|
+
)
|
|
63
|
+
from .util.noise_util import EmbeddedNoise
|
|
64
|
+
from .util.py_util import resolve_path
|
|
65
|
+
from .util.registration_util import try_load_motion_est
|
|
66
|
+
from .util.waveform_util import make_channel_index
|
|
67
|
+
|
|
68
|
+
__version__ = importlib.metadata.version("dartsort")
|
dartsort/cli.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import spikeinterface.core as sc
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from .util import cli_util
|
|
6
|
+
from . import config, main
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def dartsort_cli():
|
|
10
|
+
"""dartsort's CLI, invoked by `dartsort` at the terminal.
|
|
11
|
+
|
|
12
|
+
Try `dartsort --help` to start.
|
|
13
|
+
|
|
14
|
+
---<!!> Not stable.
|
|
15
|
+
|
|
16
|
+
I am figuring out how to do preprocessing still. It may be configured?
|
|
17
|
+
"""
|
|
18
|
+
# -- define CLI
|
|
19
|
+
assert dartsort_cli.__doc__ is not None
|
|
20
|
+
ap = argparse.ArgumentParser(
|
|
21
|
+
prog="dartsort",
|
|
22
|
+
epilog=dartsort_cli.__doc__.split("---")[1],
|
|
23
|
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
24
|
+
)
|
|
25
|
+
ap.add_argument("recording", help="Path to SpikeInterface RecordingExtractor.")
|
|
26
|
+
ap.add_argument(
|
|
27
|
+
"output_dir",
|
|
28
|
+
nargs="?",
|
|
29
|
+
help="Folder where outputs will be saved. If this is unset, then "
|
|
30
|
+
"--config-toml must be supplied, and the output folder will be the one where "
|
|
31
|
+
"that configuration lives. If both are supplied, output_dir will be "
|
|
32
|
+
"used (not the config file's parent dir).",
|
|
33
|
+
)
|
|
34
|
+
ap.add_argument(
|
|
35
|
+
"--config-toml",
|
|
36
|
+
type=str,
|
|
37
|
+
default=None,
|
|
38
|
+
help="Path to configuration in TOML format. Arguments passed on the "
|
|
39
|
+
"command line will override their values in the TOML file.",
|
|
40
|
+
)
|
|
41
|
+
ap.add_argument("--overwrite", action="store_true")
|
|
42
|
+
ap.add_argument(
|
|
43
|
+
"--loglevel",
|
|
44
|
+
default="WARNING",
|
|
45
|
+
type=str.upper,
|
|
46
|
+
choices=logging._nameToLevel.keys(),
|
|
47
|
+
)
|
|
48
|
+
# user-facing API
|
|
49
|
+
cli_util.dataclass_to_argparse(config.DARTsortUserConfig, parser=ap)
|
|
50
|
+
|
|
51
|
+
# super secret developer-only args
|
|
52
|
+
dev_args = ap.add_argument_group("Secret development flags ($1.50 fee to use)")
|
|
53
|
+
cli_util.dataclass_to_argparse(
|
|
54
|
+
config.DeveloperConfig,
|
|
55
|
+
parser=dev_args,
|
|
56
|
+
skipnames=cli_util.fieldnames(config.DARTsortUserConfig),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# -- parse args
|
|
60
|
+
args = ap.parse_args()
|
|
61
|
+
|
|
62
|
+
# set up logging
|
|
63
|
+
logging.basicConfig(level=args.loglevel)
|
|
64
|
+
|
|
65
|
+
# check if we have config file
|
|
66
|
+
config_toml = None
|
|
67
|
+
if args.config_toml:
|
|
68
|
+
config_toml = cli_util.ensurepath(args.config_toml)
|
|
69
|
+
|
|
70
|
+
# determine output directory
|
|
71
|
+
if args.output_dir:
|
|
72
|
+
output_dir = cli_util.ensurepath(args.output_dir, strict=False)
|
|
73
|
+
elif config_toml is None:
|
|
74
|
+
print(f"No output directory given, exiting. See `{ap.prog} -h`.")
|
|
75
|
+
return 1
|
|
76
|
+
else:
|
|
77
|
+
output_dir = config_toml.parent
|
|
78
|
+
|
|
79
|
+
# determine the config from the command line args
|
|
80
|
+
cfg = cli_util.combine_toml_and_argv(
|
|
81
|
+
(config.DARTsortUserConfig, config.DeveloperConfig),
|
|
82
|
+
config.DeveloperConfig,
|
|
83
|
+
config_toml,
|
|
84
|
+
args,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# load the recording
|
|
88
|
+
# TODO: preprocessing management
|
|
89
|
+
try:
|
|
90
|
+
rec = sc.load(cli_util.ensurepath(args.recording))
|
|
91
|
+
assert isinstance(rec, sc.BaseRecording)
|
|
92
|
+
except FileNotFoundError as e:
|
|
93
|
+
ee = FileNotFoundError(
|
|
94
|
+
f"The recording path passed to {ap.prog}, '{args.recording}', doesn't exist."
|
|
95
|
+
)
|
|
96
|
+
raise ee from e
|
|
97
|
+
except Exception as e:
|
|
98
|
+
ee = ValueError(
|
|
99
|
+
f"{ap.prog} couldn't load your recording. Detailed error above."
|
|
100
|
+
)
|
|
101
|
+
raise ee from e
|
|
102
|
+
|
|
103
|
+
# -- run
|
|
104
|
+
# TODO: maybe this script should dump to Phy?
|
|
105
|
+
ret = main.dartsort(rec, output_dir, cfg=cfg, overwrite=args.overwrite)
|
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
3
|
+
import h5py
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
from hdbscan import HDBSCAN
|
|
7
|
+
except ImportError:
|
|
8
|
+
from sklearn.cluster import HDBSCAN # type: ignore
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
from scipy.cluster.hierarchy import fcluster, linkage
|
|
12
|
+
from scipy.spatial import KDTree
|
|
13
|
+
from sklearn.neighbors import KNeighborsClassifier
|
|
14
|
+
|
|
15
|
+
from ..util import data_util, drift_util, waveform_util
|
|
16
|
+
from ..util.logging_util import get_logger
|
|
17
|
+
from dredge.motion_util import IdentityMotionEstimate
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
logger = get_logger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def agglomerate(labels, distances, linkage_method="complete", threshold=1.0, eps=1e-5):
|
|
24
|
+
""""""
|
|
25
|
+
n = distances.shape[0]
|
|
26
|
+
assert eps < threshold # that would be confusing.
|
|
27
|
+
if n <= 1:
|
|
28
|
+
return labels, np.arange(n)
|
|
29
|
+
pdist = distances[np.triu_indices(n, k=1)]
|
|
30
|
+
# tolearate some numerical zeros.
|
|
31
|
+
pdist[np.logical_and(pdist > -eps, pdist < 0)] = 0.0
|
|
32
|
+
if pdist.min() > threshold:
|
|
33
|
+
if labels is None:
|
|
34
|
+
return None, np.arange(n)
|
|
35
|
+
else:
|
|
36
|
+
ids = np.unique(labels)
|
|
37
|
+
return labels, ids[ids >= 0]
|
|
38
|
+
|
|
39
|
+
finite = np.isfinite(pdist)
|
|
40
|
+
if not finite.all():
|
|
41
|
+
inf = max(0, pdist[finite].max()) + threshold + 1.0
|
|
42
|
+
pdist[np.logical_not(finite)] = inf
|
|
43
|
+
|
|
44
|
+
Z = linkage(pdist, method=linkage_method)
|
|
45
|
+
try:
|
|
46
|
+
new_ids = fcluster(Z, threshold, criterion="distance")
|
|
47
|
+
except ValueError as e:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"fcluster failed with {threshold=} and smallest pdist {pdist.min()}."
|
|
50
|
+
) from e
|
|
51
|
+
# offset by 1, I think always, but I don't want to be wrong?
|
|
52
|
+
new_ids -= new_ids.min()
|
|
53
|
+
|
|
54
|
+
if labels is None:
|
|
55
|
+
new_labels = None
|
|
56
|
+
else:
|
|
57
|
+
kept = np.flatnonzero(labels >= 0)
|
|
58
|
+
new_labels = np.full_like(labels, -1)
|
|
59
|
+
new_labels[kept] = new_ids[labels[kept]]
|
|
60
|
+
|
|
61
|
+
return new_labels, new_ids
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def leafsets(Z, max_distance=np.inf):
|
|
65
|
+
"""For a linkage Z, get the leaves in each non-leaf cluster."""
|
|
66
|
+
n = len(Z) + 1
|
|
67
|
+
leaves = {}
|
|
68
|
+
for i, row in enumerate(Z):
|
|
69
|
+
pa, pb, dist, nab = row
|
|
70
|
+
if dist > max_distance:
|
|
71
|
+
break
|
|
72
|
+
leavesa = leaves.get(pa, [int(pa)])
|
|
73
|
+
leavesb = leaves.get(pb, [int(pb)])
|
|
74
|
+
leaves[n + i] = leavesa + leavesb
|
|
75
|
+
leaves[n + i].sort()
|
|
76
|
+
return leaves
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def maximal_leaf_groups(
|
|
80
|
+
Z, distances: np.ndarray, max_distance=np.inf, max_group_size: int = 100
|
|
81
|
+
):
|
|
82
|
+
"""Get largest groups in linkage Z within some max complete dist and group size."""
|
|
83
|
+
n = len(Z) + 1
|
|
84
|
+
covered = set()
|
|
85
|
+
leaves = leafsets(Z, max_distance=max_distance)
|
|
86
|
+
leaves = {k: set(v) for k, v in leaves.items()}
|
|
87
|
+
group_parents = []
|
|
88
|
+
for i, row in reversed(list(enumerate(Z))):
|
|
89
|
+
pa, pb, dist, nab = row
|
|
90
|
+
if nab > max_group_size:
|
|
91
|
+
continue
|
|
92
|
+
if n + i not in leaves:
|
|
93
|
+
# this is the distance check, since leafsets covers that.
|
|
94
|
+
continue
|
|
95
|
+
if leaves[n + i].issubset(covered):
|
|
96
|
+
continue
|
|
97
|
+
group_parents.append(n + i)
|
|
98
|
+
covered.update(leaves[n + i])
|
|
99
|
+
|
|
100
|
+
groups = [tuple(leaves[p]) for p in group_parents]
|
|
101
|
+
|
|
102
|
+
# at this point, some nodes are not covered, but it's possible that they may
|
|
103
|
+
# still be close enough to a group. this is a greedy algorithm to find all
|
|
104
|
+
# of those nodes and add them to the best groups starting with the closest
|
|
105
|
+
# matches first. we'll enforce complete linkage here (the most strict).
|
|
106
|
+
if len(groups):
|
|
107
|
+
too_big_penalty = np.array([len(g) >= max_group_size for g in groups])
|
|
108
|
+
too_big_penalty = np.where(too_big_penalty, np.inf, 0.0)
|
|
109
|
+
uncovered = [i for i in range(n) if i not in covered]
|
|
110
|
+
group_distances = [
|
|
111
|
+
np.array([distances[ui, g].max() for g in groups]) + too_big_penalty
|
|
112
|
+
for ui in uncovered
|
|
113
|
+
]
|
|
114
|
+
while len(uncovered):
|
|
115
|
+
min_distances = [gd.min() for gd in group_distances]
|
|
116
|
+
argmin_leaf = int(np.argmin(min_distances))
|
|
117
|
+
if min_distances[argmin_leaf] > max_distance:
|
|
118
|
+
break
|
|
119
|
+
|
|
120
|
+
argmin_group = group_distances[argmin_leaf].argmin()
|
|
121
|
+
|
|
122
|
+
# add leaf to group and coverage
|
|
123
|
+
new_leaf = uncovered[argmin_leaf]
|
|
124
|
+
groups[argmin_group] = (*groups[argmin_group], new_leaf)
|
|
125
|
+
covered.add(new_leaf)
|
|
126
|
+
|
|
127
|
+
# remove leaf from uncovered
|
|
128
|
+
del uncovered[argmin_leaf]
|
|
129
|
+
del group_distances[argmin_leaf]
|
|
130
|
+
|
|
131
|
+
# update other leaves' dists to this group
|
|
132
|
+
g = groups[argmin_group]
|
|
133
|
+
for j, ui in enumerate(uncovered):
|
|
134
|
+
if len(g) >= max_group_size:
|
|
135
|
+
group_distances[j][argmin_group] = np.inf
|
|
136
|
+
else:
|
|
137
|
+
group_distances[j][argmin_group] = distances[ui, g].max()
|
|
138
|
+
|
|
139
|
+
# now the true singletons are added
|
|
140
|
+
for i in range(n):
|
|
141
|
+
if i not in covered:
|
|
142
|
+
groups.append((i,))
|
|
143
|
+
|
|
144
|
+
assert max(map(len, groups)) <= max_group_size
|
|
145
|
+
assert sum(map(len, groups)) == n
|
|
146
|
+
assert set(gv for g in groups for gv in g) == set(range(n))
|
|
147
|
+
|
|
148
|
+
groups = [tuple(sorted(g)) for g in groups]
|
|
149
|
+
|
|
150
|
+
return groups
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def is_largest_set_smaller_than(Z, leaf_descendants, max_size=5):
|
|
154
|
+
n_branches = len(Z)
|
|
155
|
+
n_units = n_branches + 1
|
|
156
|
+
indicator = np.zeros(n_branches, dtype=bool)
|
|
157
|
+
# this walks up from the leaves
|
|
158
|
+
# at each branch, if #descendents<size, branch gets true and
|
|
159
|
+
# its parents are set to false. that way only one ancestor
|
|
160
|
+
# is true for each leaf.
|
|
161
|
+
for i, (pa, pb, dist, nab) in enumerate(Z):
|
|
162
|
+
sz = len(leaf_descendants[n_units + i])
|
|
163
|
+
if sz > max_size:
|
|
164
|
+
continue
|
|
165
|
+
pa = int(pa)
|
|
166
|
+
pb = int(pb)
|
|
167
|
+
indicator[i] = True
|
|
168
|
+
if pa >= n_units:
|
|
169
|
+
indicator[pa - n_units] = False
|
|
170
|
+
if pb >= n_units:
|
|
171
|
+
indicator[pb - n_units] = False
|
|
172
|
+
# assert that at most one ancestor is true for each leaf
|
|
173
|
+
counts = np.zeros(n_units, dtype=int)
|
|
174
|
+
for i, ind in enumerate(indicator):
|
|
175
|
+
counts[leaf_descendants[n_units + i]] += ind
|
|
176
|
+
assert counts.max() <= 1
|
|
177
|
+
return indicator
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def combine_distances(
|
|
181
|
+
distances,
|
|
182
|
+
thresholds,
|
|
183
|
+
agg_function=np.maximum,
|
|
184
|
+
sym_function=np.maximum,
|
|
185
|
+
):
|
|
186
|
+
"""Combine several distance matrices and symmetrize them
|
|
187
|
+
|
|
188
|
+
They have different reference thresholds, but the result of this function
|
|
189
|
+
has threshold 1.
|
|
190
|
+
"""
|
|
191
|
+
dists = distances[0] / thresholds[0]
|
|
192
|
+
for dist, thresh in zip(distances[1:], thresholds[1:]):
|
|
193
|
+
dists = agg_function(dists, dist / thresh)
|
|
194
|
+
return sym_function(dists, dists.T)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def combine_disjoint(inds_a, labels_a, inds_b, labels_b):
|
|
198
|
+
labels = np.full(labels_a.size + labels_b.size, -1, dtype=labels_a.dtype)
|
|
199
|
+
labels[inds_a] = labels_a
|
|
200
|
+
labels[inds_b] = labels_b
|
|
201
|
+
return labels
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def reorder_by_depth(sorting, motion_est=None):
|
|
205
|
+
kept = np.flatnonzero(sorting.labels >= 0)
|
|
206
|
+
kept_labels = sorting.labels[kept]
|
|
207
|
+
|
|
208
|
+
units, kept_labels = np.unique(kept_labels, return_inverse=True)
|
|
209
|
+
|
|
210
|
+
depths = sorting.point_source_localizations[kept, 2]
|
|
211
|
+
if motion_est is not None:
|
|
212
|
+
depths = motion_est.correct_s(sorting.times_seconds[kept], depths)
|
|
213
|
+
|
|
214
|
+
centroids = np.zeros(units.size)
|
|
215
|
+
for u in range(units.size):
|
|
216
|
+
inu = np.flatnonzero(kept_labels == u)
|
|
217
|
+
centroids[u] = np.median(depths[inu])
|
|
218
|
+
|
|
219
|
+
labels = sorting.labels.copy()
|
|
220
|
+
# this one is some food for thought, lol.
|
|
221
|
+
labels[kept] = np.argsort(np.argsort(centroids))[kept_labels]
|
|
222
|
+
|
|
223
|
+
return sorting.ephemeral_replace(labels=labels)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def closest_registered_channels(
|
|
227
|
+
times_seconds, x, z_abs, geom, z_reg=None, motion_est=None
|
|
228
|
+
) -> np.ndarray:
|
|
229
|
+
"""Assign spikes to the drift-extended channel closest to their registered position"""
|
|
230
|
+
if motion_est is None:
|
|
231
|
+
motion_est = IdentityMotionEstimate()
|
|
232
|
+
registered_geom = drift_util.registered_geometry(geom, motion_est)
|
|
233
|
+
if z_reg is None:
|
|
234
|
+
z_reg = motion_est.correct_s(times_seconds, z_abs)
|
|
235
|
+
reg_pos = np.c_[x, z_reg]
|
|
236
|
+
|
|
237
|
+
registered_kdt = KDTree(registered_geom)
|
|
238
|
+
_, reg_channels = registered_kdt.query(reg_pos)
|
|
239
|
+
reg_channels = np.atleast_1d(reg_channels)
|
|
240
|
+
|
|
241
|
+
return reg_channels
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def grid_snap(
|
|
245
|
+
times_seconds,
|
|
246
|
+
x,
|
|
247
|
+
z_abs,
|
|
248
|
+
geom,
|
|
249
|
+
grid_dx=15.0,
|
|
250
|
+
grid_dz=15.0,
|
|
251
|
+
z_reg=None,
|
|
252
|
+
motion_est=None,
|
|
253
|
+
) -> np.ndarray:
|
|
254
|
+
if motion_est is None:
|
|
255
|
+
motion_est = IdentityMotionEstimate()
|
|
256
|
+
if z_reg is None:
|
|
257
|
+
z_reg = motion_est.correct_s(times_seconds, z_abs)
|
|
258
|
+
reg_pos = np.c_[x, z_reg]
|
|
259
|
+
|
|
260
|
+
# make a grid inside the registered geom bounding box
|
|
261
|
+
registered_geom = drift_util.registered_geometry(geom, motion_est)
|
|
262
|
+
min_x, max_x = registered_geom[:, 0].min(), registered_geom[:, 0].max()
|
|
263
|
+
min_z, max_z = registered_geom[:, 1].min(), registered_geom[:, 1].max()
|
|
264
|
+
grid_x = np.arange(min_x, max_x, grid_dx)
|
|
265
|
+
grid_x += (min_x + max_x) / 2 - grid_x.mean()
|
|
266
|
+
grid_z = np.arange(min_z, max_z, grid_dz)
|
|
267
|
+
grid_z += (min_z + max_z) / 2 - grid_z.mean()
|
|
268
|
+
grid_xx, grid_zz = np.meshgrid(grid_x, grid_z, indexing="ij")
|
|
269
|
+
grid = np.c_[grid_xx.ravel(), grid_zz.ravel()]
|
|
270
|
+
|
|
271
|
+
# snap to closest grid point
|
|
272
|
+
registered_kdt = KDTree(grid)
|
|
273
|
+
_, reg_channels = registered_kdt.query(reg_pos)
|
|
274
|
+
reg_channels = np.atleast_1d(reg_channels)
|
|
275
|
+
|
|
276
|
+
return reg_channels
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def recursive_hdbscan_clustering(
|
|
280
|
+
features,
|
|
281
|
+
min_cluster_size=25,
|
|
282
|
+
min_samples=25,
|
|
283
|
+
cluster_selection_epsilon=1,
|
|
284
|
+
recursive=True,
|
|
285
|
+
):
|
|
286
|
+
clusterer = HDBSCAN(
|
|
287
|
+
min_cluster_size=min_cluster_size,
|
|
288
|
+
cluster_selection_epsilon=cluster_selection_epsilon,
|
|
289
|
+
min_samples=min_samples,
|
|
290
|
+
)
|
|
291
|
+
clusterer.fit(features)
|
|
292
|
+
|
|
293
|
+
if not recursive:
|
|
294
|
+
return clusterer.labels_
|
|
295
|
+
|
|
296
|
+
# recursively split clusters as long as HDBSCAN keeps finding more than 1
|
|
297
|
+
units = np.unique(clusterer.labels_)
|
|
298
|
+
if units[units >= 0].size <= 1:
|
|
299
|
+
return np.zeros_like(clusterer.labels_)
|
|
300
|
+
|
|
301
|
+
# else, recursively enter all labels and split them
|
|
302
|
+
labels = clusterer.labels_.copy()
|
|
303
|
+
next_label = units.max() + 1
|
|
304
|
+
for unit in units[units >= 0]:
|
|
305
|
+
in_unit = np.flatnonzero(clusterer.labels_ == unit)
|
|
306
|
+
split_labels = recursive_hdbscan_clustering(
|
|
307
|
+
features[in_unit],
|
|
308
|
+
min_cluster_size=min_cluster_size,
|
|
309
|
+
min_samples=min_samples,
|
|
310
|
+
cluster_selection_epsilon=cluster_selection_epsilon,
|
|
311
|
+
recursive=recursive,
|
|
312
|
+
)
|
|
313
|
+
kept = split_labels >= 0
|
|
314
|
+
dropped = np.logical_not(kept)
|
|
315
|
+
labels[in_unit[dropped]] = split_labels[dropped]
|
|
316
|
+
labels[in_unit[kept]] = split_labels[kept] + next_label
|
|
317
|
+
next_label += split_labels.max() + 1
|
|
318
|
+
|
|
319
|
+
# reindex
|
|
320
|
+
_, labels[labels >= 0] = np.unique(labels[labels >= 0], return_inverse=True)
|
|
321
|
+
return labels
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def knn_reassign_outliers(labels, features):
|
|
325
|
+
outliers = labels < 0
|
|
326
|
+
outliers_idx = np.flatnonzero(outliers)
|
|
327
|
+
if not outliers_idx.size:
|
|
328
|
+
return labels
|
|
329
|
+
knn = KNeighborsClassifier()
|
|
330
|
+
knn.fit(features[~outliers], labels[~outliers])
|
|
331
|
+
new_labels = labels.copy()
|
|
332
|
+
new_labels[outliers_idx] = knn.predict(features[outliers_idx])
|
|
333
|
+
return new_labels
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def get_main_channel_pcs(
|
|
337
|
+
sorting,
|
|
338
|
+
which=slice(None),
|
|
339
|
+
rank=1,
|
|
340
|
+
show_progress=False,
|
|
341
|
+
dataset_name="collisioncleaned_tpca_features",
|
|
342
|
+
):
|
|
343
|
+
mask = np.zeros(len(sorting), dtype=bool)
|
|
344
|
+
mask[which] = True
|
|
345
|
+
channels = sorting.channels[which]
|
|
346
|
+
|
|
347
|
+
features = getattr(sorting, "collisioncleaned_tpca_features", None)
|
|
348
|
+
channel_index = getattr(sorting, "channel_index", None)
|
|
349
|
+
if features is not None and channel_index is not None:
|
|
350
|
+
features = features[which][:, :rank]
|
|
351
|
+
return waveform_util.grab_main_channels(features, channels, channel_index)
|
|
352
|
+
|
|
353
|
+
features = np.empty((mask.sum(), rank), dtype=np.float32)
|
|
354
|
+
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
355
|
+
feats_dset = h5[dataset_name]
|
|
356
|
+
channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
|
|
357
|
+
for ixs, feats in data_util.yield_masked_chunks(
|
|
358
|
+
mask, feats_dset, show_progress=show_progress, desc_prefix="Main channel"
|
|
359
|
+
):
|
|
360
|
+
feats = feats[:, :rank]
|
|
361
|
+
feats = waveform_util.grab_main_channels(
|
|
362
|
+
feats, channels[ixs], channel_index
|
|
363
|
+
)
|
|
364
|
+
features[ixs] = feats
|
|
365
|
+
return features
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def decrumb(labels, min_size=5, in_place=False, flatten=True):
|
|
369
|
+
kept = np.flatnonzero(labels >= 0)
|
|
370
|
+
labels_kept = labels[kept]
|
|
371
|
+
labels = labels if in_place else labels.copy()
|
|
372
|
+
|
|
373
|
+
units_sparse, counts_sparse = np.unique(labels_kept, return_counts=True)
|
|
374
|
+
if not units_sparse.size:
|
|
375
|
+
return labels
|
|
376
|
+
|
|
377
|
+
k = units_sparse.max() + 1
|
|
378
|
+
counts = np.zeros(k, dtype=counts_sparse.dtype)
|
|
379
|
+
counts[units_sparse] = counts_sparse
|
|
380
|
+
units = np.arange(k)
|
|
381
|
+
|
|
382
|
+
big_enough = counts >= min_size
|
|
383
|
+
k1 = big_enough.sum()
|
|
384
|
+
units[np.logical_not(big_enough)] = -1
|
|
385
|
+
if flatten:
|
|
386
|
+
units[big_enough] = np.arange(k1)
|
|
387
|
+
|
|
388
|
+
logger.dartsortdebug(f"decrumb ({min_size}): {k}->{k1}.")
|
|
389
|
+
|
|
390
|
+
labels[kept] = units[labels_kept]
|
|
391
|
+
|
|
392
|
+
return labels
|