DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/__init__.py ADDED
@@ -0,0 +1,68 @@
1
+ import importlib.metadata
2
+
3
+ from . import detect, util
4
+ from .clustering import (
5
+ cluster_util,
6
+ clustering_strategies,
7
+ density,
8
+ get_clusterer,
9
+ get_clustering_features,
10
+ kmeans,
11
+ merge,
12
+ refinement_strategies,
13
+ )
14
+ from .config import *
15
+ from .evaluate import config_grid, hybrid_util, simkit, simlib
16
+ from .evaluate.analysis import DARTsortAnalysis
17
+ from .evaluate.comparison import DARTsortGroundTruthComparison, DARTsortGTVersus
18
+ from .localize.localize_util import (
19
+ localize_amplitude_vectors,
20
+ localize_hdf5,
21
+ localize_waveforms,
22
+ )
23
+ from .main import (
24
+ ObjectiveUpdateTemplateMatchingPeeler,
25
+ SubtractionPeeler,
26
+ check_recording,
27
+ cluster,
28
+ dartsort,
29
+ estimate_motion,
30
+ grab,
31
+ logger,
32
+ match,
33
+ run_peeler,
34
+ subtract,
35
+ threshold,
36
+ universal_match,
37
+ )
38
+ from .peel.grab import GrabAndFeaturize
39
+ from .templates import (
40
+ TemplateData,
41
+ estimate_template_library,
42
+ postprocess_util,
43
+ realign,
44
+ )
45
+ from .transform import WaveformPipeline
46
+ from .util import data_util, logging_util, noise_util, spiketorch, waveform_util
47
+ from .util.data_util import (
48
+ DARTsortSorting,
49
+ get_featurization_pipeline,
50
+ load_h5,
51
+ get_tpca,
52
+ load_stored_tsvd,
53
+ check_recording,
54
+ )
55
+ from .util.drift_util import registered_geometry
56
+ from .util.internal_config import *
57
+ from .util.logging_util import (
58
+ DARTSORTDEBUG,
59
+ DARTSORTVERBOSE,
60
+ DARTsortLogger,
61
+ get_logger,
62
+ )
63
+ from .util.noise_util import EmbeddedNoise
64
+ from .util.py_util import resolve_path
65
+ from .util.registration_util import try_load_motion_est
66
+ from .util.waveform_util import make_channel_index
67
+
68
+ __version__ = importlib.metadata.version("dartsort")
dartsort/cli.py ADDED
@@ -0,0 +1,105 @@
1
+ import argparse
2
+ import spikeinterface.core as sc
3
+ import logging
4
+
5
+ from .util import cli_util
6
+ from . import config, main
7
+
8
+
9
+ def dartsort_cli():
10
+ """dartsort's CLI, invoked by `dartsort` at the terminal.
11
+
12
+ Try `dartsort --help` to start.
13
+
14
+ ---<!!> Not stable.
15
+
16
+ I am figuring out how to do preprocessing still. It may be configured?
17
+ """
18
+ # -- define CLI
19
+ assert dartsort_cli.__doc__ is not None
20
+ ap = argparse.ArgumentParser(
21
+ prog="dartsort",
22
+ epilog=dartsort_cli.__doc__.split("---")[1],
23
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
24
+ )
25
+ ap.add_argument("recording", help="Path to SpikeInterface RecordingExtractor.")
26
+ ap.add_argument(
27
+ "output_dir",
28
+ nargs="?",
29
+ help="Folder where outputs will be saved. If this is unset, then "
30
+ "--config-toml must be supplied, and the output folder will be the one where "
31
+ "that configuration lives. If both are supplied, output_dir will be "
32
+ "used (not the config file's parent dir).",
33
+ )
34
+ ap.add_argument(
35
+ "--config-toml",
36
+ type=str,
37
+ default=None,
38
+ help="Path to configuration in TOML format. Arguments passed on the "
39
+ "command line will override their values in the TOML file.",
40
+ )
41
+ ap.add_argument("--overwrite", action="store_true")
42
+ ap.add_argument(
43
+ "--loglevel",
44
+ default="WARNING",
45
+ type=str.upper,
46
+ choices=logging._nameToLevel.keys(),
47
+ )
48
+ # user-facing API
49
+ cli_util.dataclass_to_argparse(config.DARTsortUserConfig, parser=ap)
50
+
51
+ # super secret developer-only args
52
+ dev_args = ap.add_argument_group("Secret development flags ($1.50 fee to use)")
53
+ cli_util.dataclass_to_argparse(
54
+ config.DeveloperConfig,
55
+ parser=dev_args,
56
+ skipnames=cli_util.fieldnames(config.DARTsortUserConfig),
57
+ )
58
+
59
+ # -- parse args
60
+ args = ap.parse_args()
61
+
62
+ # set up logging
63
+ logging.basicConfig(level=args.loglevel)
64
+
65
+ # check if we have config file
66
+ config_toml = None
67
+ if args.config_toml:
68
+ config_toml = cli_util.ensurepath(args.config_toml)
69
+
70
+ # determine output directory
71
+ if args.output_dir:
72
+ output_dir = cli_util.ensurepath(args.output_dir, strict=False)
73
+ elif config_toml is None:
74
+ print(f"No output directory given, exiting. See `{ap.prog} -h`.")
75
+ return 1
76
+ else:
77
+ output_dir = config_toml.parent
78
+
79
+ # determine the config from the command line args
80
+ cfg = cli_util.combine_toml_and_argv(
81
+ (config.DARTsortUserConfig, config.DeveloperConfig),
82
+ config.DeveloperConfig,
83
+ config_toml,
84
+ args,
85
+ )
86
+
87
+ # load the recording
88
+ # TODO: preprocessing management
89
+ try:
90
+ rec = sc.load(cli_util.ensurepath(args.recording))
91
+ assert isinstance(rec, sc.BaseRecording)
92
+ except FileNotFoundError as e:
93
+ ee = FileNotFoundError(
94
+ f"The recording path passed to {ap.prog}, '{args.recording}', doesn't exist."
95
+ )
96
+ raise ee from e
97
+ except Exception as e:
98
+ ee = ValueError(
99
+ f"{ap.prog} couldn't load your recording. Detailed error above."
100
+ )
101
+ raise ee from e
102
+
103
+ # -- run
104
+ # TODO: maybe this script should dump to Phy?
105
+ ret = main.dartsort(rec, output_dir, cfg=cfg, overwrite=args.overwrite)
@@ -0,0 +1,3 @@
1
+ from .clustering_features import get_clustering_features, SimpleMatrixFeatures
2
+ from .clustering import get_clusterer, clustering_strategies, refinement_strategies
3
+ from .gmm.stable_features import StableSpikeDataset, SpikeFeatures, SpikeNeighborhoods
@@ -0,0 +1,392 @@
1
+ from typing import cast
2
+
3
+ import h5py
4
+
5
+ try:
6
+ from hdbscan import HDBSCAN
7
+ except ImportError:
8
+ from sklearn.cluster import HDBSCAN # type: ignore
9
+
10
+ import numpy as np
11
+ from scipy.cluster.hierarchy import fcluster, linkage
12
+ from scipy.spatial import KDTree
13
+ from sklearn.neighbors import KNeighborsClassifier
14
+
15
+ from ..util import data_util, drift_util, waveform_util
16
+ from ..util.logging_util import get_logger
17
+ from dredge.motion_util import IdentityMotionEstimate
18
+
19
+
20
+ logger = get_logger(__name__)
21
+
22
+
23
+ def agglomerate(labels, distances, linkage_method="complete", threshold=1.0, eps=1e-5):
24
+ """"""
25
+ n = distances.shape[0]
26
+ assert eps < threshold # that would be confusing.
27
+ if n <= 1:
28
+ return labels, np.arange(n)
29
+ pdist = distances[np.triu_indices(n, k=1)]
30
+ # tolearate some numerical zeros.
31
+ pdist[np.logical_and(pdist > -eps, pdist < 0)] = 0.0
32
+ if pdist.min() > threshold:
33
+ if labels is None:
34
+ return None, np.arange(n)
35
+ else:
36
+ ids = np.unique(labels)
37
+ return labels, ids[ids >= 0]
38
+
39
+ finite = np.isfinite(pdist)
40
+ if not finite.all():
41
+ inf = max(0, pdist[finite].max()) + threshold + 1.0
42
+ pdist[np.logical_not(finite)] = inf
43
+
44
+ Z = linkage(pdist, method=linkage_method)
45
+ try:
46
+ new_ids = fcluster(Z, threshold, criterion="distance")
47
+ except ValueError as e:
48
+ raise ValueError(
49
+ f"fcluster failed with {threshold=} and smallest pdist {pdist.min()}."
50
+ ) from e
51
+ # offset by 1, I think always, but I don't want to be wrong?
52
+ new_ids -= new_ids.min()
53
+
54
+ if labels is None:
55
+ new_labels = None
56
+ else:
57
+ kept = np.flatnonzero(labels >= 0)
58
+ new_labels = np.full_like(labels, -1)
59
+ new_labels[kept] = new_ids[labels[kept]]
60
+
61
+ return new_labels, new_ids
62
+
63
+
64
+ def leafsets(Z, max_distance=np.inf):
65
+ """For a linkage Z, get the leaves in each non-leaf cluster."""
66
+ n = len(Z) + 1
67
+ leaves = {}
68
+ for i, row in enumerate(Z):
69
+ pa, pb, dist, nab = row
70
+ if dist > max_distance:
71
+ break
72
+ leavesa = leaves.get(pa, [int(pa)])
73
+ leavesb = leaves.get(pb, [int(pb)])
74
+ leaves[n + i] = leavesa + leavesb
75
+ leaves[n + i].sort()
76
+ return leaves
77
+
78
+
79
+ def maximal_leaf_groups(
80
+ Z, distances: np.ndarray, max_distance=np.inf, max_group_size: int = 100
81
+ ):
82
+ """Get largest groups in linkage Z within some max complete dist and group size."""
83
+ n = len(Z) + 1
84
+ covered = set()
85
+ leaves = leafsets(Z, max_distance=max_distance)
86
+ leaves = {k: set(v) for k, v in leaves.items()}
87
+ group_parents = []
88
+ for i, row in reversed(list(enumerate(Z))):
89
+ pa, pb, dist, nab = row
90
+ if nab > max_group_size:
91
+ continue
92
+ if n + i not in leaves:
93
+ # this is the distance check, since leafsets covers that.
94
+ continue
95
+ if leaves[n + i].issubset(covered):
96
+ continue
97
+ group_parents.append(n + i)
98
+ covered.update(leaves[n + i])
99
+
100
+ groups = [tuple(leaves[p]) for p in group_parents]
101
+
102
+ # at this point, some nodes are not covered, but it's possible that they may
103
+ # still be close enough to a group. this is a greedy algorithm to find all
104
+ # of those nodes and add them to the best groups starting with the closest
105
+ # matches first. we'll enforce complete linkage here (the most strict).
106
+ if len(groups):
107
+ too_big_penalty = np.array([len(g) >= max_group_size for g in groups])
108
+ too_big_penalty = np.where(too_big_penalty, np.inf, 0.0)
109
+ uncovered = [i for i in range(n) if i not in covered]
110
+ group_distances = [
111
+ np.array([distances[ui, g].max() for g in groups]) + too_big_penalty
112
+ for ui in uncovered
113
+ ]
114
+ while len(uncovered):
115
+ min_distances = [gd.min() for gd in group_distances]
116
+ argmin_leaf = int(np.argmin(min_distances))
117
+ if min_distances[argmin_leaf] > max_distance:
118
+ break
119
+
120
+ argmin_group = group_distances[argmin_leaf].argmin()
121
+
122
+ # add leaf to group and coverage
123
+ new_leaf = uncovered[argmin_leaf]
124
+ groups[argmin_group] = (*groups[argmin_group], new_leaf)
125
+ covered.add(new_leaf)
126
+
127
+ # remove leaf from uncovered
128
+ del uncovered[argmin_leaf]
129
+ del group_distances[argmin_leaf]
130
+
131
+ # update other leaves' dists to this group
132
+ g = groups[argmin_group]
133
+ for j, ui in enumerate(uncovered):
134
+ if len(g) >= max_group_size:
135
+ group_distances[j][argmin_group] = np.inf
136
+ else:
137
+ group_distances[j][argmin_group] = distances[ui, g].max()
138
+
139
+ # now the true singletons are added
140
+ for i in range(n):
141
+ if i not in covered:
142
+ groups.append((i,))
143
+
144
+ assert max(map(len, groups)) <= max_group_size
145
+ assert sum(map(len, groups)) == n
146
+ assert set(gv for g in groups for gv in g) == set(range(n))
147
+
148
+ groups = [tuple(sorted(g)) for g in groups]
149
+
150
+ return groups
151
+
152
+
153
+ def is_largest_set_smaller_than(Z, leaf_descendants, max_size=5):
154
+ n_branches = len(Z)
155
+ n_units = n_branches + 1
156
+ indicator = np.zeros(n_branches, dtype=bool)
157
+ # this walks up from the leaves
158
+ # at each branch, if #descendents<size, branch gets true and
159
+ # its parents are set to false. that way only one ancestor
160
+ # is true for each leaf.
161
+ for i, (pa, pb, dist, nab) in enumerate(Z):
162
+ sz = len(leaf_descendants[n_units + i])
163
+ if sz > max_size:
164
+ continue
165
+ pa = int(pa)
166
+ pb = int(pb)
167
+ indicator[i] = True
168
+ if pa >= n_units:
169
+ indicator[pa - n_units] = False
170
+ if pb >= n_units:
171
+ indicator[pb - n_units] = False
172
+ # assert that at most one ancestor is true for each leaf
173
+ counts = np.zeros(n_units, dtype=int)
174
+ for i, ind in enumerate(indicator):
175
+ counts[leaf_descendants[n_units + i]] += ind
176
+ assert counts.max() <= 1
177
+ return indicator
178
+
179
+
180
+ def combine_distances(
181
+ distances,
182
+ thresholds,
183
+ agg_function=np.maximum,
184
+ sym_function=np.maximum,
185
+ ):
186
+ """Combine several distance matrices and symmetrize them
187
+
188
+ They have different reference thresholds, but the result of this function
189
+ has threshold 1.
190
+ """
191
+ dists = distances[0] / thresholds[0]
192
+ for dist, thresh in zip(distances[1:], thresholds[1:]):
193
+ dists = agg_function(dists, dist / thresh)
194
+ return sym_function(dists, dists.T)
195
+
196
+
197
+ def combine_disjoint(inds_a, labels_a, inds_b, labels_b):
198
+ labels = np.full(labels_a.size + labels_b.size, -1, dtype=labels_a.dtype)
199
+ labels[inds_a] = labels_a
200
+ labels[inds_b] = labels_b
201
+ return labels
202
+
203
+
204
+ def reorder_by_depth(sorting, motion_est=None):
205
+ kept = np.flatnonzero(sorting.labels >= 0)
206
+ kept_labels = sorting.labels[kept]
207
+
208
+ units, kept_labels = np.unique(kept_labels, return_inverse=True)
209
+
210
+ depths = sorting.point_source_localizations[kept, 2]
211
+ if motion_est is not None:
212
+ depths = motion_est.correct_s(sorting.times_seconds[kept], depths)
213
+
214
+ centroids = np.zeros(units.size)
215
+ for u in range(units.size):
216
+ inu = np.flatnonzero(kept_labels == u)
217
+ centroids[u] = np.median(depths[inu])
218
+
219
+ labels = sorting.labels.copy()
220
+ # this one is some food for thought, lol.
221
+ labels[kept] = np.argsort(np.argsort(centroids))[kept_labels]
222
+
223
+ return sorting.ephemeral_replace(labels=labels)
224
+
225
+
226
+ def closest_registered_channels(
227
+ times_seconds, x, z_abs, geom, z_reg=None, motion_est=None
228
+ ) -> np.ndarray:
229
+ """Assign spikes to the drift-extended channel closest to their registered position"""
230
+ if motion_est is None:
231
+ motion_est = IdentityMotionEstimate()
232
+ registered_geom = drift_util.registered_geometry(geom, motion_est)
233
+ if z_reg is None:
234
+ z_reg = motion_est.correct_s(times_seconds, z_abs)
235
+ reg_pos = np.c_[x, z_reg]
236
+
237
+ registered_kdt = KDTree(registered_geom)
238
+ _, reg_channels = registered_kdt.query(reg_pos)
239
+ reg_channels = np.atleast_1d(reg_channels)
240
+
241
+ return reg_channels
242
+
243
+
244
+ def grid_snap(
245
+ times_seconds,
246
+ x,
247
+ z_abs,
248
+ geom,
249
+ grid_dx=15.0,
250
+ grid_dz=15.0,
251
+ z_reg=None,
252
+ motion_est=None,
253
+ ) -> np.ndarray:
254
+ if motion_est is None:
255
+ motion_est = IdentityMotionEstimate()
256
+ if z_reg is None:
257
+ z_reg = motion_est.correct_s(times_seconds, z_abs)
258
+ reg_pos = np.c_[x, z_reg]
259
+
260
+ # make a grid inside the registered geom bounding box
261
+ registered_geom = drift_util.registered_geometry(geom, motion_est)
262
+ min_x, max_x = registered_geom[:, 0].min(), registered_geom[:, 0].max()
263
+ min_z, max_z = registered_geom[:, 1].min(), registered_geom[:, 1].max()
264
+ grid_x = np.arange(min_x, max_x, grid_dx)
265
+ grid_x += (min_x + max_x) / 2 - grid_x.mean()
266
+ grid_z = np.arange(min_z, max_z, grid_dz)
267
+ grid_z += (min_z + max_z) / 2 - grid_z.mean()
268
+ grid_xx, grid_zz = np.meshgrid(grid_x, grid_z, indexing="ij")
269
+ grid = np.c_[grid_xx.ravel(), grid_zz.ravel()]
270
+
271
+ # snap to closest grid point
272
+ registered_kdt = KDTree(grid)
273
+ _, reg_channels = registered_kdt.query(reg_pos)
274
+ reg_channels = np.atleast_1d(reg_channels)
275
+
276
+ return reg_channels
277
+
278
+
279
+ def recursive_hdbscan_clustering(
280
+ features,
281
+ min_cluster_size=25,
282
+ min_samples=25,
283
+ cluster_selection_epsilon=1,
284
+ recursive=True,
285
+ ):
286
+ clusterer = HDBSCAN(
287
+ min_cluster_size=min_cluster_size,
288
+ cluster_selection_epsilon=cluster_selection_epsilon,
289
+ min_samples=min_samples,
290
+ )
291
+ clusterer.fit(features)
292
+
293
+ if not recursive:
294
+ return clusterer.labels_
295
+
296
+ # recursively split clusters as long as HDBSCAN keeps finding more than 1
297
+ units = np.unique(clusterer.labels_)
298
+ if units[units >= 0].size <= 1:
299
+ return np.zeros_like(clusterer.labels_)
300
+
301
+ # else, recursively enter all labels and split them
302
+ labels = clusterer.labels_.copy()
303
+ next_label = units.max() + 1
304
+ for unit in units[units >= 0]:
305
+ in_unit = np.flatnonzero(clusterer.labels_ == unit)
306
+ split_labels = recursive_hdbscan_clustering(
307
+ features[in_unit],
308
+ min_cluster_size=min_cluster_size,
309
+ min_samples=min_samples,
310
+ cluster_selection_epsilon=cluster_selection_epsilon,
311
+ recursive=recursive,
312
+ )
313
+ kept = split_labels >= 0
314
+ dropped = np.logical_not(kept)
315
+ labels[in_unit[dropped]] = split_labels[dropped]
316
+ labels[in_unit[kept]] = split_labels[kept] + next_label
317
+ next_label += split_labels.max() + 1
318
+
319
+ # reindex
320
+ _, labels[labels >= 0] = np.unique(labels[labels >= 0], return_inverse=True)
321
+ return labels
322
+
323
+
324
+ def knn_reassign_outliers(labels, features):
325
+ outliers = labels < 0
326
+ outliers_idx = np.flatnonzero(outliers)
327
+ if not outliers_idx.size:
328
+ return labels
329
+ knn = KNeighborsClassifier()
330
+ knn.fit(features[~outliers], labels[~outliers])
331
+ new_labels = labels.copy()
332
+ new_labels[outliers_idx] = knn.predict(features[outliers_idx])
333
+ return new_labels
334
+
335
+
336
+ def get_main_channel_pcs(
337
+ sorting,
338
+ which=slice(None),
339
+ rank=1,
340
+ show_progress=False,
341
+ dataset_name="collisioncleaned_tpca_features",
342
+ ):
343
+ mask = np.zeros(len(sorting), dtype=bool)
344
+ mask[which] = True
345
+ channels = sorting.channels[which]
346
+
347
+ features = getattr(sorting, "collisioncleaned_tpca_features", None)
348
+ channel_index = getattr(sorting, "channel_index", None)
349
+ if features is not None and channel_index is not None:
350
+ features = features[which][:, :rank]
351
+ return waveform_util.grab_main_channels(features, channels, channel_index)
352
+
353
+ features = np.empty((mask.sum(), rank), dtype=np.float32)
354
+ with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
355
+ feats_dset = h5[dataset_name]
356
+ channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
357
+ for ixs, feats in data_util.yield_masked_chunks(
358
+ mask, feats_dset, show_progress=show_progress, desc_prefix="Main channel"
359
+ ):
360
+ feats = feats[:, :rank]
361
+ feats = waveform_util.grab_main_channels(
362
+ feats, channels[ixs], channel_index
363
+ )
364
+ features[ixs] = feats
365
+ return features
366
+
367
+
368
+ def decrumb(labels, min_size=5, in_place=False, flatten=True):
369
+ kept = np.flatnonzero(labels >= 0)
370
+ labels_kept = labels[kept]
371
+ labels = labels if in_place else labels.copy()
372
+
373
+ units_sparse, counts_sparse = np.unique(labels_kept, return_counts=True)
374
+ if not units_sparse.size:
375
+ return labels
376
+
377
+ k = units_sparse.max() + 1
378
+ counts = np.zeros(k, dtype=counts_sparse.dtype)
379
+ counts[units_sparse] = counts_sparse
380
+ units = np.arange(k)
381
+
382
+ big_enough = counts >= min_size
383
+ k1 = big_enough.sum()
384
+ units[np.logical_not(big_enough)] = -1
385
+ if flatten:
386
+ units[big_enough] = np.arange(k1)
387
+
388
+ logger.dartsortdebug(f"decrumb ({min_size}): {k}->{k1}.")
389
+
390
+ labels[kept] = units[labels_kept]
391
+
392
+ return labels