DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/main.py
ADDED
|
@@ -0,0 +1,657 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import traceback
|
|
3
|
+
from dataclasses import asdict
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from tempfile import TemporaryDirectory
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from spikeinterface.core import BaseRecording
|
|
10
|
+
|
|
11
|
+
from .clustering import SimpleMatrixFeatures, get_clusterer, get_clustering_features
|
|
12
|
+
from .config import DARTsortUserConfig, DeveloperConfig
|
|
13
|
+
from .peel import (
|
|
14
|
+
GrabAndFeaturize,
|
|
15
|
+
ObjectiveUpdateTemplateMatchingPeeler,
|
|
16
|
+
SubtractionPeeler,
|
|
17
|
+
ThresholdAndFeaturize,
|
|
18
|
+
UniversalTemplatesMatchingPeeler,
|
|
19
|
+
)
|
|
20
|
+
from .templates import TemplateData, estimate_template_library
|
|
21
|
+
from .util.data_util import DARTsortSorting, check_recording
|
|
22
|
+
from .util.internal_config import (
|
|
23
|
+
ClusteringConfig,
|
|
24
|
+
ClusteringFeaturesConfig,
|
|
25
|
+
ComputationConfig,
|
|
26
|
+
DARTsortInternalConfig,
|
|
27
|
+
MatchingConfig,
|
|
28
|
+
RefinementConfig,
|
|
29
|
+
SubtractionConfig,
|
|
30
|
+
ThresholdingConfig,
|
|
31
|
+
UniversalMatchingConfig,
|
|
32
|
+
default_clustering_cfg,
|
|
33
|
+
default_clustering_features_cfg,
|
|
34
|
+
default_dartsort_cfg,
|
|
35
|
+
default_featurization_cfg,
|
|
36
|
+
default_matching_cfg,
|
|
37
|
+
default_peeling_fit_sampling_cfg,
|
|
38
|
+
default_subtraction_cfg,
|
|
39
|
+
default_template_cfg,
|
|
40
|
+
default_thresholding_cfg,
|
|
41
|
+
default_universal_cfg,
|
|
42
|
+
default_waveform_cfg,
|
|
43
|
+
to_internal_config,
|
|
44
|
+
)
|
|
45
|
+
from .util.logging_util import get_logger
|
|
46
|
+
from .util.main_util import (
|
|
47
|
+
ds_all_to_workdir,
|
|
48
|
+
ds_dump_config,
|
|
49
|
+
ds_fast_forward,
|
|
50
|
+
ds_handle_delete_intermediate_features,
|
|
51
|
+
ds_save_features,
|
|
52
|
+
ds_save_intermediate_labels,
|
|
53
|
+
ds_save_motion_est,
|
|
54
|
+
)
|
|
55
|
+
from .util.peel_util import run_peeler
|
|
56
|
+
from .util.py_util import dartcopytree, resolve_path
|
|
57
|
+
from .util.registration_util import estimate_motion
|
|
58
|
+
|
|
59
|
+
logger = get_logger(__name__)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def dartsort(
|
|
63
|
+
recording: BaseRecording,
|
|
64
|
+
output_dir: str | Path,
|
|
65
|
+
cfg: (
|
|
66
|
+
DARTsortUserConfig | str | Path | DeveloperConfig | DARTsortInternalConfig
|
|
67
|
+
) = default_dartsort_cfg,
|
|
68
|
+
motion_est=None,
|
|
69
|
+
overwrite=False,
|
|
70
|
+
):
|
|
71
|
+
"""dartsort
|
|
72
|
+
|
|
73
|
+
This function runs a spike sorter called dartsort.
|
|
74
|
+
|
|
75
|
+
Arguments
|
|
76
|
+
---------
|
|
77
|
+
recording : BaseRecording
|
|
78
|
+
A spikeinterface.BaseRecording object
|
|
79
|
+
output_dir: str | Path
|
|
80
|
+
Folder where outputs are stored
|
|
81
|
+
cfg: DARTsortUserConfig | DARTsortInternalConfig | str | Path
|
|
82
|
+
Your settings. Either create a DARTsortUserConfig directly in code, or
|
|
83
|
+
you can pass a string or Path pointing to a .toml file here.
|
|
84
|
+
motion_est: optional dredge.MotionEstimate
|
|
85
|
+
This is meant to allow users to pass their own external motion estimate.
|
|
86
|
+
To do: support spikeinterface Motion objects here.
|
|
87
|
+
overwrite : bool, default=False
|
|
88
|
+
Ignore and overwrite stored results, if any. Otherwise, dartsort will
|
|
89
|
+
try to resume from the last step that ran, or if it had finished then
|
|
90
|
+
it will do nothing.
|
|
91
|
+
|
|
92
|
+
Returns
|
|
93
|
+
-------
|
|
94
|
+
Dictionary of sorting results, with the following keys:
|
|
95
|
+
- "sorting": DARTsortSorting
|
|
96
|
+
This has a sorting.to_numpy_sorting() method for those who want to
|
|
97
|
+
export back to spikeinterface. Which would be everyone? :)
|
|
98
|
+
Alternatively, you could visualize your results using the functions
|
|
99
|
+
in the `import dartsort.vis as dartvis` library.
|
|
100
|
+
- "motion_est": dredge.MotionEstimate
|
|
101
|
+
|
|
102
|
+
TODO: add the key "motion" with a spikeinterface Motion object.
|
|
103
|
+
"""
|
|
104
|
+
output_dir = resolve_path(output_dir)
|
|
105
|
+
output_dir.mkdir(exist_ok=True)
|
|
106
|
+
|
|
107
|
+
# convert cfg to internal format and store it for posterity
|
|
108
|
+
cfg = to_internal_config(cfg)
|
|
109
|
+
ds_dump_config(cfg, output_dir)
|
|
110
|
+
|
|
111
|
+
if cfg.work_in_tmpdir:
|
|
112
|
+
with TemporaryDirectory(prefix="dartsort", dir=cfg.tmpdir_parent) as work_dir:
|
|
113
|
+
# copy files and possibly recording to temporary directory
|
|
114
|
+
work_dir = resolve_path(work_dir)
|
|
115
|
+
logger.dartsortdebug(f"Working in {work_dir}, outputs to {output_dir}.")
|
|
116
|
+
recording, work_dir = ds_all_to_workdir(
|
|
117
|
+
internal_cfg=cfg,
|
|
118
|
+
output_dir=output_dir,
|
|
119
|
+
work_dir=work_dir,
|
|
120
|
+
recording=recording,
|
|
121
|
+
overwrite=overwrite,
|
|
122
|
+
)
|
|
123
|
+
assert work_dir is not None
|
|
124
|
+
|
|
125
|
+
# run the sorter, with extra error handlers for grabbing stuff from
|
|
126
|
+
# the temporary directory if user asked for that.
|
|
127
|
+
try:
|
|
128
|
+
return _dartsort_impl(
|
|
129
|
+
recording, output_dir, cfg, motion_est, work_dir, overwrite
|
|
130
|
+
)
|
|
131
|
+
except Exception as e:
|
|
132
|
+
traceback_path = output_dir / "traceback.txt"
|
|
133
|
+
error_data_path = output_dir / "error_state"
|
|
134
|
+
with open(traceback_path, "w") as f:
|
|
135
|
+
traceback.print_exception(e, file=f)
|
|
136
|
+
logger.exception(e)
|
|
137
|
+
if cfg.save_everything_on_error:
|
|
138
|
+
logger.critical(
|
|
139
|
+
f"Hit an error. Copying outputs to {error_data_path} "
|
|
140
|
+
f"and writing traceback to {traceback_path}."
|
|
141
|
+
)
|
|
142
|
+
dartcopytree(cfg, work_dir, error_data_path)
|
|
143
|
+
else:
|
|
144
|
+
logger.critical(
|
|
145
|
+
f"Hit an error. Writing traceback to {traceback_path}."
|
|
146
|
+
" work_in_tmpdir was true, so the files won't be kept."
|
|
147
|
+
" Set save_everything_on_error to keep them."
|
|
148
|
+
)
|
|
149
|
+
raise
|
|
150
|
+
|
|
151
|
+
# run the sorter regular with no tempdir, log exception to a
|
|
152
|
+
# traceback file in case of a crash for debugging
|
|
153
|
+
try:
|
|
154
|
+
return _dartsort_impl(recording, output_dir, cfg, motion_est, None, overwrite)
|
|
155
|
+
except Exception as e:
|
|
156
|
+
traceback_path = output_dir / "traceback.txt"
|
|
157
|
+
with open(traceback_path, "w") as f:
|
|
158
|
+
traceback.print_exception(e, file=f)
|
|
159
|
+
logger.exception(e)
|
|
160
|
+
logger.critical(f"Hit an error. Wrote traceback to {traceback_path}.")
|
|
161
|
+
raise
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _dartsort_impl(
|
|
165
|
+
recording: BaseRecording,
|
|
166
|
+
output_dir: Path,
|
|
167
|
+
cfg: DARTsortInternalConfig = default_dartsort_cfg,
|
|
168
|
+
motion_est=None,
|
|
169
|
+
work_dir=None,
|
|
170
|
+
overwrite=False,
|
|
171
|
+
):
|
|
172
|
+
"""Internal helper function which implements dartsort's main logic."""
|
|
173
|
+
ret = {}
|
|
174
|
+
|
|
175
|
+
# are we storing files in the work dir or the output dir?
|
|
176
|
+
store_dir = output_dir if work_dir is None else work_dir
|
|
177
|
+
|
|
178
|
+
# if there are previous results stored, resume where they leave off
|
|
179
|
+
# TODO uhh. overwrite, right?
|
|
180
|
+
next_step, sorting, motion_est = ds_fast_forward(store_dir, cfg)
|
|
181
|
+
ret["motion_est"] = motion_est
|
|
182
|
+
|
|
183
|
+
if next_step == 0:
|
|
184
|
+
# first step: initial detection and motion estimation
|
|
185
|
+
sorting = initial_detection(
|
|
186
|
+
output_dir=store_dir,
|
|
187
|
+
recording=recording,
|
|
188
|
+
cfg=cfg,
|
|
189
|
+
overwrite=overwrite,
|
|
190
|
+
motion_est=motion_est,
|
|
191
|
+
)
|
|
192
|
+
assert sorting is not None
|
|
193
|
+
logger.info(f"Initial detection: {sorting}")
|
|
194
|
+
is_final = cfg.detect_only or cfg.dredge_only or not cfg.matching_iterations
|
|
195
|
+
ds_save_features(cfg, sorting, output_dir, work_dir, is_final=is_final)
|
|
196
|
+
|
|
197
|
+
if cfg.detect_only:
|
|
198
|
+
ret["sorting"] = sorting
|
|
199
|
+
return ret
|
|
200
|
+
|
|
201
|
+
if motion_est is None:
|
|
202
|
+
logger.dartsortdebug("-- Estimate motion")
|
|
203
|
+
motion_est = estimate_motion(
|
|
204
|
+
output_directory=store_dir,
|
|
205
|
+
recording=recording,
|
|
206
|
+
sorting=sorting,
|
|
207
|
+
overwrite=overwrite,
|
|
208
|
+
device=cfg.computation_cfg.actual_device(),
|
|
209
|
+
**asdict(cfg.motion_estimation_cfg),
|
|
210
|
+
)
|
|
211
|
+
ret["motion_est"] = motion_est
|
|
212
|
+
ds_save_motion_est(motion_est, output_dir, work_dir, overwrite)
|
|
213
|
+
|
|
214
|
+
if cfg.dredge_only:
|
|
215
|
+
ret["sorting"] = sorting
|
|
216
|
+
return ret
|
|
217
|
+
|
|
218
|
+
# clustering: initialization and first refinement
|
|
219
|
+
r_cfgs = [
|
|
220
|
+
cfg.pre_refinement_cfg,
|
|
221
|
+
cfg.initial_refinement_cfg,
|
|
222
|
+
cfg.post_refinement_cfg,
|
|
223
|
+
]
|
|
224
|
+
sorting = cluster(
|
|
225
|
+
recording,
|
|
226
|
+
sorting,
|
|
227
|
+
motion_est=motion_est,
|
|
228
|
+
refinement_cfgs=r_cfgs,
|
|
229
|
+
clustering_cfg=cfg.clustering_cfg,
|
|
230
|
+
clustering_features_cfg=cfg.clustering_features_cfg,
|
|
231
|
+
_save_cfg=cfg,
|
|
232
|
+
_save_dir=output_dir,
|
|
233
|
+
)
|
|
234
|
+
logger.info(f"First clustering: {sorting}")
|
|
235
|
+
ds_save_intermediate_labels(
|
|
236
|
+
step_name="refined0",
|
|
237
|
+
step_sorting=sorting,
|
|
238
|
+
output_dir=output_dir,
|
|
239
|
+
cfg=cfg,
|
|
240
|
+
work_dir=work_dir,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
# be sure to start matching at step 1
|
|
244
|
+
next_step += 1
|
|
245
|
+
|
|
246
|
+
assert sorting is not None
|
|
247
|
+
assert (motion_est is not None) == cfg.motion_estimation_cfg.do_motion_estimation
|
|
248
|
+
assert next_step > 0 # matching starts at 1
|
|
249
|
+
|
|
250
|
+
for step in range(next_step, cfg.matching_iterations + 1):
|
|
251
|
+
is_final = step == cfg.matching_iterations
|
|
252
|
+
|
|
253
|
+
if step == 0:
|
|
254
|
+
assert False
|
|
255
|
+
elif step == 1:
|
|
256
|
+
previous_detection_cfg = cfg.initial_detection_cfg
|
|
257
|
+
else:
|
|
258
|
+
previous_detection_cfg = cfg.matching_cfg
|
|
259
|
+
|
|
260
|
+
# TODO
|
|
261
|
+
# prop = 1.0 if is_final else cfg.intermediate_matching_subsampling
|
|
262
|
+
|
|
263
|
+
logger.dartsortdebug(f"-- Matching {step}")
|
|
264
|
+
sorting = match(
|
|
265
|
+
output_dir=store_dir,
|
|
266
|
+
recording=recording,
|
|
267
|
+
sorting=sorting,
|
|
268
|
+
motion_est=motion_est,
|
|
269
|
+
template_cfg=cfg.template_cfg,
|
|
270
|
+
waveform_cfg=cfg.waveform_cfg,
|
|
271
|
+
featurization_cfg=cfg.featurization_cfg,
|
|
272
|
+
matching_cfg=cfg.matching_cfg,
|
|
273
|
+
overwrite=overwrite,
|
|
274
|
+
previous_detection_cfg=previous_detection_cfg,
|
|
275
|
+
computation_cfg=cfg.computation_cfg,
|
|
276
|
+
hdf5_filename=f"matching{step}.h5",
|
|
277
|
+
model_subdir=f"matching{step}_models",
|
|
278
|
+
)
|
|
279
|
+
logger.info(f"Matching step {step}: {sorting}")
|
|
280
|
+
ds_save_features(cfg, sorting, output_dir, work_dir, is_final)
|
|
281
|
+
|
|
282
|
+
if is_final and not cfg.final_refinement:
|
|
283
|
+
break
|
|
284
|
+
|
|
285
|
+
if cfg.recluster_after_first_matching:
|
|
286
|
+
step_clustering_cfg = cfg.clustering_cfg
|
|
287
|
+
step_features_cfg = cfg.clustering_features_cfg
|
|
288
|
+
else:
|
|
289
|
+
step_clustering_cfg = step_features_cfg = None
|
|
290
|
+
|
|
291
|
+
r_cfgs = [
|
|
292
|
+
cfg.pre_refinement_cfg,
|
|
293
|
+
cfg.refinement_cfg,
|
|
294
|
+
cfg.post_refinement_cfg,
|
|
295
|
+
]
|
|
296
|
+
|
|
297
|
+
sorting = cluster(
|
|
298
|
+
recording,
|
|
299
|
+
sorting,
|
|
300
|
+
motion_est=motion_est,
|
|
301
|
+
refinement_cfgs=r_cfgs,
|
|
302
|
+
clustering_cfg=step_clustering_cfg,
|
|
303
|
+
clustering_features_cfg=step_features_cfg,
|
|
304
|
+
_save_cfg=cfg,
|
|
305
|
+
_save_dir=output_dir,
|
|
306
|
+
_save_initial_name=f"recluster{step}",
|
|
307
|
+
_save_refined_name_fmt=f"refined{step}{{stepname}}",
|
|
308
|
+
)
|
|
309
|
+
ds_save_intermediate_labels(
|
|
310
|
+
step_name=f"refined{step}",
|
|
311
|
+
step_sorting=sorting,
|
|
312
|
+
output_dir=output_dir,
|
|
313
|
+
cfg=cfg,
|
|
314
|
+
work_dir=work_dir,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
# finally handle scratch directory and delete intermediate files if requested
|
|
318
|
+
if work_dir is not None:
|
|
319
|
+
orig_h5_path = resolve_path(sorting.parent_h5_path, strict=True)
|
|
320
|
+
final_h5_path = output_dir / orig_h5_path.name
|
|
321
|
+
assert final_h5_path.exists()
|
|
322
|
+
sorting.parent_h5_path = final_h5_path
|
|
323
|
+
ds_handle_delete_intermediate_features(cfg, sorting, output_dir, work_dir)
|
|
324
|
+
|
|
325
|
+
sorting.save(output_dir / "dartsort_sorting.npz")
|
|
326
|
+
ret["sorting"] = sorting
|
|
327
|
+
|
|
328
|
+
return ret
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def initial_detection(
|
|
332
|
+
output_dir: str | Path,
|
|
333
|
+
recording,
|
|
334
|
+
cfg: DARTsortInternalConfig,
|
|
335
|
+
motion_est=None,
|
|
336
|
+
overwrite=False,
|
|
337
|
+
show_progress=True,
|
|
338
|
+
):
|
|
339
|
+
if cfg.detection_type == "subtract":
|
|
340
|
+
assert isinstance(cfg.initial_detection_cfg, SubtractionConfig)
|
|
341
|
+
return subtract(
|
|
342
|
+
output_dir=output_dir,
|
|
343
|
+
recording=recording,
|
|
344
|
+
waveform_cfg=cfg.waveform_cfg,
|
|
345
|
+
featurization_cfg=cfg.featurization_cfg,
|
|
346
|
+
subtraction_cfg=cfg.initial_detection_cfg,
|
|
347
|
+
sampling_cfg=cfg.peeler_sampling_cfg,
|
|
348
|
+
computation_cfg=cfg.computation_cfg,
|
|
349
|
+
overwrite=overwrite,
|
|
350
|
+
show_progress=show_progress,
|
|
351
|
+
)
|
|
352
|
+
elif cfg.detection_type == "threshold":
|
|
353
|
+
assert isinstance(cfg.initial_detection_cfg, ThresholdingConfig)
|
|
354
|
+
return threshold(
|
|
355
|
+
output_dir=output_dir,
|
|
356
|
+
recording=recording,
|
|
357
|
+
waveform_cfg=cfg.waveform_cfg,
|
|
358
|
+
thresholding_cfg=cfg.initial_detection_cfg,
|
|
359
|
+
sampling_cfg=cfg.peeler_sampling_cfg,
|
|
360
|
+
featurization_cfg=cfg.featurization_cfg,
|
|
361
|
+
overwrite=overwrite,
|
|
362
|
+
show_progress=show_progress,
|
|
363
|
+
computation_cfg=cfg.computation_cfg,
|
|
364
|
+
)
|
|
365
|
+
elif cfg.detection_type == "match":
|
|
366
|
+
assert isinstance(cfg.initial_detection_cfg, MatchingConfig)
|
|
367
|
+
return match(
|
|
368
|
+
output_dir=output_dir,
|
|
369
|
+
recording=recording,
|
|
370
|
+
waveform_cfg=cfg.waveform_cfg,
|
|
371
|
+
template_cfg=cfg.template_cfg,
|
|
372
|
+
featurization_cfg=cfg.featurization_cfg,
|
|
373
|
+
matching_cfg=cfg.initial_detection_cfg,
|
|
374
|
+
sampling_cfg=cfg.peeler_sampling_cfg,
|
|
375
|
+
motion_est=motion_est,
|
|
376
|
+
overwrite=overwrite,
|
|
377
|
+
show_progress=show_progress,
|
|
378
|
+
computation_cfg=cfg.computation_cfg,
|
|
379
|
+
)
|
|
380
|
+
elif cfg.detection_type == "universal":
|
|
381
|
+
assert isinstance(cfg.initial_detection_cfg, UniversalMatchingConfig)
|
|
382
|
+
return universal_match(
|
|
383
|
+
output_dir=output_dir,
|
|
384
|
+
recording=recording,
|
|
385
|
+
universal_cfg=cfg.initial_detection_cfg,
|
|
386
|
+
sampling_cfg=cfg.peeler_sampling_cfg,
|
|
387
|
+
featurization_cfg=cfg.featurization_cfg,
|
|
388
|
+
overwrite=overwrite,
|
|
389
|
+
show_progress=show_progress,
|
|
390
|
+
computation_cfg=cfg.computation_cfg,
|
|
391
|
+
)
|
|
392
|
+
else:
|
|
393
|
+
raise ValueError(f"Unknown detection_type {cfg.detection_type}.")
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def subtract(
|
|
397
|
+
output_dir: str | Path,
|
|
398
|
+
recording: BaseRecording,
|
|
399
|
+
waveform_cfg=default_waveform_cfg,
|
|
400
|
+
featurization_cfg=default_featurization_cfg,
|
|
401
|
+
subtraction_cfg=default_subtraction_cfg,
|
|
402
|
+
sampling_cfg=default_peeling_fit_sampling_cfg,
|
|
403
|
+
computation_cfg: ComputationConfig | None = None,
|
|
404
|
+
chunk_starts_samples=None,
|
|
405
|
+
overwrite=False,
|
|
406
|
+
residual_filename: str | None = None,
|
|
407
|
+
show_progress=True,
|
|
408
|
+
hdf5_filename="subtraction.h5",
|
|
409
|
+
model_subdir="subtraction_models",
|
|
410
|
+
) -> DARTsortSorting | None:
|
|
411
|
+
output_dir = resolve_path(output_dir)
|
|
412
|
+
check_recording(recording)
|
|
413
|
+
subtraction_peeler = SubtractionPeeler.from_config(
|
|
414
|
+
recording=recording,
|
|
415
|
+
sampling_cfg=sampling_cfg,
|
|
416
|
+
waveform_cfg=waveform_cfg,
|
|
417
|
+
subtraction_cfg=subtraction_cfg,
|
|
418
|
+
featurization_cfg=featurization_cfg,
|
|
419
|
+
)
|
|
420
|
+
detections = run_peeler(
|
|
421
|
+
subtraction_peeler,
|
|
422
|
+
output_directory=output_dir,
|
|
423
|
+
hdf5_filename=hdf5_filename,
|
|
424
|
+
model_subdir=model_subdir,
|
|
425
|
+
featurization_cfg=featurization_cfg,
|
|
426
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
427
|
+
overwrite=overwrite,
|
|
428
|
+
computation_cfg=computation_cfg,
|
|
429
|
+
residual_filename=residual_filename,
|
|
430
|
+
show_progress=show_progress,
|
|
431
|
+
fit_only=subtraction_cfg.fit_only,
|
|
432
|
+
)
|
|
433
|
+
return detections
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def match(
|
|
437
|
+
output_dir: str | Path,
|
|
438
|
+
recording: BaseRecording,
|
|
439
|
+
sorting: DARTsortSorting | None = None,
|
|
440
|
+
motion_est=None,
|
|
441
|
+
waveform_cfg=default_waveform_cfg,
|
|
442
|
+
template_cfg=default_template_cfg,
|
|
443
|
+
featurization_cfg=default_featurization_cfg,
|
|
444
|
+
matching_cfg=default_matching_cfg,
|
|
445
|
+
sampling_cfg=default_peeling_fit_sampling_cfg,
|
|
446
|
+
previous_detection_cfg: Any | None = None,
|
|
447
|
+
chunk_starts_samples=None,
|
|
448
|
+
overwrite=False,
|
|
449
|
+
residual_filename: str | None = None,
|
|
450
|
+
show_progress=True,
|
|
451
|
+
hdf5_filename="matching0.h5",
|
|
452
|
+
model_subdir="matching0_models",
|
|
453
|
+
template_data: TemplateData | None = None,
|
|
454
|
+
template_npz_filename="template_data.npz",
|
|
455
|
+
computation_cfg: ComputationConfig | None = None,
|
|
456
|
+
template_denoising_tsvd=None,
|
|
457
|
+
) -> DARTsortSorting:
|
|
458
|
+
output_dir = resolve_path(output_dir)
|
|
459
|
+
model_dir = output_dir / model_subdir
|
|
460
|
+
|
|
461
|
+
if template_data is None and not matching_cfg.precomputed_templates_npz:
|
|
462
|
+
assert sorting is not None
|
|
463
|
+
sorting, template_data = estimate_template_library(
|
|
464
|
+
recording=recording,
|
|
465
|
+
sorting=sorting,
|
|
466
|
+
motion_est=motion_est,
|
|
467
|
+
min_template_ptp=matching_cfg.min_template_ptp,
|
|
468
|
+
min_template_snr=matching_cfg.min_template_snr,
|
|
469
|
+
min_template_count=matching_cfg.min_template_count,
|
|
470
|
+
max_cc_flag_rate=matching_cfg.max_cc_flag_rate,
|
|
471
|
+
cc_flag_entropy_cutoff=matching_cfg.cc_flag_entropy_cutoff,
|
|
472
|
+
depth_order=matching_cfg.depth_order,
|
|
473
|
+
waveform_cfg=waveform_cfg,
|
|
474
|
+
template_cfg=template_cfg,
|
|
475
|
+
realign_cfg=matching_cfg.template_realignment_cfg,
|
|
476
|
+
template_merge_cfg=matching_cfg.template_merge_cfg,
|
|
477
|
+
computation_cfg=computation_cfg,
|
|
478
|
+
detection_cfg=previous_detection_cfg,
|
|
479
|
+
tsvd=template_denoising_tsvd,
|
|
480
|
+
template_npz_path=model_dir / template_npz_filename,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
matching_peeler = ObjectiveUpdateTemplateMatchingPeeler.from_config(
|
|
484
|
+
recording=recording,
|
|
485
|
+
waveform_cfg=waveform_cfg,
|
|
486
|
+
matching_cfg=matching_cfg,
|
|
487
|
+
sampling_cfg=sampling_cfg,
|
|
488
|
+
featurization_cfg=featurization_cfg,
|
|
489
|
+
template_data=template_data,
|
|
490
|
+
motion_est=motion_est,
|
|
491
|
+
parent_sorting_hdf5_path=getattr(sorting, "parent_h5_path", None),
|
|
492
|
+
)
|
|
493
|
+
sorting = run_peeler(
|
|
494
|
+
matching_peeler,
|
|
495
|
+
output_directory=output_dir,
|
|
496
|
+
hdf5_filename=hdf5_filename,
|
|
497
|
+
model_subdir=model_subdir,
|
|
498
|
+
featurization_cfg=featurization_cfg,
|
|
499
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
500
|
+
overwrite=overwrite,
|
|
501
|
+
residual_filename=residual_filename,
|
|
502
|
+
show_progress=show_progress,
|
|
503
|
+
computation_cfg=computation_cfg,
|
|
504
|
+
)
|
|
505
|
+
return sorting
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def grab(
|
|
509
|
+
output_dir: str | Path,
|
|
510
|
+
recording: BaseRecording,
|
|
511
|
+
sorting: DARTsortSorting,
|
|
512
|
+
waveform_cfg=default_waveform_cfg,
|
|
513
|
+
featurization_cfg=default_featurization_cfg,
|
|
514
|
+
sampling_cfg=default_peeling_fit_sampling_cfg,
|
|
515
|
+
chunk_starts_samples=None,
|
|
516
|
+
overwrite=False,
|
|
517
|
+
show_progress=True,
|
|
518
|
+
hdf5_filename="grab.h5",
|
|
519
|
+
model_subdir="grab_models",
|
|
520
|
+
computation_cfg: ComputationConfig | None = None,
|
|
521
|
+
) -> DARTsortSorting:
|
|
522
|
+
output_dir = resolve_path(output_dir)
|
|
523
|
+
grabber = GrabAndFeaturize.from_config(
|
|
524
|
+
sorting=sorting,
|
|
525
|
+
recording=recording,
|
|
526
|
+
waveform_cfg=waveform_cfg,
|
|
527
|
+
sampling_cfg=sampling_cfg,
|
|
528
|
+
featurization_cfg=featurization_cfg,
|
|
529
|
+
)
|
|
530
|
+
sorting = run_peeler(
|
|
531
|
+
grabber,
|
|
532
|
+
output_directory=output_dir,
|
|
533
|
+
hdf5_filename=hdf5_filename,
|
|
534
|
+
model_subdir=model_subdir,
|
|
535
|
+
featurization_cfg=featurization_cfg,
|
|
536
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
537
|
+
overwrite=overwrite,
|
|
538
|
+
show_progress=show_progress,
|
|
539
|
+
computation_cfg=computation_cfg,
|
|
540
|
+
)
|
|
541
|
+
return sorting
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def threshold(
|
|
545
|
+
output_dir: str | Path,
|
|
546
|
+
recording: BaseRecording,
|
|
547
|
+
waveform_cfg=default_waveform_cfg,
|
|
548
|
+
thresholding_cfg=default_thresholding_cfg,
|
|
549
|
+
featurization_cfg=default_featurization_cfg,
|
|
550
|
+
sampling_cfg=default_peeling_fit_sampling_cfg,
|
|
551
|
+
chunk_starts_samples=None,
|
|
552
|
+
overwrite=False,
|
|
553
|
+
show_progress=True,
|
|
554
|
+
hdf5_filename="threshold.h5",
|
|
555
|
+
model_subdir="threshold_models",
|
|
556
|
+
computation_cfg: ComputationConfig | None = None,
|
|
557
|
+
) -> DARTsortSorting:
|
|
558
|
+
output_dir = resolve_path(output_dir)
|
|
559
|
+
thresholder = ThresholdAndFeaturize.from_config(
|
|
560
|
+
recording=recording,
|
|
561
|
+
waveform_cfg=waveform_cfg,
|
|
562
|
+
thresholding_cfg=thresholding_cfg,
|
|
563
|
+
featurization_cfg=featurization_cfg,
|
|
564
|
+
sampling_cfg=sampling_cfg,
|
|
565
|
+
)
|
|
566
|
+
sorting = run_peeler(
|
|
567
|
+
thresholder,
|
|
568
|
+
output_directory=output_dir,
|
|
569
|
+
hdf5_filename=hdf5_filename,
|
|
570
|
+
model_subdir=model_subdir,
|
|
571
|
+
featurization_cfg=featurization_cfg,
|
|
572
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
573
|
+
overwrite=overwrite,
|
|
574
|
+
show_progress=show_progress,
|
|
575
|
+
computation_cfg=computation_cfg,
|
|
576
|
+
)
|
|
577
|
+
return sorting
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def cluster(
|
|
581
|
+
recording,
|
|
582
|
+
sorting,
|
|
583
|
+
motion_est=None,
|
|
584
|
+
clustering_cfg: ClusteringConfig | None = default_clustering_cfg,
|
|
585
|
+
clustering_features_cfg: (
|
|
586
|
+
ClusteringFeaturesConfig | None
|
|
587
|
+
) = default_clustering_features_cfg,
|
|
588
|
+
refinement_cfgs: list[RefinementConfig | None] | None = None,
|
|
589
|
+
computation_cfg: ComputationConfig | None = None,
|
|
590
|
+
features: SimpleMatrixFeatures | None = None,
|
|
591
|
+
*,
|
|
592
|
+
_save_cfg: DARTsortInternalConfig | None = None,
|
|
593
|
+
_save_initial_name="initial",
|
|
594
|
+
_save_refined_name_fmt="refined0{stepname}",
|
|
595
|
+
_save_dir=None,
|
|
596
|
+
):
|
|
597
|
+
if features is None:
|
|
598
|
+
features = get_clustering_features(
|
|
599
|
+
recording,
|
|
600
|
+
sorting,
|
|
601
|
+
motion_est=motion_est,
|
|
602
|
+
clustering_features_cfg=clustering_features_cfg,
|
|
603
|
+
)
|
|
604
|
+
assert features is not None
|
|
605
|
+
clusterer = get_clusterer(
|
|
606
|
+
clustering_cfg=clustering_cfg,
|
|
607
|
+
refinement_cfgs=refinement_cfgs,
|
|
608
|
+
computation_cfg=computation_cfg,
|
|
609
|
+
save_cfg=_save_cfg,
|
|
610
|
+
save_labels_dir=_save_dir,
|
|
611
|
+
initial_name=_save_initial_name,
|
|
612
|
+
refine_labels_fmt=_save_refined_name_fmt,
|
|
613
|
+
)
|
|
614
|
+
result = clusterer.cluster(
|
|
615
|
+
recording=recording, sorting=sorting, features=features, motion_est=motion_est
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
del features, clusterer
|
|
619
|
+
gc.collect()
|
|
620
|
+
torch.cuda.empty_cache()
|
|
621
|
+
|
|
622
|
+
return result
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def universal_match(
|
|
626
|
+
output_dir: str | Path,
|
|
627
|
+
recording: BaseRecording,
|
|
628
|
+
universal_cfg=default_universal_cfg,
|
|
629
|
+
featurization_cfg=default_featurization_cfg,
|
|
630
|
+
sampling_cfg=default_peeling_fit_sampling_cfg,
|
|
631
|
+
chunk_starts_samples=None,
|
|
632
|
+
overwrite=False,
|
|
633
|
+
show_progress=True,
|
|
634
|
+
hdf5_filename="universal.h5",
|
|
635
|
+
model_subdir="universal_models",
|
|
636
|
+
computation_cfg: ComputationConfig | None = None,
|
|
637
|
+
) -> DARTsortSorting:
|
|
638
|
+
output_dir = resolve_path(output_dir)
|
|
639
|
+
universal_matcher = UniversalTemplatesMatchingPeeler.from_config(
|
|
640
|
+
recording=recording,
|
|
641
|
+
universal_cfg=universal_cfg,
|
|
642
|
+
waveform_cfg=universal_cfg.waveform_cfg, # TODO...
|
|
643
|
+
featurization_cfg=featurization_cfg,
|
|
644
|
+
sampling_cfg=sampling_cfg,
|
|
645
|
+
)
|
|
646
|
+
sorting = run_peeler(
|
|
647
|
+
universal_matcher,
|
|
648
|
+
output_directory=output_dir,
|
|
649
|
+
hdf5_filename=hdf5_filename,
|
|
650
|
+
model_subdir=model_subdir,
|
|
651
|
+
featurization_cfg=featurization_cfg,
|
|
652
|
+
chunk_starts_samples=chunk_starts_samples,
|
|
653
|
+
overwrite=overwrite,
|
|
654
|
+
show_progress=show_progress,
|
|
655
|
+
computation_cfg=computation_cfg,
|
|
656
|
+
)
|
|
657
|
+
return sorting
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from .grab import GrabAndFeaturize
|
|
2
|
+
from .matching import ObjectiveUpdateTemplateMatchingPeeler
|
|
3
|
+
from .subtract import SubtractionPeeler, subtract_chunk
|
|
4
|
+
from .threshold import ThresholdAndFeaturize
|
|
5
|
+
from .universal import UniversalTemplatesMatchingPeeler
|
|
6
|
+
from .matching_util import *
|