DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/main.py ADDED
@@ -0,0 +1,657 @@
1
+ import gc
2
+ import traceback
3
+ from dataclasses import asdict
4
+ from pathlib import Path
5
+ from tempfile import TemporaryDirectory
6
+ from typing import Any
7
+
8
+ import torch
9
+ from spikeinterface.core import BaseRecording
10
+
11
+ from .clustering import SimpleMatrixFeatures, get_clusterer, get_clustering_features
12
+ from .config import DARTsortUserConfig, DeveloperConfig
13
+ from .peel import (
14
+ GrabAndFeaturize,
15
+ ObjectiveUpdateTemplateMatchingPeeler,
16
+ SubtractionPeeler,
17
+ ThresholdAndFeaturize,
18
+ UniversalTemplatesMatchingPeeler,
19
+ )
20
+ from .templates import TemplateData, estimate_template_library
21
+ from .util.data_util import DARTsortSorting, check_recording
22
+ from .util.internal_config import (
23
+ ClusteringConfig,
24
+ ClusteringFeaturesConfig,
25
+ ComputationConfig,
26
+ DARTsortInternalConfig,
27
+ MatchingConfig,
28
+ RefinementConfig,
29
+ SubtractionConfig,
30
+ ThresholdingConfig,
31
+ UniversalMatchingConfig,
32
+ default_clustering_cfg,
33
+ default_clustering_features_cfg,
34
+ default_dartsort_cfg,
35
+ default_featurization_cfg,
36
+ default_matching_cfg,
37
+ default_peeling_fit_sampling_cfg,
38
+ default_subtraction_cfg,
39
+ default_template_cfg,
40
+ default_thresholding_cfg,
41
+ default_universal_cfg,
42
+ default_waveform_cfg,
43
+ to_internal_config,
44
+ )
45
+ from .util.logging_util import get_logger
46
+ from .util.main_util import (
47
+ ds_all_to_workdir,
48
+ ds_dump_config,
49
+ ds_fast_forward,
50
+ ds_handle_delete_intermediate_features,
51
+ ds_save_features,
52
+ ds_save_intermediate_labels,
53
+ ds_save_motion_est,
54
+ )
55
+ from .util.peel_util import run_peeler
56
+ from .util.py_util import dartcopytree, resolve_path
57
+ from .util.registration_util import estimate_motion
58
+
59
+ logger = get_logger(__name__)
60
+
61
+
62
+ def dartsort(
63
+ recording: BaseRecording,
64
+ output_dir: str | Path,
65
+ cfg: (
66
+ DARTsortUserConfig | str | Path | DeveloperConfig | DARTsortInternalConfig
67
+ ) = default_dartsort_cfg,
68
+ motion_est=None,
69
+ overwrite=False,
70
+ ):
71
+ """dartsort
72
+
73
+ This function runs a spike sorter called dartsort.
74
+
75
+ Arguments
76
+ ---------
77
+ recording : BaseRecording
78
+ A spikeinterface.BaseRecording object
79
+ output_dir: str | Path
80
+ Folder where outputs are stored
81
+ cfg: DARTsortUserConfig | DARTsortInternalConfig | str | Path
82
+ Your settings. Either create a DARTsortUserConfig directly in code, or
83
+ you can pass a string or Path pointing to a .toml file here.
84
+ motion_est: optional dredge.MotionEstimate
85
+ This is meant to allow users to pass their own external motion estimate.
86
+ To do: support spikeinterface Motion objects here.
87
+ overwrite : bool, default=False
88
+ Ignore and overwrite stored results, if any. Otherwise, dartsort will
89
+ try to resume from the last step that ran, or if it had finished then
90
+ it will do nothing.
91
+
92
+ Returns
93
+ -------
94
+ Dictionary of sorting results, with the following keys:
95
+ - "sorting": DARTsortSorting
96
+ This has a sorting.to_numpy_sorting() method for those who want to
97
+ export back to spikeinterface. Which would be everyone? :)
98
+ Alternatively, you could visualize your results using the functions
99
+ in the `import dartsort.vis as dartvis` library.
100
+ - "motion_est": dredge.MotionEstimate
101
+
102
+ TODO: add the key "motion" with a spikeinterface Motion object.
103
+ """
104
+ output_dir = resolve_path(output_dir)
105
+ output_dir.mkdir(exist_ok=True)
106
+
107
+ # convert cfg to internal format and store it for posterity
108
+ cfg = to_internal_config(cfg)
109
+ ds_dump_config(cfg, output_dir)
110
+
111
+ if cfg.work_in_tmpdir:
112
+ with TemporaryDirectory(prefix="dartsort", dir=cfg.tmpdir_parent) as work_dir:
113
+ # copy files and possibly recording to temporary directory
114
+ work_dir = resolve_path(work_dir)
115
+ logger.dartsortdebug(f"Working in {work_dir}, outputs to {output_dir}.")
116
+ recording, work_dir = ds_all_to_workdir(
117
+ internal_cfg=cfg,
118
+ output_dir=output_dir,
119
+ work_dir=work_dir,
120
+ recording=recording,
121
+ overwrite=overwrite,
122
+ )
123
+ assert work_dir is not None
124
+
125
+ # run the sorter, with extra error handlers for grabbing stuff from
126
+ # the temporary directory if user asked for that.
127
+ try:
128
+ return _dartsort_impl(
129
+ recording, output_dir, cfg, motion_est, work_dir, overwrite
130
+ )
131
+ except Exception as e:
132
+ traceback_path = output_dir / "traceback.txt"
133
+ error_data_path = output_dir / "error_state"
134
+ with open(traceback_path, "w") as f:
135
+ traceback.print_exception(e, file=f)
136
+ logger.exception(e)
137
+ if cfg.save_everything_on_error:
138
+ logger.critical(
139
+ f"Hit an error. Copying outputs to {error_data_path} "
140
+ f"and writing traceback to {traceback_path}."
141
+ )
142
+ dartcopytree(cfg, work_dir, error_data_path)
143
+ else:
144
+ logger.critical(
145
+ f"Hit an error. Writing traceback to {traceback_path}."
146
+ " work_in_tmpdir was true, so the files won't be kept."
147
+ " Set save_everything_on_error to keep them."
148
+ )
149
+ raise
150
+
151
+ # run the sorter regular with no tempdir, log exception to a
152
+ # traceback file in case of a crash for debugging
153
+ try:
154
+ return _dartsort_impl(recording, output_dir, cfg, motion_est, None, overwrite)
155
+ except Exception as e:
156
+ traceback_path = output_dir / "traceback.txt"
157
+ with open(traceback_path, "w") as f:
158
+ traceback.print_exception(e, file=f)
159
+ logger.exception(e)
160
+ logger.critical(f"Hit an error. Wrote traceback to {traceback_path}.")
161
+ raise
162
+
163
+
164
+ def _dartsort_impl(
165
+ recording: BaseRecording,
166
+ output_dir: Path,
167
+ cfg: DARTsortInternalConfig = default_dartsort_cfg,
168
+ motion_est=None,
169
+ work_dir=None,
170
+ overwrite=False,
171
+ ):
172
+ """Internal helper function which implements dartsort's main logic."""
173
+ ret = {}
174
+
175
+ # are we storing files in the work dir or the output dir?
176
+ store_dir = output_dir if work_dir is None else work_dir
177
+
178
+ # if there are previous results stored, resume where they leave off
179
+ # TODO uhh. overwrite, right?
180
+ next_step, sorting, motion_est = ds_fast_forward(store_dir, cfg)
181
+ ret["motion_est"] = motion_est
182
+
183
+ if next_step == 0:
184
+ # first step: initial detection and motion estimation
185
+ sorting = initial_detection(
186
+ output_dir=store_dir,
187
+ recording=recording,
188
+ cfg=cfg,
189
+ overwrite=overwrite,
190
+ motion_est=motion_est,
191
+ )
192
+ assert sorting is not None
193
+ logger.info(f"Initial detection: {sorting}")
194
+ is_final = cfg.detect_only or cfg.dredge_only or not cfg.matching_iterations
195
+ ds_save_features(cfg, sorting, output_dir, work_dir, is_final=is_final)
196
+
197
+ if cfg.detect_only:
198
+ ret["sorting"] = sorting
199
+ return ret
200
+
201
+ if motion_est is None:
202
+ logger.dartsortdebug("-- Estimate motion")
203
+ motion_est = estimate_motion(
204
+ output_directory=store_dir,
205
+ recording=recording,
206
+ sorting=sorting,
207
+ overwrite=overwrite,
208
+ device=cfg.computation_cfg.actual_device(),
209
+ **asdict(cfg.motion_estimation_cfg),
210
+ )
211
+ ret["motion_est"] = motion_est
212
+ ds_save_motion_est(motion_est, output_dir, work_dir, overwrite)
213
+
214
+ if cfg.dredge_only:
215
+ ret["sorting"] = sorting
216
+ return ret
217
+
218
+ # clustering: initialization and first refinement
219
+ r_cfgs = [
220
+ cfg.pre_refinement_cfg,
221
+ cfg.initial_refinement_cfg,
222
+ cfg.post_refinement_cfg,
223
+ ]
224
+ sorting = cluster(
225
+ recording,
226
+ sorting,
227
+ motion_est=motion_est,
228
+ refinement_cfgs=r_cfgs,
229
+ clustering_cfg=cfg.clustering_cfg,
230
+ clustering_features_cfg=cfg.clustering_features_cfg,
231
+ _save_cfg=cfg,
232
+ _save_dir=output_dir,
233
+ )
234
+ logger.info(f"First clustering: {sorting}")
235
+ ds_save_intermediate_labels(
236
+ step_name="refined0",
237
+ step_sorting=sorting,
238
+ output_dir=output_dir,
239
+ cfg=cfg,
240
+ work_dir=work_dir,
241
+ )
242
+
243
+ # be sure to start matching at step 1
244
+ next_step += 1
245
+
246
+ assert sorting is not None
247
+ assert (motion_est is not None) == cfg.motion_estimation_cfg.do_motion_estimation
248
+ assert next_step > 0 # matching starts at 1
249
+
250
+ for step in range(next_step, cfg.matching_iterations + 1):
251
+ is_final = step == cfg.matching_iterations
252
+
253
+ if step == 0:
254
+ assert False
255
+ elif step == 1:
256
+ previous_detection_cfg = cfg.initial_detection_cfg
257
+ else:
258
+ previous_detection_cfg = cfg.matching_cfg
259
+
260
+ # TODO
261
+ # prop = 1.0 if is_final else cfg.intermediate_matching_subsampling
262
+
263
+ logger.dartsortdebug(f"-- Matching {step}")
264
+ sorting = match(
265
+ output_dir=store_dir,
266
+ recording=recording,
267
+ sorting=sorting,
268
+ motion_est=motion_est,
269
+ template_cfg=cfg.template_cfg,
270
+ waveform_cfg=cfg.waveform_cfg,
271
+ featurization_cfg=cfg.featurization_cfg,
272
+ matching_cfg=cfg.matching_cfg,
273
+ overwrite=overwrite,
274
+ previous_detection_cfg=previous_detection_cfg,
275
+ computation_cfg=cfg.computation_cfg,
276
+ hdf5_filename=f"matching{step}.h5",
277
+ model_subdir=f"matching{step}_models",
278
+ )
279
+ logger.info(f"Matching step {step}: {sorting}")
280
+ ds_save_features(cfg, sorting, output_dir, work_dir, is_final)
281
+
282
+ if is_final and not cfg.final_refinement:
283
+ break
284
+
285
+ if cfg.recluster_after_first_matching:
286
+ step_clustering_cfg = cfg.clustering_cfg
287
+ step_features_cfg = cfg.clustering_features_cfg
288
+ else:
289
+ step_clustering_cfg = step_features_cfg = None
290
+
291
+ r_cfgs = [
292
+ cfg.pre_refinement_cfg,
293
+ cfg.refinement_cfg,
294
+ cfg.post_refinement_cfg,
295
+ ]
296
+
297
+ sorting = cluster(
298
+ recording,
299
+ sorting,
300
+ motion_est=motion_est,
301
+ refinement_cfgs=r_cfgs,
302
+ clustering_cfg=step_clustering_cfg,
303
+ clustering_features_cfg=step_features_cfg,
304
+ _save_cfg=cfg,
305
+ _save_dir=output_dir,
306
+ _save_initial_name=f"recluster{step}",
307
+ _save_refined_name_fmt=f"refined{step}{{stepname}}",
308
+ )
309
+ ds_save_intermediate_labels(
310
+ step_name=f"refined{step}",
311
+ step_sorting=sorting,
312
+ output_dir=output_dir,
313
+ cfg=cfg,
314
+ work_dir=work_dir,
315
+ )
316
+
317
+ # finally handle scratch directory and delete intermediate files if requested
318
+ if work_dir is not None:
319
+ orig_h5_path = resolve_path(sorting.parent_h5_path, strict=True)
320
+ final_h5_path = output_dir / orig_h5_path.name
321
+ assert final_h5_path.exists()
322
+ sorting.parent_h5_path = final_h5_path
323
+ ds_handle_delete_intermediate_features(cfg, sorting, output_dir, work_dir)
324
+
325
+ sorting.save(output_dir / "dartsort_sorting.npz")
326
+ ret["sorting"] = sorting
327
+
328
+ return ret
329
+
330
+
331
+ def initial_detection(
332
+ output_dir: str | Path,
333
+ recording,
334
+ cfg: DARTsortInternalConfig,
335
+ motion_est=None,
336
+ overwrite=False,
337
+ show_progress=True,
338
+ ):
339
+ if cfg.detection_type == "subtract":
340
+ assert isinstance(cfg.initial_detection_cfg, SubtractionConfig)
341
+ return subtract(
342
+ output_dir=output_dir,
343
+ recording=recording,
344
+ waveform_cfg=cfg.waveform_cfg,
345
+ featurization_cfg=cfg.featurization_cfg,
346
+ subtraction_cfg=cfg.initial_detection_cfg,
347
+ sampling_cfg=cfg.peeler_sampling_cfg,
348
+ computation_cfg=cfg.computation_cfg,
349
+ overwrite=overwrite,
350
+ show_progress=show_progress,
351
+ )
352
+ elif cfg.detection_type == "threshold":
353
+ assert isinstance(cfg.initial_detection_cfg, ThresholdingConfig)
354
+ return threshold(
355
+ output_dir=output_dir,
356
+ recording=recording,
357
+ waveform_cfg=cfg.waveform_cfg,
358
+ thresholding_cfg=cfg.initial_detection_cfg,
359
+ sampling_cfg=cfg.peeler_sampling_cfg,
360
+ featurization_cfg=cfg.featurization_cfg,
361
+ overwrite=overwrite,
362
+ show_progress=show_progress,
363
+ computation_cfg=cfg.computation_cfg,
364
+ )
365
+ elif cfg.detection_type == "match":
366
+ assert isinstance(cfg.initial_detection_cfg, MatchingConfig)
367
+ return match(
368
+ output_dir=output_dir,
369
+ recording=recording,
370
+ waveform_cfg=cfg.waveform_cfg,
371
+ template_cfg=cfg.template_cfg,
372
+ featurization_cfg=cfg.featurization_cfg,
373
+ matching_cfg=cfg.initial_detection_cfg,
374
+ sampling_cfg=cfg.peeler_sampling_cfg,
375
+ motion_est=motion_est,
376
+ overwrite=overwrite,
377
+ show_progress=show_progress,
378
+ computation_cfg=cfg.computation_cfg,
379
+ )
380
+ elif cfg.detection_type == "universal":
381
+ assert isinstance(cfg.initial_detection_cfg, UniversalMatchingConfig)
382
+ return universal_match(
383
+ output_dir=output_dir,
384
+ recording=recording,
385
+ universal_cfg=cfg.initial_detection_cfg,
386
+ sampling_cfg=cfg.peeler_sampling_cfg,
387
+ featurization_cfg=cfg.featurization_cfg,
388
+ overwrite=overwrite,
389
+ show_progress=show_progress,
390
+ computation_cfg=cfg.computation_cfg,
391
+ )
392
+ else:
393
+ raise ValueError(f"Unknown detection_type {cfg.detection_type}.")
394
+
395
+
396
+ def subtract(
397
+ output_dir: str | Path,
398
+ recording: BaseRecording,
399
+ waveform_cfg=default_waveform_cfg,
400
+ featurization_cfg=default_featurization_cfg,
401
+ subtraction_cfg=default_subtraction_cfg,
402
+ sampling_cfg=default_peeling_fit_sampling_cfg,
403
+ computation_cfg: ComputationConfig | None = None,
404
+ chunk_starts_samples=None,
405
+ overwrite=False,
406
+ residual_filename: str | None = None,
407
+ show_progress=True,
408
+ hdf5_filename="subtraction.h5",
409
+ model_subdir="subtraction_models",
410
+ ) -> DARTsortSorting | None:
411
+ output_dir = resolve_path(output_dir)
412
+ check_recording(recording)
413
+ subtraction_peeler = SubtractionPeeler.from_config(
414
+ recording=recording,
415
+ sampling_cfg=sampling_cfg,
416
+ waveform_cfg=waveform_cfg,
417
+ subtraction_cfg=subtraction_cfg,
418
+ featurization_cfg=featurization_cfg,
419
+ )
420
+ detections = run_peeler(
421
+ subtraction_peeler,
422
+ output_directory=output_dir,
423
+ hdf5_filename=hdf5_filename,
424
+ model_subdir=model_subdir,
425
+ featurization_cfg=featurization_cfg,
426
+ chunk_starts_samples=chunk_starts_samples,
427
+ overwrite=overwrite,
428
+ computation_cfg=computation_cfg,
429
+ residual_filename=residual_filename,
430
+ show_progress=show_progress,
431
+ fit_only=subtraction_cfg.fit_only,
432
+ )
433
+ return detections
434
+
435
+
436
+ def match(
437
+ output_dir: str | Path,
438
+ recording: BaseRecording,
439
+ sorting: DARTsortSorting | None = None,
440
+ motion_est=None,
441
+ waveform_cfg=default_waveform_cfg,
442
+ template_cfg=default_template_cfg,
443
+ featurization_cfg=default_featurization_cfg,
444
+ matching_cfg=default_matching_cfg,
445
+ sampling_cfg=default_peeling_fit_sampling_cfg,
446
+ previous_detection_cfg: Any | None = None,
447
+ chunk_starts_samples=None,
448
+ overwrite=False,
449
+ residual_filename: str | None = None,
450
+ show_progress=True,
451
+ hdf5_filename="matching0.h5",
452
+ model_subdir="matching0_models",
453
+ template_data: TemplateData | None = None,
454
+ template_npz_filename="template_data.npz",
455
+ computation_cfg: ComputationConfig | None = None,
456
+ template_denoising_tsvd=None,
457
+ ) -> DARTsortSorting:
458
+ output_dir = resolve_path(output_dir)
459
+ model_dir = output_dir / model_subdir
460
+
461
+ if template_data is None and not matching_cfg.precomputed_templates_npz:
462
+ assert sorting is not None
463
+ sorting, template_data = estimate_template_library(
464
+ recording=recording,
465
+ sorting=sorting,
466
+ motion_est=motion_est,
467
+ min_template_ptp=matching_cfg.min_template_ptp,
468
+ min_template_snr=matching_cfg.min_template_snr,
469
+ min_template_count=matching_cfg.min_template_count,
470
+ max_cc_flag_rate=matching_cfg.max_cc_flag_rate,
471
+ cc_flag_entropy_cutoff=matching_cfg.cc_flag_entropy_cutoff,
472
+ depth_order=matching_cfg.depth_order,
473
+ waveform_cfg=waveform_cfg,
474
+ template_cfg=template_cfg,
475
+ realign_cfg=matching_cfg.template_realignment_cfg,
476
+ template_merge_cfg=matching_cfg.template_merge_cfg,
477
+ computation_cfg=computation_cfg,
478
+ detection_cfg=previous_detection_cfg,
479
+ tsvd=template_denoising_tsvd,
480
+ template_npz_path=model_dir / template_npz_filename,
481
+ )
482
+
483
+ matching_peeler = ObjectiveUpdateTemplateMatchingPeeler.from_config(
484
+ recording=recording,
485
+ waveform_cfg=waveform_cfg,
486
+ matching_cfg=matching_cfg,
487
+ sampling_cfg=sampling_cfg,
488
+ featurization_cfg=featurization_cfg,
489
+ template_data=template_data,
490
+ motion_est=motion_est,
491
+ parent_sorting_hdf5_path=getattr(sorting, "parent_h5_path", None),
492
+ )
493
+ sorting = run_peeler(
494
+ matching_peeler,
495
+ output_directory=output_dir,
496
+ hdf5_filename=hdf5_filename,
497
+ model_subdir=model_subdir,
498
+ featurization_cfg=featurization_cfg,
499
+ chunk_starts_samples=chunk_starts_samples,
500
+ overwrite=overwrite,
501
+ residual_filename=residual_filename,
502
+ show_progress=show_progress,
503
+ computation_cfg=computation_cfg,
504
+ )
505
+ return sorting
506
+
507
+
508
+ def grab(
509
+ output_dir: str | Path,
510
+ recording: BaseRecording,
511
+ sorting: DARTsortSorting,
512
+ waveform_cfg=default_waveform_cfg,
513
+ featurization_cfg=default_featurization_cfg,
514
+ sampling_cfg=default_peeling_fit_sampling_cfg,
515
+ chunk_starts_samples=None,
516
+ overwrite=False,
517
+ show_progress=True,
518
+ hdf5_filename="grab.h5",
519
+ model_subdir="grab_models",
520
+ computation_cfg: ComputationConfig | None = None,
521
+ ) -> DARTsortSorting:
522
+ output_dir = resolve_path(output_dir)
523
+ grabber = GrabAndFeaturize.from_config(
524
+ sorting=sorting,
525
+ recording=recording,
526
+ waveform_cfg=waveform_cfg,
527
+ sampling_cfg=sampling_cfg,
528
+ featurization_cfg=featurization_cfg,
529
+ )
530
+ sorting = run_peeler(
531
+ grabber,
532
+ output_directory=output_dir,
533
+ hdf5_filename=hdf5_filename,
534
+ model_subdir=model_subdir,
535
+ featurization_cfg=featurization_cfg,
536
+ chunk_starts_samples=chunk_starts_samples,
537
+ overwrite=overwrite,
538
+ show_progress=show_progress,
539
+ computation_cfg=computation_cfg,
540
+ )
541
+ return sorting
542
+
543
+
544
+ def threshold(
545
+ output_dir: str | Path,
546
+ recording: BaseRecording,
547
+ waveform_cfg=default_waveform_cfg,
548
+ thresholding_cfg=default_thresholding_cfg,
549
+ featurization_cfg=default_featurization_cfg,
550
+ sampling_cfg=default_peeling_fit_sampling_cfg,
551
+ chunk_starts_samples=None,
552
+ overwrite=False,
553
+ show_progress=True,
554
+ hdf5_filename="threshold.h5",
555
+ model_subdir="threshold_models",
556
+ computation_cfg: ComputationConfig | None = None,
557
+ ) -> DARTsortSorting:
558
+ output_dir = resolve_path(output_dir)
559
+ thresholder = ThresholdAndFeaturize.from_config(
560
+ recording=recording,
561
+ waveform_cfg=waveform_cfg,
562
+ thresholding_cfg=thresholding_cfg,
563
+ featurization_cfg=featurization_cfg,
564
+ sampling_cfg=sampling_cfg,
565
+ )
566
+ sorting = run_peeler(
567
+ thresholder,
568
+ output_directory=output_dir,
569
+ hdf5_filename=hdf5_filename,
570
+ model_subdir=model_subdir,
571
+ featurization_cfg=featurization_cfg,
572
+ chunk_starts_samples=chunk_starts_samples,
573
+ overwrite=overwrite,
574
+ show_progress=show_progress,
575
+ computation_cfg=computation_cfg,
576
+ )
577
+ return sorting
578
+
579
+
580
+ def cluster(
581
+ recording,
582
+ sorting,
583
+ motion_est=None,
584
+ clustering_cfg: ClusteringConfig | None = default_clustering_cfg,
585
+ clustering_features_cfg: (
586
+ ClusteringFeaturesConfig | None
587
+ ) = default_clustering_features_cfg,
588
+ refinement_cfgs: list[RefinementConfig | None] | None = None,
589
+ computation_cfg: ComputationConfig | None = None,
590
+ features: SimpleMatrixFeatures | None = None,
591
+ *,
592
+ _save_cfg: DARTsortInternalConfig | None = None,
593
+ _save_initial_name="initial",
594
+ _save_refined_name_fmt="refined0{stepname}",
595
+ _save_dir=None,
596
+ ):
597
+ if features is None:
598
+ features = get_clustering_features(
599
+ recording,
600
+ sorting,
601
+ motion_est=motion_est,
602
+ clustering_features_cfg=clustering_features_cfg,
603
+ )
604
+ assert features is not None
605
+ clusterer = get_clusterer(
606
+ clustering_cfg=clustering_cfg,
607
+ refinement_cfgs=refinement_cfgs,
608
+ computation_cfg=computation_cfg,
609
+ save_cfg=_save_cfg,
610
+ save_labels_dir=_save_dir,
611
+ initial_name=_save_initial_name,
612
+ refine_labels_fmt=_save_refined_name_fmt,
613
+ )
614
+ result = clusterer.cluster(
615
+ recording=recording, sorting=sorting, features=features, motion_est=motion_est
616
+ )
617
+
618
+ del features, clusterer
619
+ gc.collect()
620
+ torch.cuda.empty_cache()
621
+
622
+ return result
623
+
624
+
625
+ def universal_match(
626
+ output_dir: str | Path,
627
+ recording: BaseRecording,
628
+ universal_cfg=default_universal_cfg,
629
+ featurization_cfg=default_featurization_cfg,
630
+ sampling_cfg=default_peeling_fit_sampling_cfg,
631
+ chunk_starts_samples=None,
632
+ overwrite=False,
633
+ show_progress=True,
634
+ hdf5_filename="universal.h5",
635
+ model_subdir="universal_models",
636
+ computation_cfg: ComputationConfig | None = None,
637
+ ) -> DARTsortSorting:
638
+ output_dir = resolve_path(output_dir)
639
+ universal_matcher = UniversalTemplatesMatchingPeeler.from_config(
640
+ recording=recording,
641
+ universal_cfg=universal_cfg,
642
+ waveform_cfg=universal_cfg.waveform_cfg, # TODO...
643
+ featurization_cfg=featurization_cfg,
644
+ sampling_cfg=sampling_cfg,
645
+ )
646
+ sorting = run_peeler(
647
+ universal_matcher,
648
+ output_directory=output_dir,
649
+ hdf5_filename=hdf5_filename,
650
+ model_subdir=model_subdir,
651
+ featurization_cfg=featurization_cfg,
652
+ chunk_starts_samples=chunk_starts_samples,
653
+ overwrite=overwrite,
654
+ show_progress=show_progress,
655
+ computation_cfg=computation_cfg,
656
+ )
657
+ return sorting
@@ -0,0 +1,6 @@
1
+ from .grab import GrabAndFeaturize
2
+ from .matching import ObjectiveUpdateTemplateMatchingPeeler
3
+ from .subtract import SubtractionPeeler, subtract_chunk
4
+ from .threshold import ThresholdAndFeaturize
5
+ from .universal import UniversalTemplatesMatchingPeeler
6
+ from .matching_util import *