DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1074 @@
1
+ import dataclasses
2
+ from dataclasses import field, fields
3
+ from pathlib import Path
4
+ from typing import Literal, Self
5
+
6
+ import numpy as np
7
+ import torch
8
+
9
+ from .cli_util import argfield, dataclass_from_toml
10
+ from .py_util import cfg_dataclass, float_or_none, int_or_inf, resolve_path
11
+
12
+ try:
13
+ from importlib.resources import files
14
+ except ImportError:
15
+ try:
16
+ from importlib_resources import files # pyright: ignore[reportMissingImports]
17
+ except ImportError:
18
+ raise ValueError("Need python>=3.10 or pip install importlib_resources.")
19
+
20
+ default_pretrained_path = files("dartsort.pretrained")
21
+ default_pretrained_path = default_pretrained_path.joinpath("single_chan_denoiser.pt")
22
+ default_pretrained_path = str(default_pretrained_path)
23
+
24
+
25
+ @cfg_dataclass
26
+ class WaveformConfig:
27
+ """Defaults yield 42 sample trough offset and 121 total at 30kHz."""
28
+
29
+ ms_before: float = 1.4
30
+ ms_after: float = 2.6 + 0.1 / 3
31
+
32
+ @classmethod
33
+ def from_samples(
34
+ cls, samples_before: int, samples_after: int, sampling_frequency=30_000.0
35
+ ) -> Self:
36
+ samples_per_ms = sampling_frequency / 1000
37
+ self = cls(
38
+ ms_before=samples_before / samples_per_ms,
39
+ ms_after=samples_after / samples_per_ms,
40
+ )
41
+ assert self.trough_offset_samples(sampling_frequency) == samples_before
42
+ samples_total = samples_before + samples_after
43
+ assert self.spike_length_samples(sampling_frequency) == samples_total
44
+ return self
45
+
46
+ @staticmethod
47
+ def ms_to_samples(ms, sampling_frequency=30_000.0):
48
+ return int(ms * (sampling_frequency / 1000))
49
+
50
+ def trough_offset_samples(self, sampling_frequency=30_000.0):
51
+ sampling_frequency = np.round(sampling_frequency)
52
+ return self.ms_to_samples(self.ms_before, sampling_frequency=sampling_frequency)
53
+
54
+ def spike_length_samples(self, sampling_frequency=30_000.0):
55
+ spike_len_ms = self.ms_before + self.ms_after
56
+ sampling_frequency = np.round(sampling_frequency)
57
+ length = self.ms_to_samples(spike_len_ms, sampling_frequency=sampling_frequency)
58
+ # odd is better for convolution arithmetic elsewhere
59
+ length = 2 * (length // 2) + 1
60
+ return length
61
+
62
+ def relative_slice(self, other: Self, sampling_frequency=30_000.0) -> slice:
63
+ """My trough-aligned subset of samples in other, which contains me."""
64
+ assert other.ms_before >= self.ms_before
65
+ assert other.ms_after >= self.ms_after
66
+ my_trough = self.trough_offset_samples(sampling_frequency)
67
+ my_len = self.spike_length_samples(sampling_frequency)
68
+ other_trough = other.trough_offset_samples(sampling_frequency)
69
+ other_len = other.spike_length_samples(sampling_frequency)
70
+ start_offset = other_trough - my_trough
71
+ end_offset = (other_len - other_trough) - (my_len - my_trough)
72
+ if start_offset == end_offset == 0:
73
+ return slice(None)
74
+ return slice(start_offset, other_len - end_offset)
75
+
76
+
77
+ @cfg_dataclass
78
+ class FeaturizationConfig:
79
+ """Featurization and denoising configuration
80
+
81
+ Parameters for a featurization and denoising pipeline
82
+ which has the flow:
83
+ [input waveforms]
84
+ -> [featurization of input waveforms]
85
+ -> [denoising]
86
+ -> [featurization of output waveforms]
87
+
88
+ The flags below allow users to control which features
89
+ are computed for the input waveforms, what denoising
90
+ operations are applied, and what features are computed
91
+ for the output (post-denoising) waveforms.
92
+
93
+ Users who'd rather do something not covered by this
94
+ typical case can manually instantiate a WaveformPipeline
95
+ and pass it into their peeler.
96
+ """
97
+
98
+ skip: bool = False
99
+ extract_radius: float = 100.0
100
+ stop_after_n: int | None = None
101
+ shuffle: bool = False
102
+
103
+ # -- denoising configuration
104
+ do_nn_denoise: bool = False
105
+ do_tpca_denoise: bool = True
106
+ do_enforce_decrease: bool = True
107
+ # turn off features below
108
+ denoise_only: bool = False
109
+
110
+ # -- residual snips
111
+ n_residual_snips: int = 4 * 4096
112
+ residual_later: bool = False
113
+
114
+ # -- featurization configuration
115
+ save_input_voltages: bool = True
116
+ save_input_waveforms: bool = False
117
+ save_input_tpca_projs: bool = True
118
+ save_output_waveforms: bool = False
119
+ save_output_tpca_projs: bool = False
120
+ save_amplitudes: bool = True
121
+ save_all_amplitudes: bool = False
122
+ # localization runs on output waveforms
123
+ do_localization: bool = True
124
+ localization_radius: float = 100.0
125
+ # these are saved always if do_localization
126
+ localization_amplitude_type: Literal["peak", "ptp"] = "peak"
127
+ localization_model: Literal["pointsource", "dipole"] = "pointsource"
128
+ nn_localization: bool = True
129
+ additional_com_localization: bool = False
130
+ localization_noise_floor: bool = False
131
+
132
+ # -- further info about denoising
133
+ # in the future we may add multi-channel or other nns
134
+ nn_denoiser_class_name: str = "SingleChannelWaveformDenoiser"
135
+ nn_denoiser_pretrained_path: str | None = default_pretrained_path
136
+ nn_denoiser_train_epochs: int = 100
137
+ nn_denoiser_epoch_size: int = 200 * 256
138
+ nn_denoiser_extra_kwargs: dict | None = argfield(None, cli=False)
139
+
140
+ # optionally restrict how many channels TPCA are fit on
141
+ tpca_fit_radius: float = 75.0
142
+ tpca_rank: int = 8
143
+ tpca_centered: bool = False
144
+ learn_cleaned_tpca_basis: bool = True
145
+ input_tpca_waveform_cfg: WaveformConfig | None = WaveformConfig(
146
+ ms_before=0.75, ms_after=1.25
147
+ )
148
+ tpca_max_waveforms: int = 40_000
149
+
150
+ # used when naming datasets saved to h5 files
151
+ input_waveforms_name: str = "collisioncleaned"
152
+ output_waveforms_name: str = "denoised"
153
+
154
+
155
+ InterpMethod = Literal[
156
+ "kriging", "kernel", "normalized", "krigingnormalized", "zero", "nearest"
157
+ ]
158
+ InterpKernel = Literal[
159
+ "zero", "nearest", "idw", "rbf", "multiquadric", "rq", "thinplate"
160
+ ]
161
+
162
+
163
+ @cfg_dataclass
164
+ class InterpolationParams:
165
+ method: InterpMethod = "kriging"
166
+ kernel: InterpKernel = "thinplate"
167
+ extrap_method: InterpMethod | None = None
168
+ extrap_kernel: InterpKernel | None = None
169
+ kriging_poly_degree: int = 1
170
+ sigma: float = 10.0
171
+ rq_alpha: float = 0.5
172
+ smoothing_lambda: float = 0.0
173
+
174
+ @property
175
+ def actual_extrap_method(self):
176
+ if self.extrap_method is None:
177
+ return self.method
178
+ return self.extrap_method
179
+
180
+ @property
181
+ def actual_extrap_kernel(self):
182
+ if self.extrap_kernel is None:
183
+ return self.kernel
184
+ return self.extrap_kernel
185
+
186
+ def extrap_diff(self):
187
+ if self.actual_extrap_method != self.method:
188
+ return True
189
+ if self.actual_extrap_kernel != self.kernel:
190
+ return True
191
+ return False
192
+
193
+ def normalize(self) -> Self:
194
+ method = self.method
195
+ kernel = self.kernel
196
+ if method == "nearest":
197
+ method = "kernel"
198
+ kernel = "nearest"
199
+ elif method == "zero":
200
+ method = "kernel"
201
+ kernel = "zero"
202
+
203
+ extrap_method = self.extrap_method
204
+ extrap_kernel = self.extrap_kernel
205
+ if extrap_method == "nearest":
206
+ extrap_method = "kernel"
207
+ extrap_kernel = "nearest"
208
+ elif extrap_method == "zero":
209
+ extrap_method = "kernel"
210
+ extrap_kernel = "zero"
211
+
212
+ return self.__class__(
213
+ method=method,
214
+ kernel=kernel,
215
+ extrap_method=extrap_method,
216
+ extrap_kernel=extrap_kernel,
217
+ kriging_poly_degree=self.kriging_poly_degree,
218
+ sigma=self.sigma,
219
+ rq_alpha=self.rq_alpha,
220
+ smoothing_lambda=self.smoothing_lambda,
221
+ )
222
+
223
+
224
+ default_interpolation_params = InterpolationParams()
225
+ default_extrapolation_params = InterpolationParams(
226
+ method="kernel", kernel="rq", sigma=10.0
227
+ )
228
+
229
+ FitSamplingMethod = Literal["random", "amp_reweighted"]
230
+ default_fit_sampling_method = "amp_reweighted"
231
+ default_fit_max_reweighting = 4.0
232
+
233
+
234
+ @cfg_dataclass
235
+ class FitSamplingConfig:
236
+ max_waveforms_fit: int = 50_000
237
+ n_waveforms_fit: int = 40_000
238
+ fit_subsampling_random_state: int = 0
239
+ fit_sampling: FitSamplingMethod = "amp_reweighted"
240
+ fit_max_reweighting: float = default_fit_max_reweighting
241
+
242
+
243
+ default_peeling_fit_sampling_cfg = FitSamplingConfig()
244
+ default_clustering_fit_sampling_cfg = FitSamplingConfig(
245
+ max_waveforms_fit=500_000, n_waveforms_fit=500_000
246
+ )
247
+ default_refinement_fit_sampling_cfg = FitSamplingConfig(
248
+ max_waveforms_fit=1000 * 1024, n_waveforms_fit=1000 * 1024
249
+ )
250
+
251
+
252
+ @cfg_dataclass
253
+ class SubtractionConfig:
254
+ # peeling common
255
+ chunk_length_samples: int = 30_000
256
+ n_seconds_fit: int = 100
257
+ fit_only: bool = False
258
+
259
+ # subtraction
260
+ detection_threshold: float = 4.0
261
+ peak_sign: Literal["pos", "neg", "both"] = "both"
262
+ realign_to_denoiser: bool = True
263
+ denoiser_realignment_channel: Literal["detection", "denoised"] = "detection"
264
+ denoiser_realignment_shift: int = 5
265
+ relative_peak_radius_samples: int = 5
266
+ relative_peak_radius_um: float | None = 35.0
267
+ spatial_dedup_radius: float | None = 100.0
268
+ temporal_dedup_radius_samples: int = 11
269
+ remove_exact_duplicates: bool = True
270
+ positive_temporal_dedup_radius_samples: int = 41
271
+ subtract_radius: float = 200.0
272
+ residnorm_decrease_threshold: float = 12.0
273
+ growth_tolerance: float | None = None
274
+ trough_priority: float | None = 2.0
275
+ use_singlechan_templates: bool = False
276
+ singlechan_threshold: float = 50.0
277
+ n_singlechan_templates: int = 10
278
+ singlechan_alignment_padding_ms: float = 1.5
279
+ cumulant_order: int | None = None
280
+ convexity_threshold: float | None = None
281
+ convexity_radius: int = 7
282
+
283
+ # how will waveforms be denoised before subtraction?
284
+ # users can also save waveforms/features during subtraction
285
+ subtraction_denoising_cfg: FeaturizationConfig = FeaturizationConfig(
286
+ denoise_only=True,
287
+ do_nn_denoise=True,
288
+ extract_radius=200.0,
289
+ input_waveforms_name="raw",
290
+ output_waveforms_name="subtracted",
291
+ )
292
+
293
+ # initial denoiser fitting parameters
294
+ first_denoiser_max_waveforms_fit: int = 250_000
295
+ first_denoiser_thinning: float = 0.5
296
+ first_denoiser_temporal_jitter: int = 3
297
+ first_denoiser_spatial_jitter: float = 35.0
298
+
299
+ # for debugging / vis
300
+ save_iteration: bool = False
301
+ save_residnorm_decrease: bool = False
302
+
303
+
304
+ @cfg_dataclass
305
+ class ThresholdingConfig:
306
+ # peeling common
307
+ chunk_length_samples: int = 30_000
308
+ n_seconds_fit: int = 100
309
+ sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg
310
+
311
+ # thresholding
312
+ detection_threshold: float = 5.0
313
+ max_spikes_per_chunk: int | None = None
314
+ peak_sign: Literal["pos", "neg", "both"] = "both"
315
+ spatial_dedup_radius: float = 150.0
316
+ relative_peak_radius_um: float = 35.0
317
+ relative_peak_radius_samples: int = 5
318
+ temporal_dedup_radius_samples: int = 11
319
+ remove_exact_duplicates: bool = True
320
+ cumulant_order: int | None = None
321
+ convexity_threshold: float | None = -50.0
322
+ convexity_radius: int = 7
323
+
324
+ thinning: float = 0.0
325
+ time_jitter: int = 0
326
+ spatial_jitter_radius: float = 0.0
327
+ trough_priority: float | None = 2.0
328
+
329
+
330
+ @cfg_dataclass
331
+ class TemplateConfig:
332
+ spikes_per_unit: int = 500
333
+ with_raw_std_dev: bool = False
334
+ reduction: Literal["median", "mean"] = "mean"
335
+ algorithm: Literal["by_chunk", "by_unit", "chunk_if_mean"] = "chunk_if_mean"
336
+ denoising_method: Literal["none", "exp_weighted", "loot", "t", "coll"] = (
337
+ "exp_weighted"
338
+ )
339
+ use_raw: bool = True
340
+ use_svd: bool = True
341
+ use_zero: bool = False
342
+ use_outlier: bool = False
343
+ use_raw_outlier: bool = False
344
+ use_svd_outlier: bool = False
345
+ templates_at_once: int = 384
346
+ max_templates_at_once: int = 512
347
+ raw_templates_at_once: int = 1024
348
+
349
+ # -- template construction parameters
350
+ # registered templates?
351
+ registered_templates: bool = True
352
+
353
+ # superresolved templates
354
+ superres_templates: bool = False
355
+ superres_bin_size_um: float = 5.0
356
+ superres_bin_min_spikes: int = 50
357
+ superres_strategy: str = "motion_estimate"
358
+
359
+ # low rank denoising?
360
+ denoising_rank: int = 5
361
+ denoising_fit_radius: float = 75.0
362
+ recompute_tsvd: bool = True
363
+ denoising_fit_sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg
364
+
365
+ # exp weight denoising
366
+ exp_weight_snr_threshold: float = 50.0
367
+
368
+ # t denoising
369
+ initial_t_df: float = 3.0
370
+ fixed_t_df: float | tuple[float, ...] | None = (float("inf"), 1.0, 1.0)
371
+ t_iters: int = 1
372
+ svd_inside_t: bool = False
373
+ loot_cov: Literal["diag", "global"] = "global"
374
+
375
+ # where to find data if needed
376
+ amplitudes_dataset_name: str = "denoised_ptp_amplitudes"
377
+ localizations_dataset_name: str = "point_source_localizations"
378
+
379
+ def actual_algorithm(self) -> Literal["by_chunk", "by_unit"]:
380
+ if self.algorithm == "chunk_if_mean":
381
+ if self.reduction == "mean":
382
+ return "by_chunk"
383
+ else:
384
+ return "by_unit"
385
+ return self.algorithm
386
+
387
+ def __post_init__(self):
388
+ if self.algorithm in ("t", "loot") and self.reduction == "median":
389
+ raise ValueError("Median reduction not supported for 't' templates.")
390
+
391
+
392
+ RealignStrategy = Literal[
393
+ "mainchan_trough_factor",
394
+ "dredge",
395
+ "snr_weighted_trough_factor",
396
+ "normsq_weighted_trough_factor",
397
+ "ampsq_weighted_trough_factor",
398
+ "mainchan_svd_trough_factor",
399
+ "snr_weighted_svd_trough_factor",
400
+ "normsq_weighted_svd_trough_factor",
401
+ "ampsq_weighted_svd_trough_factor",
402
+ ]
403
+
404
+
405
+ @cfg_dataclass
406
+ class TemplateRealignmentConfig:
407
+ realign_peaks: bool = True
408
+ realign_strategy: RealignStrategy = "snr_weighted_trough_factor"
409
+ realign_shift_ms: float = 1.5
410
+ trough_factor: float = 3.0
411
+ template_cfg: TemplateConfig = TemplateConfig(denoising_method="none")
412
+ min_pair_corr: float = 0.8
413
+
414
+
415
+ @cfg_dataclass
416
+ class TemplateMergeConfig:
417
+ distance_kind: str = "rms"
418
+ linkage: str = "complete"
419
+ merge_distance_threshold: float = 0.25
420
+ cross_merge_distance_threshold: float = 0.5
421
+ min_spatial_cosine: float = 0.75
422
+ temporal_upsampling_factor: int = 4
423
+ amplitude_scaling_variance: float = 0.01**2
424
+ amplitude_scaling_boundary: float = 0.333
425
+ svd_compression_rank: int = 20
426
+ max_shift_ms: float = 1.6666
427
+
428
+
429
+ @cfg_dataclass
430
+ class MatchingConfig:
431
+ # peeling common
432
+ chunk_length_samples: int = 30_000
433
+ n_seconds_fit: int = 100
434
+ max_spikes_per_second: int = 16384
435
+ cd_iter: int = 0
436
+ coarse_cd: bool = True
437
+
438
+ # template matching parameters
439
+ threshold: float | Literal["fp_control"] = 10.0
440
+ template_svd_compression_rank: int = 10
441
+ template_temporal_upsampling_factor: int = 8
442
+ upsampling_radius: int = 8
443
+ template_min_channel_amplitude: float = 0.0
444
+ refractory_radius_frames: int = 10
445
+ amplitude_scaling_variance: float = 0.01**2
446
+ amplitude_scaling_boundary: float = 0.333
447
+ max_iter: int = 100
448
+ conv_ignore_threshold: float = 0.0
449
+ coarse_approx_error_threshold: float = 0.0
450
+ coarse_objective: bool = True
451
+ channel_selection_radius: float | None = None
452
+ template_type: Literal["individual_compressed_upsampled", "drifty", "debug"] = (
453
+ "drifty"
454
+ )
455
+ up_method: Literal["interpolation", "keys3", "keys4", "direct"] = "keys4"
456
+ drift_interp_neighborhood_radius: float = 200.0
457
+ drift_interp_params: InterpolationParams = default_interpolation_params
458
+ upsampling_compression_map: Literal["yass", "none"] = "yass"
459
+ whiten: bool = False
460
+
461
+ # template postprocessing parameters
462
+ min_template_ptp: float = 1.0
463
+ min_template_snr: float = 40.0
464
+ min_template_count: int = 50
465
+ max_cc_flag_rate: float = 0.5
466
+ cc_flag_entropy_cutoff: float = 2.0
467
+ depth_order: bool = True
468
+ template_merge_cfg: TemplateMergeConfig | None = TemplateMergeConfig(
469
+ merge_distance_threshold=0.025
470
+ )
471
+ template_realignment_cfg: TemplateRealignmentConfig = TemplateRealignmentConfig()
472
+ precomputed_templates_npz: str | None = None
473
+ delete_pconv: bool = True
474
+
475
+
476
+ @cfg_dataclass
477
+ class UniversalMatchingConfig:
478
+ # peeling common
479
+ chunk_length_samples: int = 1_000
480
+ n_seconds_fit: int = 100
481
+
482
+ n_sigmas: int = 5
483
+ n_centroids: int = 6
484
+ threshold: float = 10.0
485
+ detection_threshold: float = 6.0
486
+ alignment_padding_ms: float = 1.5
487
+
488
+ waveform_cfg: WaveformConfig = WaveformConfig(ms_before=0.75, ms_after=1.25)
489
+
490
+
491
+ @cfg_dataclass
492
+ class MotionEstimationConfig:
493
+ """Configure motion estimation."""
494
+
495
+ do_motion_estimation: bool = True
496
+
497
+ # sometimes spikes can be localized far away from the probe, causing
498
+ # issues with motion estimation, we will ignore such spikes
499
+ probe_boundary_padding_um: float = 100.0
500
+
501
+ # DREDge parameters
502
+ spatial_bin_length_um: float = 1.0
503
+ temporal_bin_length_s: float = 1.0
504
+ window_step_um: float = 400.0
505
+ window_scale_um: float = 450.0
506
+ window_margin_um: float | None = argfield(default=None, arg_type=float)
507
+ max_dt_s: float = 1000.0
508
+ max_disp_um: float | None = argfield(default=None, arg_type=float)
509
+ correlation_threshold: float = 0.1
510
+ min_amplitude: float | None = argfield(default=None, arg_type=float)
511
+ rigid: bool = False
512
+
513
+
514
+ @cfg_dataclass
515
+ class SplitConfig:
516
+ split_strategy: str = "FeatureSplit"
517
+ recursive_split: bool = True
518
+ split_strategy_kwargs: dict | None = field(
519
+ default_factory=lambda: dict(max_spikes=20_000)
520
+ )
521
+
522
+
523
+ @cfg_dataclass
524
+ class ClusteringFeaturesConfig:
525
+ features_type: Literal["simple_matrix", "stable_waveforms"] = "simple_matrix"
526
+
527
+ # simple matrix feature controls
528
+ use_x: bool = True
529
+ use_z: bool = True
530
+ motion_aware: bool = True
531
+ use_amplitude: bool = False
532
+ use_signed_amplitude: bool = True
533
+ log_transform_amplitude: bool = True
534
+ amp_log_c: float = 5.0
535
+ amp_scale: float = 3.0
536
+ x_scale: float = 1.0
537
+ n_main_channel_pcs: int = 5
538
+ pc_scale: float = 2.0
539
+ pc_transform: Literal["log", "sqrt", "none"] | None = "none"
540
+ pc_pre_transform_scale: float = 0.5
541
+ adaptive_feature_scales: bool = False
542
+ workers: int = 5
543
+
544
+ amplitudes_dataset_name: str = "denoised_ptp_amplitudes"
545
+ voltages_dataset_name: str = "collisioncleaned_voltages"
546
+ amplitude_vectors_dataset_name: str = "denoised_ptp_amplitude_vectors"
547
+ localizations_dataset_name: str = "point_source_localizations"
548
+ pca_dataset_name: str = "collisioncleaned_tpca_features"
549
+
550
+ interp_params: InterpolationParams = default_interpolation_params
551
+
552
+
553
+ @cfg_dataclass
554
+ class ClusteringConfig:
555
+ cluster_strategy: str = "dpc"
556
+ sampling_cfg: FitSamplingConfig = default_clustering_fit_sampling_cfg
557
+
558
+ # global parameters
559
+ workers: int = 5
560
+ random_seed: int = 0
561
+ min_cluster_size: int = 25
562
+
563
+ # density peaks parameters
564
+ knn_k: int | None = None
565
+ sigma_local: float = 5.0
566
+ sigma_regional: float | None = argfield(default=25.0, arg_type=float_or_none)
567
+ n_neighbors_search: int = 50
568
+ radius_search: float = 25.0
569
+ noise_density: float = 0.0
570
+ outlier_radius: float = 25.0
571
+ outlier_neighbor_count: int = 10
572
+
573
+ # gmm density peaks additional parameters
574
+ kmeanspp_initializations: int = 10
575
+ kmeans_iter: int = 100
576
+ components_per_channel: int = 20
577
+ component_overlap: float = 0.95
578
+ hellinger_strong: float = 0.0
579
+ hellinger_weak: float = 0.0
580
+ use_hellinger: bool = True
581
+ gmmdpc_max_sigma: float = 5.0
582
+ mop: bool = True
583
+
584
+ # hdbscan parameters
585
+ min_samples: int = 25
586
+ cluster_selection_epsilon: int = 1
587
+ recursive: bool = False
588
+
589
+ # grid snap parameters
590
+ grid_dx: float = 15.0
591
+ grid_dz: float = 15.0
592
+
593
+ # sklearn clusterer params
594
+ sklearn_class_name: str = "DBSCAN"
595
+ sklearn_kwargs: dict | None = None
596
+
597
+
598
+ @cfg_dataclass
599
+ class RefinementConfig:
600
+ refinement_strategy: str = "tmm"
601
+ sampling_cfg: FitSamplingConfig = default_refinement_fit_sampling_cfg
602
+
603
+ # pcmerge
604
+ pc_merge_threshold: float = 0.1
605
+ pc_merge_metric: str = "normeuc"
606
+ pc_merge_spikes_per_unit: int = 4096
607
+ pc_merge_linkage: str = "complete"
608
+ pc_merge_rank: int = 5
609
+ pc_merge_min_iou: float = 0.95
610
+
611
+ # -- gmm parameters
612
+ # noise params
613
+ cov_kind: str = "factorizednoise"
614
+ glasso_alpha: float | int | None = None
615
+
616
+ # model params
617
+ neighb_overlap: float = 0.75
618
+ explore_neighb_steps: int = 0
619
+ min_count: int = 25
620
+ split_min_count: int = 8
621
+ channels_count_min: int = 1
622
+ signal_rank: int = 5
623
+ feature_rank: int = 8
624
+ initialize_at_rank_0: bool = False
625
+ cl_alpha: float = 1.0
626
+ latent_prior_std: float = 1.0
627
+ initial_basis_shrinkage: float = 1.0
628
+ n_spikes_fit: int = 4096
629
+ distance_metric: Literal["cosine", "normeuc"] = "normeuc"
630
+ n_candidates: int = 3
631
+ merge_group_size: int = 5
632
+ n_search: int | None = None
633
+ n_explore: int | None = None
634
+ train_batch_size: int = 512
635
+ eval_batch_size: int = 512
636
+ split_distance_threshold: float = 1.0
637
+ merge_distance_threshold: float = 1.0
638
+ criterion_em_iters: int = 3
639
+ hold_out_criterion: bool = True
640
+ n_em_iters: int = 250
641
+ em_converged_atol: float = 5e-3
642
+ n_total_iters: int = 1
643
+ mixture_steps: Literal["neither", "merge", "split", "both"] = "both"
644
+ prior_pseudocount: float = 0.0
645
+ kmeansk: int = 4
646
+ full_proposal_every: int = 10
647
+ search_adj: Literal["top", "explore"] = "top"
648
+
649
+ # TODO... reintroduce this if wanted. or remove
650
+ split_cfg: SplitConfig | None = None
651
+ merge_cfg: TemplateMergeConfig | None = None
652
+ merge_template_cfg: TemplateConfig | None = None
653
+
654
+ # forward_backward parameters
655
+ chunk_size_s: float = 300.0
656
+ log_c: float = 5.0
657
+ feature_scales: tuple[float, float, float] = (1.0, 1.0, 50.0)
658
+ adaptive_feature_scales: bool = False
659
+
660
+ # stable waveform feature controls
661
+ cov_radius: float = 500.0
662
+ core_radius: float | Literal["extract"] = "extract"
663
+ val_proportion: float = 0.25
664
+ impute_kind: Literal["interp", "impute"] = "impute"
665
+ interp_params: InterpolationParams = default_interpolation_params
666
+ noise_interp_params: InterpolationParams = default_interpolation_params
667
+
668
+
669
+ @cfg_dataclass
670
+ class ComputationConfig:
671
+ n_jobs_cpu: int = 0
672
+ n_jobs_gpu: int = 0
673
+ executor: str = "threading_unless_multigpu"
674
+ device: str | None = argfield(default=None, arg_type=str)
675
+
676
+ @classmethod
677
+ def from_n_jobs(cls, n_jobs):
678
+ return cls(n_jobs_cpu=n_jobs, n_jobs_gpu=n_jobs)
679
+
680
+ def actual_device(self):
681
+ if self.device is None:
682
+ have_cuda = torch.cuda.is_available()
683
+ if have_cuda:
684
+ return torch.device("cuda")
685
+ return torch.device("cpu")
686
+ return torch.device(self.device)
687
+
688
+ def actual_n_jobs(self):
689
+ if self.actual_device().type == "cuda":
690
+ return self.n_jobs_gpu
691
+ return self.n_jobs_cpu
692
+
693
+ def is_multi_gpu(self):
694
+ if self.n_jobs_gpu in (0, 1):
695
+ return False
696
+ dev = self.actual_device()
697
+ if dev.type != "cuda":
698
+ return False
699
+ if dev.index is not None:
700
+ return False
701
+ return torch.cuda.device_count() > 1
702
+
703
+
704
+ # default configs, used as defaults for kwargs in main.py etc
705
+ default_waveform_cfg = WaveformConfig()
706
+ default_featurization_cfg = FeaturizationConfig(learn_cleaned_tpca_basis=True)
707
+ default_subtraction_cfg = SubtractionConfig()
708
+ default_thresholding_cfg = ThresholdingConfig()
709
+ default_template_cfg = TemplateConfig()
710
+ default_clustering_cfg = ClusteringConfig()
711
+ default_clustering_features_cfg = ClusteringFeaturesConfig()
712
+ default_matching_cfg = MatchingConfig()
713
+ default_motion_estimation_cfg = MotionEstimationConfig()
714
+ default_computation_cfg = ComputationConfig()
715
+ default_refinement_cfg = RefinementConfig(mixture_steps="merge")
716
+ default_universal_cfg = UniversalMatchingConfig()
717
+ default_initial_refinement_cfg = RefinementConfig(mixture_steps="split")
718
+ default_pre_refinement_cfg = RefinementConfig(refinement_strategy="pcmerge")
719
+
720
+
721
+ @cfg_dataclass
722
+ class DARTsortInternalConfig:
723
+ """This is an internal object. Make a DARTsortUserConfig, not one of these."""
724
+
725
+ waveform_cfg: WaveformConfig = default_waveform_cfg
726
+ featurization_cfg: FeaturizationConfig = default_featurization_cfg
727
+ peeler_sampling_cfg: FitSamplingConfig = default_peeling_fit_sampling_cfg
728
+ initial_detection_cfg: (
729
+ SubtractionConfig
730
+ | MatchingConfig
731
+ | ThresholdingConfig
732
+ | UniversalMatchingConfig
733
+ ) = default_subtraction_cfg
734
+ template_cfg: TemplateConfig = default_template_cfg
735
+ clustering_cfg: ClusteringConfig = default_clustering_cfg
736
+ clustering_features_cfg: ClusteringFeaturesConfig = default_clustering_features_cfg
737
+ initial_refinement_cfg: RefinementConfig = default_initial_refinement_cfg
738
+ pre_refinement_cfg: RefinementConfig | None = default_pre_refinement_cfg
739
+ refinement_cfg: RefinementConfig = default_refinement_cfg
740
+ post_refinement_cfg: RefinementConfig | None = default_pre_refinement_cfg
741
+ matching_cfg: MatchingConfig = default_matching_cfg
742
+ motion_estimation_cfg: MotionEstimationConfig = default_motion_estimation_cfg
743
+ computation_cfg: ComputationConfig = default_computation_cfg
744
+
745
+ # high level behavior
746
+ detect_only: bool = False
747
+ dredge_only: bool = False
748
+ detection_type: Literal["subtract", "match", "threshold", "universal"] = "subtract"
749
+ final_refinement: bool = True
750
+ matching_iterations: int = 1
751
+ recluster_after_first_matching: bool = True
752
+ intermediate_matching_subsampling: float = 1.0
753
+
754
+ # development / debugging flags
755
+ work_in_tmpdir: bool = False
756
+ copy_recording_to_tmpdir: bool = False
757
+ workdir_follow_symlinks: bool = False
758
+ workdir_copier: Literal["shutil", "rsync"] = "shutil"
759
+ tmpdir_parent: str | Path | None = None
760
+ save_intermediate_labels: bool = False
761
+ save_intermediate_features: bool = False
762
+ save_final_features: bool = True
763
+ save_everything_on_error: bool = False
764
+
765
+
766
+ def to_internal_config(cfg) -> DARTsortInternalConfig:
767
+ """Laundromat of configuration formats
768
+
769
+ Arguments
770
+ ---------
771
+ cfg : str | Path | DARTsortUserConfig | DeveloperConfig
772
+ If str or Path, it should point to a .toml file.
773
+
774
+ Returns
775
+ -------
776
+ DARTsortInternalConfig
777
+ """
778
+ from dartsort.config import DARTsortUserConfig, DeveloperConfig
779
+
780
+ if isinstance(cfg, (str, Path)):
781
+ # load toml config
782
+ cfg0 = cfg
783
+
784
+ try:
785
+ cfg = resolve_path(cfg, strict=True)
786
+ except OSError as e:
787
+ raise ValueError(f"Configuration file {cfg0} does not exist.") from e
788
+
789
+ try:
790
+ cfg = dataclass_from_toml((DeveloperConfig,), cfg)
791
+ except Exception as e:
792
+ raise ValueError(
793
+ f"Could not read configuration from {cfg0}. More error info above."
794
+ ) from e
795
+
796
+ if isinstance(cfg, DARTsortInternalConfig):
797
+ return cfg
798
+ else:
799
+ assert isinstance(cfg, (DARTsortUserConfig, DeveloperConfig))
800
+
801
+ # if we have a user cfg, dump into dev cfg, and work from there
802
+ if isinstance(cfg, DARTsortUserConfig):
803
+ cfg = DeveloperConfig(**dataclasses.asdict(cfg))
804
+
805
+ waveform_cfg = WaveformConfig(ms_before=cfg.ms_before, ms_after=cfg.ms_after)
806
+ tpca_waveform_cfg = WaveformConfig(
807
+ ms_before=cfg.feature_ms_before, ms_after=cfg.feature_ms_after
808
+ )
809
+ featurization_cfg = FeaturizationConfig(
810
+ tpca_rank=cfg.temporal_pca_rank,
811
+ extract_radius=cfg.featurization_radius_um,
812
+ input_tpca_waveform_cfg=tpca_waveform_cfg,
813
+ localization_radius=cfg.localization_radius_um,
814
+ tpca_fit_radius=cfg.fit_radius_um,
815
+ tpca_max_waveforms=cfg.n_waveforms_fit,
816
+ save_input_waveforms=cfg.save_collisioncleaned_waveforms,
817
+ learn_cleaned_tpca_basis=True,
818
+ )
819
+ peeler_fit_sampling_cfg = FitSamplingConfig(
820
+ n_waveforms_fit=cfg.n_waveforms_fit,
821
+ max_waveforms_fit=cfg.max_waveforms_fit,
822
+ fit_sampling=cfg.fit_sampling,
823
+ )
824
+ if cfg.detection_type == "subtract":
825
+ subtraction_denoising_cfg = FeaturizationConfig(
826
+ denoise_only=True,
827
+ extract_radius=cfg.subtraction_radius_um,
828
+ do_nn_denoise=cfg.use_nn_in_subtraction,
829
+ do_tpca_denoise=cfg.do_tpca_denoise,
830
+ tpca_rank=cfg.temporal_pca_rank,
831
+ tpca_fit_radius=cfg.fit_radius_um,
832
+ input_waveforms_name="raw",
833
+ output_waveforms_name="subtracted",
834
+ save_output_waveforms=cfg.save_subtracted_waveforms,
835
+ nn_denoiser_class_name=cfg.nn_denoiser_class_name,
836
+ nn_denoiser_pretrained_path=cfg.nn_denoiser_pretrained_path,
837
+ )
838
+ initial_detection_cfg = SubtractionConfig(
839
+ peak_sign=cfg.peak_sign,
840
+ detection_threshold=cfg.voltage_threshold,
841
+ spatial_dedup_radius=cfg.deduplication_radius_um,
842
+ subtract_radius=cfg.subtraction_radius_um,
843
+ realign_to_denoiser=cfg.realign_to_denoiser,
844
+ singlechan_alignment_padding_ms=cfg.alignment_ms,
845
+ use_singlechan_templates=cfg.use_singlechan_templates,
846
+ residnorm_decrease_threshold=cfg.initial_threshold,
847
+ chunk_length_samples=cfg.chunk_length_samples,
848
+ first_denoiser_thinning=cfg.first_denoiser_thinning,
849
+ first_denoiser_max_waveforms_fit=cfg.nn_denoiser_max_waveforms_fit,
850
+ subtraction_denoising_cfg=subtraction_denoising_cfg,
851
+ )
852
+ elif cfg.detection_type == "threshold":
853
+ initial_detection_cfg = ThresholdingConfig(
854
+ peak_sign=cfg.peak_sign,
855
+ detection_threshold=cfg.voltage_threshold,
856
+ spatial_dedup_radius=cfg.deduplication_radius_um,
857
+ chunk_length_samples=cfg.chunk_length_samples,
858
+ )
859
+ elif cfg.detection_type == "match":
860
+ assert cfg.precomputed_templates_npz is not None
861
+ initial_detection_cfg = MatchingConfig(
862
+ threshold=cfg.matching_threshold,
863
+ amplitude_scaling_variance=cfg.amplitude_scaling_stddev**2,
864
+ amplitude_scaling_boundary=cfg.amplitude_scaling_boundary,
865
+ template_temporal_upsampling_factor=cfg.temporal_upsamples,
866
+ chunk_length_samples=cfg.chunk_length_samples,
867
+ precomputed_templates_npz=cfg.precomputed_templates_npz,
868
+ channel_selection_radius=cfg.channel_selection_radius,
869
+ template_type=cfg.matching_template_type,
870
+ up_method=cfg.matching_up_method,
871
+ template_min_channel_amplitude=cfg.matching_template_min_amplitude,
872
+ )
873
+ elif cfg.detection_type == "universal":
874
+ initial_detection_cfg = UniversalMatchingConfig(
875
+ waveform_cfg=tpca_waveform_cfg,
876
+ threshold=cfg.initial_threshold,
877
+ )
878
+ else:
879
+ raise ValueError(f"Unknown detection_type {cfg.detection_type}.")
880
+
881
+ template_cfg = TemplateConfig(
882
+ denoising_fit_radius=cfg.fit_radius_um,
883
+ spikes_per_unit=cfg.template_spikes_per_unit,
884
+ reduction=cfg.template_reduction,
885
+ denoising_method=cfg.template_denoising_method,
886
+ use_zero=cfg.template_mix_zero,
887
+ use_svd=cfg.template_mix_svd,
888
+ recompute_tsvd=cfg.always_recompute_tsvd,
889
+ )
890
+ clustering_cfg = ClusteringConfig(
891
+ cluster_strategy=cfg.cluster_strategy,
892
+ sigma_local=cfg.density_bandwidth,
893
+ sigma_regional=5 * cfg.density_bandwidth,
894
+ n_neighbors_search=cfg.n_neighbors_search or cfg.min_cluster_size,
895
+ outlier_radius=5 * cfg.density_bandwidth,
896
+ radius_search=5 * cfg.density_bandwidth,
897
+ min_cluster_size=cfg.min_cluster_size,
898
+ workers=cfg.clustering_workers,
899
+ use_hellinger=cfg.use_hellinger,
900
+ component_overlap=cfg.component_overlap,
901
+ hellinger_strong=cfg.hellinger_strong,
902
+ hellinger_weak=cfg.hellinger_weak,
903
+ mop=cfg.dpc_mop,
904
+ sampling_cfg=FitSamplingConfig(
905
+ n_waveforms_fit=cfg.clustering_max_spikes,
906
+ max_waveforms_fit=cfg.clustering_max_spikes,
907
+ fit_sampling=cfg.fit_sampling,
908
+ ),
909
+ )
910
+ clustering_features_cfg = ClusteringFeaturesConfig(
911
+ use_amplitude=cfg.initial_amp_feat,
912
+ use_signed_amplitude=cfg.initial_signed_amp_feat,
913
+ n_main_channel_pcs=cfg.initial_pc_feats,
914
+ pc_transform=cfg.initial_pc_transform,
915
+ pc_scale=cfg.initial_pc_scale,
916
+ pc_pre_transform_scale=cfg.initial_pc_pre_scale,
917
+ motion_aware=cfg.motion_aware_clustering,
918
+ workers=cfg.clustering_workers,
919
+ )
920
+
921
+ if cfg.gmm_metric == "cosine":
922
+ dist_thresh = cfg.gmm_cosine_threshold
923
+ elif cfg.gmm_metric == "kl":
924
+ dist_thresh = cfg.gmm_kl_threshold
925
+ elif cfg.gmm_metric == "euclidean":
926
+ dist_thresh = cfg.gmm_euclidean_threshold
927
+ elif cfg.gmm_metric == "normeuc":
928
+ dist_thresh = cfg.gmm_normeuc_threshold
929
+ else:
930
+ assert False
931
+ interp_params = InterpolationParams(
932
+ method=cfg.interp_method,
933
+ kernel=cfg.interp_kernel,
934
+ extrap_method=cfg.extrap_method,
935
+ extrap_kernel=cfg.extrap_kernel,
936
+ kriging_poly_degree=cfg.kriging_poly_degree,
937
+ sigma=cfg.interp_sigma,
938
+ rq_alpha=cfg.rq_alpha,
939
+ smoothing_lambda=cfg.smoothing_lambda,
940
+ ).normalize()
941
+ refinement_cfg = RefinementConfig(
942
+ refinement_strategy=cfg.refinement_strategy,
943
+ min_count=cfg.min_cluster_size,
944
+ signal_rank=cfg.signal_rank,
945
+ feature_rank=cfg.temporal_pca_rank,
946
+ initialize_at_rank_0=cfg.initialize_at_rank_0,
947
+ n_total_iters=cfg.n_refinement_iters,
948
+ n_em_iters=cfg.n_em_iters,
949
+ sampling_cfg=FitSamplingConfig(
950
+ n_waveforms_fit=cfg.gmm_max_spikes,
951
+ max_waveforms_fit=cfg.gmm_max_spikes,
952
+ fit_sampling=cfg.fit_sampling,
953
+ ),
954
+ em_converged_atol=cfg.gmm_em_atol,
955
+ val_proportion=cfg.gmm_val_proportion,
956
+ distance_metric=cfg.gmm_metric,
957
+ n_candidates=cfg.gmm_n_candidates,
958
+ n_search=cfg.gmm_n_search,
959
+ merge_distance_threshold=dist_thresh,
960
+ prior_pseudocount=cfg.prior_pseudocount,
961
+ initial_basis_shrinkage=cfg.initial_basis_shrinkage,
962
+ cov_kind=cfg.cov_kind,
963
+ interp_params=interp_params,
964
+ mixture_steps=cfg.later_steps,
965
+ kmeansk=cfg.kmeansk,
966
+ cl_alpha=cfg.gmm_cl_alpha,
967
+ )
968
+ if cfg.initial_rank is None:
969
+ irank = refinement_cfg.signal_rank
970
+ else:
971
+ irank = cfg.initial_rank
972
+ initial_refinement_cfg = dataclasses.replace(
973
+ refinement_cfg, mixture_steps=cfg.initial_steps, signal_rank=irank
974
+ )
975
+ if cfg.pre_refinement_merge:
976
+ pre_refinement_cfg = RefinementConfig(
977
+ refinement_strategy="pcmerge",
978
+ pc_merge_metric=cfg.pre_refinement_merge_metric,
979
+ pc_merge_threshold=cfg.pre_refinement_merge_threshold,
980
+ pc_merge_rank=cfg.initial_pc_feats,
981
+ )
982
+ else:
983
+ pre_refinement_cfg = None
984
+ motion_estimation_cfg = MotionEstimationConfig(
985
+ **{k.name: getattr(cfg, k.name) for k in fields(MotionEstimationConfig)}
986
+ )
987
+ matching_cfg = MatchingConfig(
988
+ threshold="fp_control" if cfg.matching_fp_control else cfg.matching_threshold,
989
+ amplitude_scaling_variance=cfg.amplitude_scaling_stddev**2,
990
+ amplitude_scaling_boundary=cfg.amplitude_scaling_boundary,
991
+ template_temporal_upsampling_factor=cfg.temporal_upsamples,
992
+ chunk_length_samples=cfg.chunk_length_samples,
993
+ template_merge_cfg=TemplateMergeConfig(
994
+ merge_distance_threshold=cfg.postprocessing_merge_threshold,
995
+ ),
996
+ cd_iter=cfg.matching_cd_iter,
997
+ coarse_cd=cfg.matching_coarse_cd,
998
+ channel_selection_radius=cfg.channel_selection_radius,
999
+ template_type=cfg.matching_template_type,
1000
+ up_method=cfg.matching_up_method,
1001
+ template_min_channel_amplitude=cfg.matching_template_min_amplitude,
1002
+ min_template_snr=cfg.min_template_snr,
1003
+ min_template_count=cfg.min_template_count,
1004
+ whiten=cfg.whiten_matching,
1005
+ template_realignment_cfg=TemplateRealignmentConfig(
1006
+ trough_factor=cfg.trough_factor,
1007
+ realign_strategy=cfg.realign_strategy,
1008
+ realign_shift_ms=cfg.alignment_ms,
1009
+ template_cfg=TemplateConfig(
1010
+ denoising_method="none",
1011
+ spikes_per_unit=cfg.template_spikes_per_unit,
1012
+ reduction=cfg.template_reduction,
1013
+ ),
1014
+ ),
1015
+ )
1016
+ computation_cfg = ComputationConfig(
1017
+ n_jobs_cpu=cfg.n_jobs_cpu,
1018
+ n_jobs_gpu=cfg.n_jobs_gpu,
1019
+ device=cfg.device,
1020
+ executor=cfg.executor,
1021
+ )
1022
+
1023
+ return DARTsortInternalConfig(
1024
+ waveform_cfg=waveform_cfg,
1025
+ featurization_cfg=featurization_cfg,
1026
+ initial_detection_cfg=initial_detection_cfg,
1027
+ peeler_sampling_cfg=peeler_fit_sampling_cfg,
1028
+ template_cfg=template_cfg,
1029
+ clustering_cfg=clustering_cfg,
1030
+ pre_refinement_cfg=pre_refinement_cfg,
1031
+ initial_refinement_cfg=initial_refinement_cfg,
1032
+ post_refinement_cfg=pre_refinement_cfg,
1033
+ refinement_cfg=refinement_cfg,
1034
+ matching_cfg=matching_cfg,
1035
+ clustering_features_cfg=clustering_features_cfg,
1036
+ motion_estimation_cfg=motion_estimation_cfg,
1037
+ computation_cfg=computation_cfg,
1038
+ detection_type=cfg.detection_type,
1039
+ dredge_only=cfg.dredge_only,
1040
+ matching_iterations=cfg.matching_iterations,
1041
+ recluster_after_first_matching=cfg.recluster_after_first_matching,
1042
+ work_in_tmpdir=cfg.work_in_tmpdir,
1043
+ copy_recording_to_tmpdir=cfg.copy_recording_to_tmpdir,
1044
+ workdir_copier=cfg.workdir_copier,
1045
+ workdir_follow_symlinks=cfg.workdir_follow_symlinks,
1046
+ tmpdir_parent=cfg.tmpdir_parent,
1047
+ save_intermediate_labels=cfg.save_intermediates,
1048
+ save_intermediate_features=cfg.save_intermediates,
1049
+ save_final_features=cfg.save_final_features,
1050
+ save_everything_on_error=cfg.save_everything_on_error,
1051
+ )
1052
+
1053
+
1054
+ default_dartsort_cfg = DARTsortInternalConfig()
1055
+
1056
+ # configs which are commonly used for specific tasks
1057
+ raw_template_cfg = TemplateConfig(denoising_method="none")
1058
+ unshifted_raw_template_cfg = TemplateConfig(
1059
+ registered_templates=False,
1060
+ denoising_method="none",
1061
+ superres_templates=False,
1062
+ )
1063
+ waveforms_only_featurization_cfg = FeaturizationConfig(
1064
+ do_tpca_denoise=False,
1065
+ do_enforce_decrease=False,
1066
+ n_residual_snips=0,
1067
+ save_input_tpca_projs=False,
1068
+ save_amplitudes=False,
1069
+ do_localization=False,
1070
+ input_waveforms_name="raw",
1071
+ save_input_voltages=True,
1072
+ save_input_waveforms=True,
1073
+ )
1074
+ skip_featurization_cfg = FeaturizationConfig(skip=True, n_residual_snips=0)