DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/config.py ADDED
@@ -0,0 +1,288 @@
1
+ from pathlib import Path
2
+ from typing import Annotated, Literal
3
+
4
+ from pydantic import Field
5
+
6
+ from .util.cli_util import argfield
7
+ from .util.internal_config import (
8
+ default_pretrained_path,
9
+ InterpMethod,
10
+ InterpKernel,
11
+ RealignStrategy,
12
+ )
13
+ from .util.py_util import (
14
+ cfg_dataclass,
15
+ float_or_none,
16
+ int_or_float_or_none,
17
+ int_or_none,
18
+ str_or_none,
19
+ )
20
+
21
+
22
+ @cfg_dataclass
23
+ class DARTsortUserConfig:
24
+ """User-facing configuration options"""
25
+
26
+ # -- high level behavior
27
+ dredge_only: bool = argfield(
28
+ False, doc="Whether to stop after initial localization and motion tracking."
29
+ )
30
+ matching_iterations: int = 1
31
+
32
+ # -- computer options
33
+ n_jobs_cpu: int = argfield(
34
+ default=0,
35
+ doc="Number of parallel workers to use when running on CPU. "
36
+ "0 means everything runs on the main thread.",
37
+ )
38
+ n_jobs_gpu: int = argfield(
39
+ default=0,
40
+ doc="Number of parallel workers to use when running on GPU. "
41
+ "0 means everything runs on the main thread.",
42
+ )
43
+ device: str | None = argfield(
44
+ default=None,
45
+ arg_type=str,
46
+ doc="The name of the PyTorch device to use. For example, 'cpu' "
47
+ "or 'cuda' or 'cuda:1'. If unset, uses n_jobs_gpu of your CUDA "
48
+ "GPUs if you have multiple, or else just the one, or your CPU.",
49
+ )
50
+ executor: str = "threading_unless_multigpu"
51
+ chunk_length_samples: int = 30_000
52
+
53
+ # -- storage behavior
54
+ work_in_tmpdir: bool = False
55
+ copy_recording_to_tmpdir: bool = False
56
+ workdir_copier: Literal["shutil", "rsync"] = "shutil"
57
+ workdir_follow_symlinks: bool = False
58
+ tmpdir_parent: str | Path | None = argfield(default=None, arg_type=str_or_none)
59
+ save_intermediates: bool = False
60
+ save_final_features: bool = True
61
+
62
+ # -- waveform snippet length parameters
63
+ ms_before: Annotated[float, Field(gt=0)] = argfield(
64
+ default=1.4,
65
+ doc="Length of time (ms) before trough (or peak) in waveform snippets. "
66
+ "Default value corresponds to 42 samples at 30kHz.",
67
+ )
68
+ ms_after: Annotated[float, Field(gt=0)] = argfield(
69
+ default=2.6 + 0.1 / 3,
70
+ doc="Length of time (ms) after trough (or peak) in waveform snippets. "
71
+ "Default value corresponds to 79 samples at 30kHz.",
72
+ )
73
+ alignment_ms: Annotated[float, Field(gt=0)] = argfield(
74
+ default=1.5,
75
+ doc="Time shift allowed when aligning events.",
76
+ )
77
+
78
+ # -- thresholds
79
+ peak_sign: Literal["neg", "both", "pos"] = argfield(
80
+ default="both",
81
+ doc="Allow only troughs or events of both signs when detecting threshold "
82
+ "crossings during initialization. Or positive only, if that's your thing.",
83
+ )
84
+ voltage_threshold: Annotated[float, Field(gt=0)] = argfield(
85
+ default=4.0,
86
+ doc="Threshold in standardized (SNR) voltage units for initial detection; "
87
+ "peaks or troughs larger than this value will be grabbed.",
88
+ )
89
+ matching_threshold: Annotated[float, Field(gt=0)] = argfield(
90
+ default=10.0,
91
+ doc="Template matching threshold. If subtracting a template leads "
92
+ "to at least this great of a decrease in the norm of the residual, "
93
+ "that match will be used.",
94
+ )
95
+ initial_threshold: Annotated[float, Field(gt=0)] = argfield(
96
+ default=12.0,
97
+ doc="Initial detection's neural net matching threshold. Same as "
98
+ "matching_threshold, except that a neural net is trying to guess "
99
+ "the true waveforms here, rather than using cluster templates.",
100
+ )
101
+
102
+ # -- featurization length, radius, rank parameters
103
+ temporal_pca_rank: Annotated[int, Field(gt=0)] = argfield(
104
+ default=8, doc="Rank of temporal PCAs used in denoising and featurization."
105
+ )
106
+ feature_ms_before: Annotated[float, Field(gt=0)] = argfield(
107
+ default=0.75,
108
+ doc="As ms_before, but used only when computing PCA features in clustering.",
109
+ )
110
+ feature_ms_after: Annotated[float, Field(gt=0)] = argfield(
111
+ default=1.25,
112
+ doc="As ms_after, but used only when computing PCA features in clustering.",
113
+ )
114
+ subtraction_radius_um: Annotated[float, Field(gt=0)] = argfield(
115
+ default=200.0,
116
+ doc="Radius of neighborhoods around spike events extracted "
117
+ "when denoising and subtracting NN-denoised events.",
118
+ )
119
+ deduplication_radius_um: Annotated[float, Field(gt=0)] = argfield(
120
+ default=100.0,
121
+ doc="During initial detection, if two spike events occur at the "
122
+ "same time within this radius, then the smaller of the two is "
123
+ "ignored. But also all of the secondary channels of the big one, "
124
+ "which is important.",
125
+ )
126
+ featurization_radius_um: Annotated[float, Field(gt=0)] = argfield(
127
+ default=100.0,
128
+ doc="Radius around detection channel or template peak channel used "
129
+ "to extract spike features for clustering.",
130
+ )
131
+ fit_radius_um: Annotated[float, Field(gt=0)] = argfield(
132
+ default=75.0,
133
+ doc="Extraction radius when fitting features like PCA; "
134
+ "smaller than other radii to include less noise.",
135
+ )
136
+ localization_radius_um: Annotated[float, Field(gt=0)] = argfield(
137
+ default=100.0,
138
+ doc="Radius around main channel used when localizing spikes.",
139
+ )
140
+
141
+ # -- matching parameters
142
+ amplitude_scaling_stddev: Annotated[float, Field(ge=0)] = 0.01
143
+ amplitude_scaling_boundary: Annotated[float, Field(ge=0)] = 0.333
144
+ temporal_upsamples: Annotated[int, Field(ge=1)] = 8
145
+
146
+ # -- motion estimation parameters
147
+ do_motion_estimation: bool = argfield(
148
+ default=True,
149
+ doc="Set this to false if your data is super stable or already motion-corrected.",
150
+ )
151
+
152
+ # DREDge parameters
153
+ rigid: bool = argfield(
154
+ default=False, doc="Use rigid registration and ignore the window parameters."
155
+ )
156
+ probe_boundary_padding_um: float = 100.0
157
+ spatial_bin_length_um: Annotated[float, Field(gt=0)] = 1.0
158
+ temporal_bin_length_s: Annotated[float, Field(gt=0)] = 1.0
159
+ window_step_um: Annotated[float, Field(gt=0)] = 400.0
160
+ window_scale_um: Annotated[float, Field(gt=0)] = 450.0
161
+ window_margin_um: Annotated[float, Field(gt=0)] | None = None
162
+ max_dt_s: Annotated[float, Field(gt=0)] = 1000.0
163
+ max_disp_um: Annotated[float, Field(gt=0)] | None = None
164
+ correlation_threshold: Annotated[float, Field(gt=0, lt=1)] = 0.1
165
+ min_amplitude: float | None = argfield(default=None, arg_type=float)
166
+
167
+
168
+ @cfg_dataclass
169
+ class DeveloperConfig(DARTsortUserConfig):
170
+ """Additional parameters for experiments. This API will never be stable."""
171
+
172
+ # high level behavior
173
+ initial_steps: Literal["neither", "split", "merge", "both"] = "split"
174
+ later_steps: Literal["neither", "split", "merge", "both"] = "merge"
175
+ detection_type: str = "subtract"
176
+ cluster_strategy: str = "dpc"
177
+ refinement_strategy: str = "tmm"
178
+ recluster_after_first_matching: bool = True
179
+
180
+ # general peeling
181
+ n_waveforms_fit: int = 40_000
182
+ max_waveforms_fit: int = 50_000
183
+ fit_sampling: Literal["random", "amp_reweighted"] = "amp_reweighted"
184
+
185
+ # initial detection
186
+ nn_denoiser_max_waveforms_fit: int = 250_000
187
+ nn_denoiser_class_name: str = "SingleChannelWaveformDenoiser"
188
+ nn_denoiser_pretrained_path: str | None = argfield(
189
+ default=default_pretrained_path, arg_type=str_or_none
190
+ )
191
+ do_tpca_denoise: bool = True
192
+ first_denoiser_thinning: float = 0.5
193
+ realign_to_denoiser: bool = True
194
+ use_nn_in_subtraction: bool = True
195
+ use_singlechan_templates: bool = False
196
+
197
+ # matching
198
+ matching_template_type: Literal["individual_compressed_upsampled", "drifty"] = (
199
+ "drifty"
200
+ )
201
+ matching_up_method: Literal["interpolation", "keys3", "keys4", "direct"] = "keys4"
202
+ matching_cd_iter: int = 0
203
+ matching_coarse_cd: bool = True
204
+ postprocessing_merge_threshold: float = 0.025
205
+ template_spikes_per_unit: int = 500
206
+ template_reduction: Literal["mean", "median"] = "mean"
207
+ template_denoising_method: Literal["none", "exp_weighted", "t", "loot"] = (
208
+ "exp_weighted"
209
+ )
210
+ template_mix_zero: bool = False
211
+ template_mix_svd: bool = True
212
+ min_template_snr: float = 40.0
213
+ min_template_count: int = 50
214
+ channel_selection_radius: float | None = argfield(
215
+ default=None, arg_type=float_or_none
216
+ )
217
+ always_recompute_tsvd: bool = True
218
+ matching_template_min_amplitude: float = 0.0
219
+ realign_strategy: RealignStrategy = "snr_weighted_trough_factor"
220
+ trough_factor: float = 3.0
221
+ whiten_matching: bool = False
222
+ matching_fp_control: bool = False
223
+
224
+ # interpolation for features
225
+ interp_method: InterpMethod = "kriging"
226
+ interp_kernel: InterpKernel = "thinplate"
227
+ extrap_method: InterpMethod | None = None
228
+ extrap_kernel: InterpKernel | None = None
229
+ kriging_poly_degree: int = 1
230
+ interp_sigma: float = 10.0
231
+ rq_alpha: float = 0.5
232
+ smoothing_lambda: float = 0.0
233
+
234
+ # initial clustering
235
+ initial_euclidean_complete_only: bool = False
236
+ initial_cosine_complete_only: bool = False
237
+ initial_amp_feat: bool = False
238
+ initial_signed_amp_feat: bool = True
239
+ initial_pc_feats: int = 5
240
+ initial_pc_transform: Literal["log", "sqrt", "none"] = "none"
241
+ initial_pc_scale: float = 2.0
242
+ initial_pc_pre_scale: float = 0.5
243
+ motion_aware_clustering: bool = True
244
+ clustering_workers: int = 5
245
+ clustering_max_spikes: Annotated[int, Field(gt=0)] = 500_000
246
+ pre_refinement_merge: bool = True
247
+ pre_refinement_merge_metric: str = "normeuc"
248
+ pre_refinement_merge_threshold: float = 0.1
249
+ use_hellinger: bool = True
250
+ density_bandwidth: Annotated[float, Field(gt=0)] = 5.0
251
+ component_overlap: float = 0.95
252
+ hellinger_strong: float = 0.0
253
+ hellinger_weak: float = 0.0
254
+ dpc_mop: bool = True
255
+ n_neighbors_search: int | None = argfield(default=50, arg_type=int_or_none)
256
+
257
+ # gaussian mixture high level
258
+ truncated: bool = True
259
+ initial_rank: int | None = argfield(default=None, arg_type=int_or_none)
260
+ initialize_at_rank_0: bool = False
261
+ signal_rank: Annotated[int, Field(ge=0)] = 5
262
+ gmm_max_spikes: Annotated[int, Field(gt=0)] = 1000 * 1024
263
+ kmeansk: int = 4
264
+ min_cluster_size: int = 25
265
+
266
+ # gausian mixture low level
267
+ n_refinement_iters: int = 1
268
+ n_em_iters: int = 250
269
+ channels_strategy: Literal["count", "all"] = "count"
270
+ gmm_cl_alpha: float = 1.0
271
+ gmm_em_atol: float = 5e-3
272
+ gmm_metric: Literal["cosine", "normeuc"] = "normeuc"
273
+ gmm_n_candidates: int = 3
274
+ gmm_n_search: int | None = argfield(default=None, arg_type=int_or_none)
275
+ gmm_val_proportion: Annotated[float, Field(gt=0)] = 0.25
276
+ initial_basis_shrinkage: float = 1.0
277
+ prior_pseudocount: float = 0.0
278
+ cov_kind: str = "factorizednoise"
279
+ gmm_euclidean_threshold: float = 5.0
280
+ gmm_kl_threshold: float = 2.0
281
+ gmm_cosine_threshold: float = 0.8
282
+ gmm_normeuc_threshold: float = 1.0
283
+
284
+ # store extra intermediates
285
+ save_subtracted_waveforms: bool = False
286
+ save_collisioncleaned_waveforms: bool = False
287
+ precomputed_templates_npz: str | None = argfield(default=None, arg_type=str_or_none)
288
+ save_everything_on_error: bool = False
@@ -0,0 +1,2 @@
1
+ from .detect import detect_and_deduplicate, singlechan_template_detect_and_deduplicate
2
+ from .detect_filters import convexity_filter