DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
dartsort/config.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Annotated, Literal
|
|
3
|
+
|
|
4
|
+
from pydantic import Field
|
|
5
|
+
|
|
6
|
+
from .util.cli_util import argfield
|
|
7
|
+
from .util.internal_config import (
|
|
8
|
+
default_pretrained_path,
|
|
9
|
+
InterpMethod,
|
|
10
|
+
InterpKernel,
|
|
11
|
+
RealignStrategy,
|
|
12
|
+
)
|
|
13
|
+
from .util.py_util import (
|
|
14
|
+
cfg_dataclass,
|
|
15
|
+
float_or_none,
|
|
16
|
+
int_or_float_or_none,
|
|
17
|
+
int_or_none,
|
|
18
|
+
str_or_none,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@cfg_dataclass
|
|
23
|
+
class DARTsortUserConfig:
|
|
24
|
+
"""User-facing configuration options"""
|
|
25
|
+
|
|
26
|
+
# -- high level behavior
|
|
27
|
+
dredge_only: bool = argfield(
|
|
28
|
+
False, doc="Whether to stop after initial localization and motion tracking."
|
|
29
|
+
)
|
|
30
|
+
matching_iterations: int = 1
|
|
31
|
+
|
|
32
|
+
# -- computer options
|
|
33
|
+
n_jobs_cpu: int = argfield(
|
|
34
|
+
default=0,
|
|
35
|
+
doc="Number of parallel workers to use when running on CPU. "
|
|
36
|
+
"0 means everything runs on the main thread.",
|
|
37
|
+
)
|
|
38
|
+
n_jobs_gpu: int = argfield(
|
|
39
|
+
default=0,
|
|
40
|
+
doc="Number of parallel workers to use when running on GPU. "
|
|
41
|
+
"0 means everything runs on the main thread.",
|
|
42
|
+
)
|
|
43
|
+
device: str | None = argfield(
|
|
44
|
+
default=None,
|
|
45
|
+
arg_type=str,
|
|
46
|
+
doc="The name of the PyTorch device to use. For example, 'cpu' "
|
|
47
|
+
"or 'cuda' or 'cuda:1'. If unset, uses n_jobs_gpu of your CUDA "
|
|
48
|
+
"GPUs if you have multiple, or else just the one, or your CPU.",
|
|
49
|
+
)
|
|
50
|
+
executor: str = "threading_unless_multigpu"
|
|
51
|
+
chunk_length_samples: int = 30_000
|
|
52
|
+
|
|
53
|
+
# -- storage behavior
|
|
54
|
+
work_in_tmpdir: bool = False
|
|
55
|
+
copy_recording_to_tmpdir: bool = False
|
|
56
|
+
workdir_copier: Literal["shutil", "rsync"] = "shutil"
|
|
57
|
+
workdir_follow_symlinks: bool = False
|
|
58
|
+
tmpdir_parent: str | Path | None = argfield(default=None, arg_type=str_or_none)
|
|
59
|
+
save_intermediates: bool = False
|
|
60
|
+
save_final_features: bool = True
|
|
61
|
+
|
|
62
|
+
# -- waveform snippet length parameters
|
|
63
|
+
ms_before: Annotated[float, Field(gt=0)] = argfield(
|
|
64
|
+
default=1.4,
|
|
65
|
+
doc="Length of time (ms) before trough (or peak) in waveform snippets. "
|
|
66
|
+
"Default value corresponds to 42 samples at 30kHz.",
|
|
67
|
+
)
|
|
68
|
+
ms_after: Annotated[float, Field(gt=0)] = argfield(
|
|
69
|
+
default=2.6 + 0.1 / 3,
|
|
70
|
+
doc="Length of time (ms) after trough (or peak) in waveform snippets. "
|
|
71
|
+
"Default value corresponds to 79 samples at 30kHz.",
|
|
72
|
+
)
|
|
73
|
+
alignment_ms: Annotated[float, Field(gt=0)] = argfield(
|
|
74
|
+
default=1.5,
|
|
75
|
+
doc="Time shift allowed when aligning events.",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# -- thresholds
|
|
79
|
+
peak_sign: Literal["neg", "both", "pos"] = argfield(
|
|
80
|
+
default="both",
|
|
81
|
+
doc="Allow only troughs or events of both signs when detecting threshold "
|
|
82
|
+
"crossings during initialization. Or positive only, if that's your thing.",
|
|
83
|
+
)
|
|
84
|
+
voltage_threshold: Annotated[float, Field(gt=0)] = argfield(
|
|
85
|
+
default=4.0,
|
|
86
|
+
doc="Threshold in standardized (SNR) voltage units for initial detection; "
|
|
87
|
+
"peaks or troughs larger than this value will be grabbed.",
|
|
88
|
+
)
|
|
89
|
+
matching_threshold: Annotated[float, Field(gt=0)] = argfield(
|
|
90
|
+
default=10.0,
|
|
91
|
+
doc="Template matching threshold. If subtracting a template leads "
|
|
92
|
+
"to at least this great of a decrease in the norm of the residual, "
|
|
93
|
+
"that match will be used.",
|
|
94
|
+
)
|
|
95
|
+
initial_threshold: Annotated[float, Field(gt=0)] = argfield(
|
|
96
|
+
default=12.0,
|
|
97
|
+
doc="Initial detection's neural net matching threshold. Same as "
|
|
98
|
+
"matching_threshold, except that a neural net is trying to guess "
|
|
99
|
+
"the true waveforms here, rather than using cluster templates.",
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# -- featurization length, radius, rank parameters
|
|
103
|
+
temporal_pca_rank: Annotated[int, Field(gt=0)] = argfield(
|
|
104
|
+
default=8, doc="Rank of temporal PCAs used in denoising and featurization."
|
|
105
|
+
)
|
|
106
|
+
feature_ms_before: Annotated[float, Field(gt=0)] = argfield(
|
|
107
|
+
default=0.75,
|
|
108
|
+
doc="As ms_before, but used only when computing PCA features in clustering.",
|
|
109
|
+
)
|
|
110
|
+
feature_ms_after: Annotated[float, Field(gt=0)] = argfield(
|
|
111
|
+
default=1.25,
|
|
112
|
+
doc="As ms_after, but used only when computing PCA features in clustering.",
|
|
113
|
+
)
|
|
114
|
+
subtraction_radius_um: Annotated[float, Field(gt=0)] = argfield(
|
|
115
|
+
default=200.0,
|
|
116
|
+
doc="Radius of neighborhoods around spike events extracted "
|
|
117
|
+
"when denoising and subtracting NN-denoised events.",
|
|
118
|
+
)
|
|
119
|
+
deduplication_radius_um: Annotated[float, Field(gt=0)] = argfield(
|
|
120
|
+
default=100.0,
|
|
121
|
+
doc="During initial detection, if two spike events occur at the "
|
|
122
|
+
"same time within this radius, then the smaller of the two is "
|
|
123
|
+
"ignored. But also all of the secondary channels of the big one, "
|
|
124
|
+
"which is important.",
|
|
125
|
+
)
|
|
126
|
+
featurization_radius_um: Annotated[float, Field(gt=0)] = argfield(
|
|
127
|
+
default=100.0,
|
|
128
|
+
doc="Radius around detection channel or template peak channel used "
|
|
129
|
+
"to extract spike features for clustering.",
|
|
130
|
+
)
|
|
131
|
+
fit_radius_um: Annotated[float, Field(gt=0)] = argfield(
|
|
132
|
+
default=75.0,
|
|
133
|
+
doc="Extraction radius when fitting features like PCA; "
|
|
134
|
+
"smaller than other radii to include less noise.",
|
|
135
|
+
)
|
|
136
|
+
localization_radius_um: Annotated[float, Field(gt=0)] = argfield(
|
|
137
|
+
default=100.0,
|
|
138
|
+
doc="Radius around main channel used when localizing spikes.",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# -- matching parameters
|
|
142
|
+
amplitude_scaling_stddev: Annotated[float, Field(ge=0)] = 0.01
|
|
143
|
+
amplitude_scaling_boundary: Annotated[float, Field(ge=0)] = 0.333
|
|
144
|
+
temporal_upsamples: Annotated[int, Field(ge=1)] = 8
|
|
145
|
+
|
|
146
|
+
# -- motion estimation parameters
|
|
147
|
+
do_motion_estimation: bool = argfield(
|
|
148
|
+
default=True,
|
|
149
|
+
doc="Set this to false if your data is super stable or already motion-corrected.",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# DREDge parameters
|
|
153
|
+
rigid: bool = argfield(
|
|
154
|
+
default=False, doc="Use rigid registration and ignore the window parameters."
|
|
155
|
+
)
|
|
156
|
+
probe_boundary_padding_um: float = 100.0
|
|
157
|
+
spatial_bin_length_um: Annotated[float, Field(gt=0)] = 1.0
|
|
158
|
+
temporal_bin_length_s: Annotated[float, Field(gt=0)] = 1.0
|
|
159
|
+
window_step_um: Annotated[float, Field(gt=0)] = 400.0
|
|
160
|
+
window_scale_um: Annotated[float, Field(gt=0)] = 450.0
|
|
161
|
+
window_margin_um: Annotated[float, Field(gt=0)] | None = None
|
|
162
|
+
max_dt_s: Annotated[float, Field(gt=0)] = 1000.0
|
|
163
|
+
max_disp_um: Annotated[float, Field(gt=0)] | None = None
|
|
164
|
+
correlation_threshold: Annotated[float, Field(gt=0, lt=1)] = 0.1
|
|
165
|
+
min_amplitude: float | None = argfield(default=None, arg_type=float)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@cfg_dataclass
|
|
169
|
+
class DeveloperConfig(DARTsortUserConfig):
|
|
170
|
+
"""Additional parameters for experiments. This API will never be stable."""
|
|
171
|
+
|
|
172
|
+
# high level behavior
|
|
173
|
+
initial_steps: Literal["neither", "split", "merge", "both"] = "split"
|
|
174
|
+
later_steps: Literal["neither", "split", "merge", "both"] = "merge"
|
|
175
|
+
detection_type: str = "subtract"
|
|
176
|
+
cluster_strategy: str = "dpc"
|
|
177
|
+
refinement_strategy: str = "tmm"
|
|
178
|
+
recluster_after_first_matching: bool = True
|
|
179
|
+
|
|
180
|
+
# general peeling
|
|
181
|
+
n_waveforms_fit: int = 40_000
|
|
182
|
+
max_waveforms_fit: int = 50_000
|
|
183
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "amp_reweighted"
|
|
184
|
+
|
|
185
|
+
# initial detection
|
|
186
|
+
nn_denoiser_max_waveforms_fit: int = 250_000
|
|
187
|
+
nn_denoiser_class_name: str = "SingleChannelWaveformDenoiser"
|
|
188
|
+
nn_denoiser_pretrained_path: str | None = argfield(
|
|
189
|
+
default=default_pretrained_path, arg_type=str_or_none
|
|
190
|
+
)
|
|
191
|
+
do_tpca_denoise: bool = True
|
|
192
|
+
first_denoiser_thinning: float = 0.5
|
|
193
|
+
realign_to_denoiser: bool = True
|
|
194
|
+
use_nn_in_subtraction: bool = True
|
|
195
|
+
use_singlechan_templates: bool = False
|
|
196
|
+
|
|
197
|
+
# matching
|
|
198
|
+
matching_template_type: Literal["individual_compressed_upsampled", "drifty"] = (
|
|
199
|
+
"drifty"
|
|
200
|
+
)
|
|
201
|
+
matching_up_method: Literal["interpolation", "keys3", "keys4", "direct"] = "keys4"
|
|
202
|
+
matching_cd_iter: int = 0
|
|
203
|
+
matching_coarse_cd: bool = True
|
|
204
|
+
postprocessing_merge_threshold: float = 0.025
|
|
205
|
+
template_spikes_per_unit: int = 500
|
|
206
|
+
template_reduction: Literal["mean", "median"] = "mean"
|
|
207
|
+
template_denoising_method: Literal["none", "exp_weighted", "t", "loot"] = (
|
|
208
|
+
"exp_weighted"
|
|
209
|
+
)
|
|
210
|
+
template_mix_zero: bool = False
|
|
211
|
+
template_mix_svd: bool = True
|
|
212
|
+
min_template_snr: float = 40.0
|
|
213
|
+
min_template_count: int = 50
|
|
214
|
+
channel_selection_radius: float | None = argfield(
|
|
215
|
+
default=None, arg_type=float_or_none
|
|
216
|
+
)
|
|
217
|
+
always_recompute_tsvd: bool = True
|
|
218
|
+
matching_template_min_amplitude: float = 0.0
|
|
219
|
+
realign_strategy: RealignStrategy = "snr_weighted_trough_factor"
|
|
220
|
+
trough_factor: float = 3.0
|
|
221
|
+
whiten_matching: bool = False
|
|
222
|
+
matching_fp_control: bool = False
|
|
223
|
+
|
|
224
|
+
# interpolation for features
|
|
225
|
+
interp_method: InterpMethod = "kriging"
|
|
226
|
+
interp_kernel: InterpKernel = "thinplate"
|
|
227
|
+
extrap_method: InterpMethod | None = None
|
|
228
|
+
extrap_kernel: InterpKernel | None = None
|
|
229
|
+
kriging_poly_degree: int = 1
|
|
230
|
+
interp_sigma: float = 10.0
|
|
231
|
+
rq_alpha: float = 0.5
|
|
232
|
+
smoothing_lambda: float = 0.0
|
|
233
|
+
|
|
234
|
+
# initial clustering
|
|
235
|
+
initial_euclidean_complete_only: bool = False
|
|
236
|
+
initial_cosine_complete_only: bool = False
|
|
237
|
+
initial_amp_feat: bool = False
|
|
238
|
+
initial_signed_amp_feat: bool = True
|
|
239
|
+
initial_pc_feats: int = 5
|
|
240
|
+
initial_pc_transform: Literal["log", "sqrt", "none"] = "none"
|
|
241
|
+
initial_pc_scale: float = 2.0
|
|
242
|
+
initial_pc_pre_scale: float = 0.5
|
|
243
|
+
motion_aware_clustering: bool = True
|
|
244
|
+
clustering_workers: int = 5
|
|
245
|
+
clustering_max_spikes: Annotated[int, Field(gt=0)] = 500_000
|
|
246
|
+
pre_refinement_merge: bool = True
|
|
247
|
+
pre_refinement_merge_metric: str = "normeuc"
|
|
248
|
+
pre_refinement_merge_threshold: float = 0.1
|
|
249
|
+
use_hellinger: bool = True
|
|
250
|
+
density_bandwidth: Annotated[float, Field(gt=0)] = 5.0
|
|
251
|
+
component_overlap: float = 0.95
|
|
252
|
+
hellinger_strong: float = 0.0
|
|
253
|
+
hellinger_weak: float = 0.0
|
|
254
|
+
dpc_mop: bool = True
|
|
255
|
+
n_neighbors_search: int | None = argfield(default=50, arg_type=int_or_none)
|
|
256
|
+
|
|
257
|
+
# gaussian mixture high level
|
|
258
|
+
truncated: bool = True
|
|
259
|
+
initial_rank: int | None = argfield(default=None, arg_type=int_or_none)
|
|
260
|
+
initialize_at_rank_0: bool = False
|
|
261
|
+
signal_rank: Annotated[int, Field(ge=0)] = 5
|
|
262
|
+
gmm_max_spikes: Annotated[int, Field(gt=0)] = 1000 * 1024
|
|
263
|
+
kmeansk: int = 4
|
|
264
|
+
min_cluster_size: int = 25
|
|
265
|
+
|
|
266
|
+
# gausian mixture low level
|
|
267
|
+
n_refinement_iters: int = 1
|
|
268
|
+
n_em_iters: int = 250
|
|
269
|
+
channels_strategy: Literal["count", "all"] = "count"
|
|
270
|
+
gmm_cl_alpha: float = 1.0
|
|
271
|
+
gmm_em_atol: float = 5e-3
|
|
272
|
+
gmm_metric: Literal["cosine", "normeuc"] = "normeuc"
|
|
273
|
+
gmm_n_candidates: int = 3
|
|
274
|
+
gmm_n_search: int | None = argfield(default=None, arg_type=int_or_none)
|
|
275
|
+
gmm_val_proportion: Annotated[float, Field(gt=0)] = 0.25
|
|
276
|
+
initial_basis_shrinkage: float = 1.0
|
|
277
|
+
prior_pseudocount: float = 0.0
|
|
278
|
+
cov_kind: str = "factorizednoise"
|
|
279
|
+
gmm_euclidean_threshold: float = 5.0
|
|
280
|
+
gmm_kl_threshold: float = 2.0
|
|
281
|
+
gmm_cosine_threshold: float = 0.8
|
|
282
|
+
gmm_normeuc_threshold: float = 1.0
|
|
283
|
+
|
|
284
|
+
# store extra intermediates
|
|
285
|
+
save_subtracted_waveforms: bool = False
|
|
286
|
+
save_collisioncleaned_waveforms: bool = False
|
|
287
|
+
precomputed_templates_npz: str | None = argfield(default=None, arg_type=str_or_none)
|
|
288
|
+
save_everything_on_error: bool = False
|