DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,191 @@
1
+ from dataclasses import replace
2
+ import gc
3
+
4
+ import torch
5
+ import h5py
6
+
7
+ from ..util.internal_config import default_refinement_cfg
8
+ from ..util import job_util, noise_util, data_util, spiketorch
9
+ from ..util.logging_util import get_logger
10
+ from ..util.main_util import ds_save_intermediate_labels
11
+ from ..transform.temporal_pca import BaseTemporalPCA
12
+ from ..templates import TemplateData
13
+ from .cluster_util import agglomerate, reorder_by_depth
14
+ from .split import split_clusters
15
+ from .merge import merge_templates
16
+ from .gmm.stable_features import StableSpikeDataset
17
+
18
+
19
+ logger = get_logger(__name__)
20
+
21
+
22
+ def split_merge(
23
+ *,
24
+ recording,
25
+ sorting,
26
+ motion_est=None,
27
+ split_cfg,
28
+ merge_cfg,
29
+ merge_template_cfg,
30
+ computation_cfg=None,
31
+ ):
32
+ if computation_cfg is None:
33
+ computation_cfg = job_util.get_global_computation_config()
34
+
35
+ split_sorting = split_clusters(
36
+ sorting,
37
+ split_strategy=split_cfg.split_strategy,
38
+ recursive=split_cfg.recursive_split,
39
+ n_jobs=computation_cfg.actual_n_jobs(),
40
+ motion_est=motion_est,
41
+ )
42
+ template_data = TemplateData.from_config(
43
+ recording=recording,
44
+ sorting=sorting,
45
+ motion_est=motion_est,
46
+ template_cfg=merge_template_cfg,
47
+ )
48
+ merge_sorting = merge_templates(
49
+ sorting=split_sorting,
50
+ template_data=template_data,
51
+ merge_distance_threshold=merge_cfg.merge_distance_threshold,
52
+ min_spatial_cosine=merge_cfg.min_spatial_cosine,
53
+ linkage=merge_cfg.linkage,
54
+ computation_cfg=computation_cfg,
55
+ )["sorting"]
56
+
57
+ return merge_sorting
58
+
59
+
60
+ def get_noise_log_priors(noise, sorting, refinement_cfg):
61
+ from dartsort.templates import TemplateData
62
+
63
+ if not refinement_cfg.noise_fp_correction:
64
+ return None
65
+
66
+ h5_name = sorting.parent_h5_path
67
+ if h5_name is None:
68
+ return None
69
+ stem = h5_name.stem
70
+
71
+ if stem.startswith("matching"):
72
+ model_dir = h5_name.parent / f"{stem}_models"
73
+ templates_npz = model_dir / "template_data.npz"
74
+ if not templates_npz.exists():
75
+ raise ValueError(f"{templates_npz} is not there?")
76
+
77
+ with h5py.File(h5_name, "r", locking=False) as h5:
78
+ matching_labels = h5["labels"][:] # type: ignore
79
+
80
+ template_data = TemplateData.from_npz(templates_npz)
81
+ tpca = data_util.get_tpca(sorting)
82
+ assert isinstance(tpca, BaseTemporalPCA)
83
+ if (sl := getattr(tpca, "temporal_slice", None)) is not None:
84
+ temps_tpca = torch.asarray(template_data.templates[:, sl])
85
+ else:
86
+ temps_tpca = torch.asarray(template_data.templates)
87
+
88
+ n, t, c = temps_tpca.shape
89
+ temps_tpca = temps_tpca.permute(0, 2, 1).reshape(n * c, t)
90
+ temps_tpca = tpca._transform_in_probe(temps_tpca)
91
+ temps_tpca = temps_tpca.reshape(n, c, -1).permute(0, 2, 1)
92
+
93
+ noise_log_priors = noise.detection_prior_log_prob(temps_tpca)
94
+ logger.dartsortdebug(
95
+ f"Got log priors ranging {noise_log_priors.min()}-{noise_log_priors.max()}."
96
+ )
97
+ noise_log_priors = noise_log_priors[matching_labels]
98
+
99
+ return noise_log_priors
100
+ elif stem.startswith("subtract"):
101
+ noise_log_priors = noise.channelwise_detection_prior_log_prob()
102
+ logger.dartsortdebug(
103
+ f"Got log priors ranging {noise_log_priors.min()}-{noise_log_priors.max()}."
104
+ )
105
+ noise_log_priors = noise_log_priors[sorting.channels]
106
+ return noise_log_priors
107
+ else:
108
+ return None
109
+
110
+
111
+ def pc_merge(sorting, refinement_cfg, motion_est=None, computation_cfg=None):
112
+ assert refinement_cfg.refinement_strategy == "pcmerge"
113
+ if computation_cfg is None:
114
+ computation_cfg = job_util.get_global_computation_config()
115
+ if not refinement_cfg.pc_merge_threshold:
116
+ return sorting
117
+
118
+ # remove blank labels just in case
119
+ sorting = data_util.subset_sorting_by_spike_count(sorting)
120
+ nu0 = sorting.labels.max() + 1
121
+ if not nu0:
122
+ return sorting
123
+
124
+ # subset the sorting to count per unit
125
+ subset_sorting = data_util.subsample_to_max_count(
126
+ sorting, max_spikes=refinement_cfg.pc_merge_spikes_per_unit
127
+ )
128
+
129
+ # make stable features, no need for core features though.
130
+ data = StableSpikeDataset.from_sorting(
131
+ subset_sorting,
132
+ motion_est=motion_est,
133
+ core_radius=None,
134
+ discard_triaged=True,
135
+ interp_params=refinement_cfg.interp_params.normalize(),
136
+ split_proportions=None,
137
+ split_names=("train",),
138
+ device=computation_cfg.actual_device(),
139
+ )
140
+
141
+ # average by unit
142
+ kept = data.kept_indices
143
+ n = len(kept)
144
+ x = data._train_extract_features.view(n, -1, data.n_channels_extract)
145
+ x = x[:, : refinement_cfg.pc_merge_rank]
146
+ labels = torch.from_numpy(subset_sorting.labels[kept]).to(x.device)
147
+ means, counts = spiketorch.average_by_label(
148
+ x, labels, data._train_extract_channels, data.n_channels
149
+ )
150
+
151
+ # compute distances
152
+ if refinement_cfg.pc_merge_metric == "cosine":
153
+ dists = spiketorch.cosine_distance(means)
154
+ elif refinement_cfg.pc_merge_metric == "maxz":
155
+ x = x.square_()
156
+ meansq, _ = spiketorch.average_by_label(
157
+ x, labels, data._train_extract_channels, data.n_channels
158
+ )
159
+ stddev = meansq.sub_(means.square()).sqrt_()
160
+ stddev = stddev.clamp_(min=torch.finfo(stddev.dtype).tiny)
161
+ stderr = stddev.div_(counts.sqrt()[:, None])
162
+ dists = spiketorch.maxz_distance(
163
+ means, stderr, counts, min_iou=refinement_cfg.pc_merge_min_iou
164
+ )
165
+ elif refinement_cfg.pc_merge_metric == "normeuc":
166
+ dists = spiketorch.weighted_normeuc_distance(
167
+ means, counts, min_iou=refinement_cfg.pc_merge_min_iou
168
+ )
169
+ elif refinement_cfg.pc_merge_metric == "normsup":
170
+ dists = spiketorch.weighted_normsup_distance(
171
+ means, counts, min_iou=refinement_cfg.pc_merge_min_iou
172
+ )
173
+ elif refinement_cfg.pc_merge_metric == "euclidean":
174
+ means = means.reshape(len(means), -1)
175
+ dists = torch.cdist(means, means).numpy(force=True)
176
+ else:
177
+ raise ValueError(f"Have not implemented {refinement_cfg.pc_merge_metric=}.")
178
+
179
+ # linkage
180
+ labels, ids = agglomerate(
181
+ sorting.labels,
182
+ dists,
183
+ linkage_method=refinement_cfg.pc_merge_linkage,
184
+ threshold=refinement_cfg.pc_merge_threshold,
185
+ )
186
+ logger.dartsortdebug(f"pc_merge: Unit count {nu0}->{ids.max() + 1}.")
187
+
188
+ sorting = sorting.ephemeral_replace(labels=labels)
189
+ sorting = reorder_by_depth(sorting, motion_est=motion_est)
190
+ return sorting
191
+
@@ -0,0 +1,109 @@
1
+ """Computing relocated waveforms used in clustering
2
+
3
+ These waveforms are first shifted by an integer number of
4
+ probe pitches to account for gross drift and then rescaled
5
+ on each channel according to the point source model to
6
+ account for sub-pitch motion.
7
+
8
+ This is in numpy, since it's part of the clustering which
9
+ is all just numpy, and that's why it lives here and not
10
+ in transform/ which has torch-based stuff for use during
11
+ peeling.
12
+ """
13
+
14
+ import numpy as np
15
+ from dartsort.util import drift_util
16
+
17
+
18
+ def relocated_waveforms_on_static_channels(
19
+ waveforms,
20
+ main_channels,
21
+ channel_index,
22
+ target_channels,
23
+ xyza_from,
24
+ z_to,
25
+ geom,
26
+ amplitude_vectors=None,
27
+ registered_geom=None,
28
+ target_kdtree=None,
29
+ match_distance=None,
30
+ fill_value=np.nan,
31
+ ):
32
+ """Compute relocated waveforms"""
33
+ two_d = waveforms.ndim == 2
34
+ if two_d:
35
+ waveforms = waveforms[:, None, :]
36
+ x, y, z_from = xyza_from[:, :3].T
37
+ if xyza_from.shape[1] == 4:
38
+ alpha = xyza_from[:, 3]
39
+ elif xyza_from.shape[1] == 3:
40
+ alpha = determine_alpha(
41
+ amplitude_vectors, x, y, z_from, geom, channels=channel_index[main_channels]
42
+ )
43
+ else:
44
+ assert False
45
+ if registered_geom is None:
46
+ registered_geom = geom
47
+
48
+ # -- handle coarse drift (larger than a pitch)
49
+ n_pitches_shift = drift_util.get_spike_pitch_shifts(
50
+ z_from, geom, registered_depths_um=z_to
51
+ )
52
+ shifted_waveforms = drift_util.get_waveforms_on_static_channels(
53
+ waveforms,
54
+ main_channels=main_channels,
55
+ channel_index=channel_index,
56
+ target_channels=target_channels,
57
+ n_pitches_shift=n_pitches_shift,
58
+ geom=geom,
59
+ registered_geom=registered_geom,
60
+ target_kdtree=target_kdtree,
61
+ match_distance=match_distance,
62
+ fill_value=fill_value,
63
+ )
64
+
65
+ # -- handle sub-pitch drift
66
+ # account for the already applied coarse drift correction
67
+ pitch = drift_util.get_pitch(geom)
68
+ z_from = z_from + pitch * n_pitches_shift
69
+ original_amplitudes = point_source_amplitude_vectors(
70
+ x, y, z_from, alpha, registered_geom, channels=target_channels
71
+ )
72
+ target_amplitudes = point_source_amplitude_vectors(
73
+ x, y, z_to, alpha, registered_geom, channels=target_channels
74
+ )
75
+ rescaling = target_amplitudes / original_amplitudes
76
+ shifted_waveforms *= rescaling[:, None, :]
77
+
78
+ if two_d:
79
+ shifted_waveforms = shifted_waveforms[:, 0, :]
80
+
81
+ return shifted_waveforms
82
+
83
+
84
+ def point_source_amplitude_vectors(x, y, z_abs, alpha, geom, channels=None):
85
+ """Point-source amplitudes on fixed channels from varying spike positions"""
86
+ if channels is not None:
87
+ geom = geom[channels]
88
+ if geom.ndim == 2:
89
+ geom = geom[None]
90
+
91
+ dx = geom[..., 0] - x[:, None]
92
+ dz = geom[..., 1] - z_abs[:, None]
93
+ dy = y[:, None]
94
+ denom = np.sqrt(dy * dy + dx * dx + dz * dz)
95
+ denom[denom < 1e-8] = 1
96
+
97
+ return alpha[:, None] / denom
98
+
99
+
100
+ def determine_alpha(ampvecs, x, y, z, geom, channels):
101
+ geom = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
102
+ mask = (channels < len(geom)).astype(ampvecs.dtype)
103
+ preds = point_source_amplitude_vectors(
104
+ x, y, z, np.ones_like(z), geom, channels=channels
105
+ )
106
+ alpha = np.nan_to_num(ampvecs * preds * mask).sum(axis=1) / np.nan_to_num(
107
+ ampvecs * ampvecs * mask
108
+ ).sum(axis=1)
109
+ return alpha