DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,202 @@
1
+ from typing import cast, Self
2
+
3
+ import h5py
4
+ import numpy as np
5
+ import torch
6
+
7
+ from ..util.internal_config import ClusteringFeaturesConfig
8
+ from ..util import drift_util, interpolation_util
9
+ from ..util.py_util import databag
10
+ from ..util.waveform_util import single_channel_index
11
+ from . import cluster_util
12
+
13
+ default_clustering_features_cfg = ClusteringFeaturesConfig()
14
+ minimal_features_cfg = ClusteringFeaturesConfig(
15
+ n_main_channel_pcs=0,
16
+ use_amplitude=False,
17
+ use_signed_amplitude=False,
18
+ use_x=False,
19
+ use_z=False,
20
+ )
21
+
22
+
23
+ def get_clustering_features(
24
+ recording,
25
+ sorting,
26
+ motion_est=None,
27
+ clustering_features_cfg: (
28
+ ClusteringFeaturesConfig | None
29
+ ) = default_clustering_features_cfg,
30
+ ) -> "SimpleMatrixFeatures":
31
+ if clustering_features_cfg is None:
32
+ clustering_features_cfg = minimal_features_cfg
33
+ if clustering_features_cfg.features_type == "simple_matrix":
34
+ return SimpleMatrixFeatures.from_config(
35
+ recording, sorting, motion_est, clustering_features_cfg
36
+ )
37
+ assert False
38
+
39
+
40
+ @databag
41
+ class SimpleMatrixFeatures:
42
+ n: int
43
+ features: np.ndarray
44
+ x: np.ndarray
45
+ z: np.ndarray
46
+ z_reg: np.ndarray
47
+ xyza: np.ndarray
48
+ signed_amplitudes: np.ndarray
49
+ amplitudes: np.ndarray
50
+
51
+ def mask(self, mask) -> Self:
52
+ x = self.x[mask]
53
+ return self.__class__(
54
+ n=x.shape[0],
55
+ features=self.features[mask],
56
+ x=x,
57
+ z=self.z[mask],
58
+ z_reg=self.z_reg[mask],
59
+ xyza=self.xyza[mask],
60
+ signed_amplitudes=self.signed_amplitudes[mask],
61
+ amplitudes=self.amplitudes[mask],
62
+ )
63
+
64
+ @classmethod
65
+ def from_config(cls, recording, sorting, motion_est, clustering_features_cfg):
66
+ assert clustering_features_cfg.features_type == "simple_matrix"
67
+
68
+ xyza = getattr(sorting, clustering_features_cfg.localizations_dataset_name)
69
+ x = xyza[:, 0]
70
+ z_reg = z = xyza[:, 2]
71
+ if motion_est is not None:
72
+ z_reg = motion_est.correct_s(sorting.times_seconds, z)
73
+
74
+ features = []
75
+
76
+ if clustering_features_cfg.use_z:
77
+ if clustering_features_cfg.motion_aware:
78
+ features.append(z_reg[:, None])
79
+ else:
80
+ features.append(z[:, None])
81
+
82
+ if clustering_features_cfg.use_x:
83
+ features.append(x[:, None] * clustering_features_cfg.x_scale)
84
+
85
+ amp = getattr(sorting, clustering_features_cfg.amplitudes_dataset_name)
86
+ if clustering_features_cfg.use_amplitude:
87
+ assert amp is not None
88
+ ampft = amp.copy()
89
+ if clustering_features_cfg.log_transform_amplitude:
90
+ ampft = np.log(clustering_features_cfg.amp_log_c + ampft)
91
+ ampft *= clustering_features_cfg.amp_scale
92
+ features.append(ampft[:, None])
93
+
94
+ v = getattr(sorting, clustering_features_cfg.voltages_dataset_name, None)
95
+ if v is None:
96
+ samp = amp.copy()
97
+ else:
98
+ samp = amp * np.sign(v)
99
+
100
+ if clustering_features_cfg.use_signed_amplitude:
101
+ samp *= clustering_features_cfg.amp_scale
102
+ features.append(clustering_features_cfg.amp_scale * samp[:, None])
103
+
104
+ do_pcs = bool(clustering_features_cfg.n_main_channel_pcs)
105
+ pcs = None
106
+ if do_pcs and not clustering_features_cfg.motion_aware:
107
+ pcs = cluster_util.get_main_channel_pcs(
108
+ sorting,
109
+ rank=clustering_features_cfg.n_main_channel_pcs,
110
+ dataset_name=clustering_features_cfg.pca_dataset_name,
111
+ )
112
+ elif do_pcs and clustering_features_cfg.motion_aware:
113
+ geom = sorting.geom
114
+ if motion_est is None:
115
+ registered_geom = geom
116
+ else:
117
+ registered_geom = drift_util.registered_geometry(
118
+ geom, motion_est=motion_est
119
+ )
120
+ res = drift_util.get_shift_info(sorting, motion_est, geom)
121
+ channels, shifts, n_pitches_shift = res
122
+ mainchan_ci = single_channel_index(len(geom))
123
+ schan, *_ = drift_util.get_stable_channels(
124
+ geom,
125
+ channels,
126
+ mainchan_ci,
127
+ registered_geom,
128
+ n_pitches_shift,
129
+ workers=clustering_features_cfg.workers,
130
+ )
131
+ mask = np.ones((1,), dtype=bool)
132
+ mask = np.broadcast_to(mask, len(schan))
133
+ with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
134
+ pcs = interpolation_util.interpolate_by_chunk(
135
+ mask=mask,
136
+ dataset=h5[clustering_features_cfg.pca_dataset_name],
137
+ geom=geom,
138
+ channel_index=cast(h5py.Dataset, h5["channel_index"])[:],
139
+ channels=sorting.channels,
140
+ shifts=shifts,
141
+ registered_geom=registered_geom,
142
+ target_channels=schan,
143
+ params=clustering_features_cfg.interp_params,
144
+ )
145
+ pcs = pcs[:, : clustering_features_cfg.n_main_channel_pcs, 0]
146
+
147
+ if do_pcs:
148
+ assert pcs is not None
149
+ if clustering_features_cfg.pc_transform == "log":
150
+ pcs = signed_log1p(
151
+ pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
152
+ )
153
+ elif clustering_features_cfg.pc_transform == "sqrt":
154
+ pcs = signed_sqrt_transform(
155
+ pcs, pre_scale=clustering_features_cfg.pc_pre_transform_scale
156
+ )
157
+ else:
158
+ assert clustering_features_cfg.pc_transform in ("none", None)
159
+ pcs *= clustering_features_cfg.pc_scale
160
+ if torch.is_tensor(pcs):
161
+ pcs = pcs.numpy(force=True)
162
+ features.append(pcs)
163
+
164
+ n = x.shape[0]
165
+ if len(features):
166
+ features = np.concatenate(features, axis=1)
167
+ else:
168
+ features = np.empty((n, 0))
169
+ return cls(
170
+ n=n,
171
+ features=features,
172
+ x=x,
173
+ z=z,
174
+ z_reg=z_reg,
175
+ xyza=xyza,
176
+ amplitudes=amp,
177
+ signed_amplitudes=samp,
178
+ )
179
+
180
+
181
+ def signed_log1p(x, pre_scale=1.0):
182
+ """sgn(x) * log(1+|x|*pre_scale)"""
183
+ x = torch.asarray(x)
184
+ xx = x.abs()
185
+ if pre_scale != 1.0:
186
+ xx.mul_(pre_scale)
187
+ torch.log1p(xx, out=xx)
188
+ xx.mul_(torch.sign(x))
189
+ return xx
190
+
191
+
192
+ def signed_sqrt_transform(x, pre_scale=1.0):
193
+ """sgn(x) * (sqrt(1 + |x|*pre_scale) - 1)"""
194
+ x = torch.asarray(x)
195
+ xx = x.abs()
196
+ if pre_scale != 1.0:
197
+ xx.mul_(pre_scale)
198
+ xx.add_(1.0)
199
+ xx.sqrt_()
200
+ xx.sub_(1.0)
201
+ xx.mul_(torch.sign(x))
202
+ return xx