DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,269 @@
1
+ import tempfile
2
+ import warnings
3
+ from pathlib import Path
4
+
5
+ import h5py
6
+ import numpy as np
7
+ import torch
8
+
9
+ from dartsort.util import spiketorch, waveform_util
10
+
11
+
12
+ def get_singlechan_centroids(
13
+ singlechan_waveforms=None,
14
+ rec=None,
15
+ detection_threshold=6.0,
16
+ deduplication_radius=150.0,
17
+ trough_offset_samples=42,
18
+ spike_length_samples=121,
19
+ temporal_dedup_radius_samples=11,
20
+ n_kmeanspp_tries=10,
21
+ alignment_padding=20,
22
+ n_centroids=10,
23
+ pca_rank=8,
24
+ n_waveforms_fit=20_000,
25
+ taper=True,
26
+ taper_start=20,
27
+ taper_end=30,
28
+ kmeanspp_initial="random",
29
+ random_seed=0,
30
+ ):
31
+ """Kmeanspp in pca space"""
32
+ from dartsort.clustering import kmeans
33
+
34
+ full_length = spike_length_samples + 2 * alignment_padding
35
+ full_trough = trough_offset_samples + alignment_padding
36
+ trim_end = spike_length_samples + alignment_padding
37
+
38
+ # a user may have better waveforms than we can provide naively
39
+ # with thresholding detection below
40
+ if singlechan_waveforms is None:
41
+ singlechan_waveforms = get_singlechan_waveforms(
42
+ rec,
43
+ detection_threshold=detection_threshold,
44
+ deduplication_radius=deduplication_radius,
45
+ temporal_dedup_radius_samples=temporal_dedup_radius_samples,
46
+ n_waveforms_fit=n_waveforms_fit,
47
+ trough_offset_samples=full_trough,
48
+ spike_length_samples=full_length,
49
+ )
50
+ else:
51
+ assert singlechan_waveforms.shape[1] == full_length
52
+
53
+ # PCA embed
54
+ trim_waveforms = singlechan_waveforms[:, alignment_padding:trim_end]
55
+ q = min(pca_rank + 10, *trim_waveforms.shape)
56
+ u, s, v = torch.pca_lowrank(trim_waveforms, q=pca_rank + 10, center=False, niter=7)
57
+ u = u[:, :pca_rank]
58
+ s = s[:pca_rank]
59
+ v = v[:, :pca_rank]
60
+ eigs = s.sqrt() / np.sqrt(len(u) - 1)
61
+ embeds = u * eigs
62
+
63
+ # kmeans++ (iter=0). seems better than kmeans for this application.
64
+ kmeans_res = kmeans.kmeans(
65
+ embeds,
66
+ n_components=n_centroids,
67
+ n_iter=0,
68
+ n_kmeanspp_tries=n_kmeanspp_tries,
69
+ kmeanspp_initial=kmeanspp_initial,
70
+ random_state=random_seed,
71
+ )
72
+ labels = kmeans_res["labels"]
73
+
74
+ # get templates by averaging and realigning single chan wfs
75
+ temps = []
76
+ for j in range(n_centroids):
77
+ temp = singlechan_waveforms[labels == j].mean(0)
78
+ peak = temp.abs().argmax().cpu().item()
79
+ shift = peak - full_trough
80
+ if abs(shift) > alignment_padding:
81
+ wning = f"Odd {shift=} observed in singlechan templates with {alignment_padding=}."
82
+ warnings.warn(wning)
83
+ shift = min(max(shift, -alignment_padding), alignment_padding)
84
+ temp = temp[shift + alignment_padding : shift + trim_end]
85
+ temps.append(torch.asarray(temp))
86
+ temps = torch.stack(temps, dim=0)
87
+
88
+ # taper and normalize
89
+ if taper:
90
+ temps = spiketorch.taper(temps, t_start=taper_start, t_end=taper_end)
91
+ temps /= torch.linalg.norm(temps, dim=1, keepdim=True)
92
+
93
+ return temps
94
+
95
+
96
+ def get_singlechan_waveforms(
97
+ rec,
98
+ detection_threshold=6.0,
99
+ deduplication_radius=150.0,
100
+ n_waveforms_fit=20_000,
101
+ trough_offset_samples=42,
102
+ spike_length_samples=121,
103
+ temporal_dedup_radius_samples=11,
104
+ show_progress=False,
105
+ ):
106
+ from dartsort.peel.threshold import ThresholdAndFeaturize
107
+ from dartsort.transform import WaveformPipeline, Waveform
108
+
109
+ channel_index = waveform_util.single_channel_index(
110
+ rec.get_num_channels(), to_torch=True
111
+ )
112
+ deduplication_index = waveform_util.make_channel_index(
113
+ rec.get_channel_locations(), radius=deduplication_radius, to_torch=True
114
+ )
115
+ baz = "hi"
116
+ wfeat = Waveform(channel_index, name=baz, spike_length_samples=spike_length_samples)
117
+ thresh = ThresholdAndFeaturize(
118
+ rec,
119
+ channel_index,
120
+ trough_offset_samples=trough_offset_samples,
121
+ spike_length_samples=spike_length_samples,
122
+ featurization_pipeline=WaveformPipeline([wfeat]),
123
+ spatial_dedup_channel_index=deduplication_index,
124
+ detection_threshold=detection_threshold,
125
+ temporal_dedup_radius_samples=temporal_dedup_radius_samples,
126
+ n_waveforms_fit=n_waveforms_fit,
127
+ )
128
+
129
+ with tempfile.TemporaryDirectory(prefix="dartsorttmp") as tdir:
130
+ tmp_h5 = Path(tdir) / "singlechan_wfs.h5"
131
+ thresh.run_subsampled_peeling(tmp_h5, show_progress=show_progress)
132
+ with h5py.File(tmp_h5) as h5:
133
+ waveforms = h5[baz][:] # type: ignore
134
+
135
+ assert isinstance(waveforms, np.ndarray)
136
+ assert waveforms.shape[2] == 1
137
+ waveforms = torch.asarray(waveforms[:, :, 0])
138
+
139
+ return waveforms
140
+
141
+
142
+ def spatial_footprint_bank(
143
+ geom,
144
+ n_sigmas=3,
145
+ min_template_size=10.0,
146
+ dsigma=2.0,
147
+ max_distance=32.0,
148
+ dx=32.0,
149
+ eps=0.025,
150
+ ):
151
+ z_unique = np.unique(geom[:, 1])
152
+ dz = 1.0 if z_unique.size == 1 else np.median(np.diff(z_unique))
153
+
154
+ z_min, z_max = z_unique.min(), z_unique.max()
155
+ z_centers = np.arange(z_min, z_max + 1e-2, dz / 2)
156
+
157
+ x_min, x_max = geom[:, 0].min(), geom[:, 0].max()
158
+ nx = np.round((x_max - x_min) / (dx / 2)) + 1
159
+ x_centers = np.linspace(x_min, x_max, num=int(nx))
160
+
161
+ # get all centers in 2d and exclude ones which are too far from the probe
162
+ # which won't be any i guess without missing chans etc
163
+ xx, zz = np.meshgrid(x_centers, z_centers, indexing="ij")
164
+ xz = np.stack((xx, zz), axis=-1).reshape(-1, 2)
165
+ distsq = np.square(xz[:, None, :] - geom).sum(2)
166
+ valid = distsq < max_distance**2
167
+ valid = valid.any(1)
168
+ xz = xz[valid]
169
+ distsq = distsq[valid]
170
+ n_centers = len(distsq)
171
+ assert distsq.shape == (n_centers, len(geom))
172
+
173
+ # puff them up
174
+ sigmas = min_template_size * (1 + dsigma * np.arange(n_sigmas))
175
+ spatial_profiles = np.exp(-distsq[:, None, :] / sigmas[:, None] ** 2)
176
+ spatial_profiles = spatial_profiles.reshape(n_centers * n_sigmas, len(geom))
177
+ spatial_profiles /= np.linalg.norm(spatial_profiles, axis=1, keepdims=True)
178
+ spatial_profiles[spatial_profiles < eps] = 0.0
179
+
180
+ return spatial_profiles
181
+
182
+
183
+ def singlechan_to_library(
184
+ singlechan_templates,
185
+ geom,
186
+ trough_offset_samples=42,
187
+ n_sigmas=5,
188
+ min_template_size=10.0,
189
+ max_distance=32.0,
190
+ dx=32.0,
191
+ ):
192
+ from dartsort.templates import TemplateData
193
+
194
+ footprints = spatial_footprint_bank(
195
+ geom,
196
+ n_sigmas=n_sigmas,
197
+ min_template_size=min_template_size,
198
+ max_distance=max_distance,
199
+ dx=dx,
200
+ )
201
+ footprints = footprints.astype(str(singlechan_templates.dtype).split(".")[1])
202
+ if torch.is_tensor(singlechan_templates):
203
+ singlechan_templates = singlechan_templates.numpy(force=True)
204
+ nf, nc = footprints.shape
205
+ nsct, nt = singlechan_templates.shape
206
+ templates = footprints[:, None, None, :] * singlechan_templates[None, :, :, None]
207
+ assert templates.shape == (nf, nsct, nt, nc)
208
+
209
+ # note: this is footprint-major
210
+ templates = templates.reshape(nf * nsct, nt, nc)
211
+ templates /= np.linalg.norm(templates, axis=(1, 2), keepdims=True)
212
+
213
+ template_data = TemplateData(
214
+ templates,
215
+ unit_ids=np.arange(nsct * nf),
216
+ spike_counts=np.ones(nsct * nf, dtype=int),
217
+ trough_offset_samples=trough_offset_samples,
218
+ )
219
+ return footprints, template_data
220
+
221
+
222
+ def universal_templates_from_data(
223
+ singlechan_waveforms=None,
224
+ rec=None,
225
+ detection_threshold=6.0,
226
+ deduplication_radius=150.0,
227
+ trough_offset_samples=42,
228
+ spike_length_samples=121,
229
+ alignment_padding=20,
230
+ n_centroids=10,
231
+ pca_rank=8,
232
+ n_waveforms_fit=20_000,
233
+ taper=True,
234
+ taper_start=20,
235
+ taper_end=30,
236
+ kmeanspp_initial="random",
237
+ random_seed=0,
238
+ n_sigmas=5,
239
+ min_template_size=10.0,
240
+ max_distance=32.0,
241
+ dx=32.0,
242
+ ):
243
+ singlechan_centroids = get_singlechan_centroids(
244
+ singlechan_waveforms=singlechan_waveforms,
245
+ rec=rec,
246
+ detection_threshold=detection_threshold,
247
+ deduplication_radius=deduplication_radius,
248
+ trough_offset_samples=trough_offset_samples,
249
+ spike_length_samples=spike_length_samples,
250
+ alignment_padding=alignment_padding,
251
+ n_centroids=n_centroids,
252
+ pca_rank=pca_rank,
253
+ n_waveforms_fit=n_waveforms_fit,
254
+ taper=taper,
255
+ taper_start=taper_start,
256
+ taper_end=taper_end,
257
+ kmeanspp_initial=kmeanspp_initial,
258
+ random_seed=random_seed,
259
+ )
260
+ footprints, template_data = singlechan_to_library(
261
+ singlechan_centroids,
262
+ rec.get_channel_locations() if rec is not None else None,
263
+ trough_offset_samples=trough_offset_samples,
264
+ n_sigmas=n_sigmas,
265
+ min_template_size=min_template_size,
266
+ max_distance=max_distance,
267
+ dx=dx,
268
+ )
269
+ return singlechan_centroids, footprints, template_data