DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,313 @@
1
+ """Helper functions for localizing things other than torch Tensors"""
2
+
3
+ import threading
4
+
5
+ import h5py
6
+ import numpy as np
7
+ import torch
8
+ from tqdm.auto import tqdm
9
+
10
+ from ..util.multiprocessing_util import get_pool
11
+ from ..util.spiketorch import ptp
12
+ from .localize_torch import localize_amplitude_vectors, vmap_point_source_find_alpha
13
+
14
+
15
+ def localize_waveforms(
16
+ waveforms,
17
+ geom,
18
+ main_channels=None,
19
+ channel_index=None,
20
+ radius=None,
21
+ n_channels_subset=None,
22
+ ):
23
+ C, dim = geom.shape
24
+ amp_vecs = ptp(waveforms, 1)
25
+ if main_channels is None:
26
+ main_channels = amp_vecs.argmax(1)
27
+
28
+ return localize_amplitude_vectors(
29
+ amp_vecs,
30
+ geom,
31
+ main_channels,
32
+ channel_index=channel_index,
33
+ radius=radius,
34
+ n_channels_subset=n_channels_subset,
35
+ )
36
+
37
+
38
+ def localize_hdf5(
39
+ hdf5_filename,
40
+ radius=None,
41
+ n_channels_subset=None,
42
+ output_dataset_name="point_source_localizations",
43
+ main_channels_dataset_name="channels",
44
+ amplitude_vectors_dataset_name="denoised_peak_amplitude_vectors",
45
+ channel_index_dataset_name="channel_index",
46
+ geometry_dataset_name="geom",
47
+ spikes_per_batch=10 * 1024,
48
+ overwrite=False,
49
+ show_progress=True,
50
+ device=None,
51
+ localization_model="pointsource",
52
+ logbarrier=True,
53
+ n_jobs=0,
54
+ ):
55
+ """Run localization on a HDF5 file with stored amplitude vectors
56
+
57
+ When this is run after a peeling pipeline which includes an AmplitudeVector
58
+ feature, that's equivalent to having had a PointSourceLocalization feature.
59
+ But, it's faster to do this after the fact.
60
+ """
61
+ if device is None:
62
+ device = "cuda" if torch.cuda.is_available() else "cpu"
63
+ device = torch.device(device)
64
+
65
+ done, do_delete, next_batch_start = check_resume_or_overwrite(
66
+ hdf5_filename,
67
+ output_dataset_name,
68
+ main_channels_dataset_name,
69
+ overwrite=overwrite,
70
+ )
71
+ if done:
72
+ return
73
+
74
+ try:
75
+ n_jobs, Executor, context, queue = get_pool(
76
+ n_jobs,
77
+ with_rank_queue=True,
78
+ rank_queue_empty=True,
79
+ cls="ProcessPoolExecutor",
80
+ )
81
+ with Executor(
82
+ max_workers=n_jobs,
83
+ mp_context=context,
84
+ initializer=_h5_localize_init,
85
+ initargs=(
86
+ queue,
87
+ hdf5_filename,
88
+ main_channels_dataset_name,
89
+ amplitude_vectors_dataset_name,
90
+ channel_index_dataset_name,
91
+ geometry_dataset_name,
92
+ radius,
93
+ n_channels_subset,
94
+ localization_model,
95
+ device,
96
+ spikes_per_batch,
97
+ logbarrier,
98
+ ),
99
+ ) as pool:
100
+ with h5py.File(hdf5_filename, "r+", locking=False) as h5:
101
+ n_spikes = h5[main_channels_dataset_name].shape[0]
102
+ if do_delete:
103
+ del h5[output_dataset_name]
104
+ localizations_dataset = h5.require_dataset(
105
+ output_dataset_name,
106
+ shape=(n_spikes, 4),
107
+ dtype=np.float64,
108
+ fillvalue=np.nan,
109
+ exact=True,
110
+ )
111
+
112
+ # workers are blocked waiting for their ranks, which
113
+ # allows us to put the h5 in swmr before they open it
114
+ if not h5.swmr_mode:
115
+ h5.swmr_mode = True
116
+ for rank in range(n_jobs):
117
+ queue.put(rank)
118
+
119
+ batches = range(next_batch_start, n_spikes, spikes_per_batch)
120
+ results = pool.map(_h5_localize_job, batches)
121
+ if show_progress:
122
+ results = tqdm(results, total=len(batches), desc="Localization")
123
+ for start_ix, end_ix, xyza_batch in results:
124
+ localizations_dataset[start_ix:end_ix] = xyza_batch
125
+ pool.shutdown()
126
+ finally:
127
+ if hasattr(_loc_context, "ctx"):
128
+ del _loc_context.ctx
129
+
130
+
131
+ def check_resume_or_overwrite(
132
+ hdf5_filename,
133
+ output_dataset_name,
134
+ main_channels_dataset_name,
135
+ overwrite=False,
136
+ ):
137
+ done = False
138
+ do_delete = False
139
+ next_batch_start = 0
140
+ with h5py.File(hdf5_filename, "r", locking=False) as h5:
141
+ if output_dataset_name in h5:
142
+ n_spikes = h5[main_channels_dataset_name].shape[0]
143
+ shape = h5[output_dataset_name].shape
144
+ if overwrite and shape != (n_spikes, 4):
145
+ do_delete = True
146
+ elif shape != (n_spikes, 4):
147
+ raise ValueError(
148
+ f"The {output_dataset_name} dataset in {hdf5_filename} "
149
+ f"has unexpected shape {shape}, where we expected {(n_spikes, 4)}."
150
+ )
151
+ else:
152
+ # else, we are resuming
153
+ nan_ix = np.flatnonzero(np.isnan(h5[output_dataset_name][:, 0]))
154
+ if nan_ix.size:
155
+ next_batch_start = nan_ix[0]
156
+ else:
157
+ done = True
158
+ return done, do_delete, next_batch_start
159
+
160
+
161
+ class H5LocalizationContext:
162
+ def __init__(
163
+ self,
164
+ hdf5_filename,
165
+ main_channels_dataset_name,
166
+ amplitude_vectors_dataset_name,
167
+ channel_index_dataset_name,
168
+ geometry_dataset_name,
169
+ device,
170
+ spikes_per_batch,
171
+ radius,
172
+ n_channels_subset,
173
+ localization_model,
174
+ logbarrier,
175
+ ):
176
+ h5 = h5py.File(hdf5_filename, "r", swmr=True, locking=False)
177
+ channels = h5[main_channels_dataset_name][:]
178
+ amp_vecs = h5[amplitude_vectors_dataset_name]
179
+ channel_index = h5[channel_index_dataset_name][:]
180
+ geom = h5[geometry_dataset_name][:]
181
+
182
+ assert geom.shape == (channel_index.shape[0], 2)
183
+ assert amp_vecs.shape == (*channels.shape, channel_index.shape[1])
184
+
185
+ self.geom = torch.tensor(geom, device=device)
186
+ self.channel_index = torch.tensor(channel_index, device=device)
187
+ self.channels = torch.tensor(channels, device=device)
188
+ self.amp_vecs = amp_vecs
189
+ self.device = device
190
+ self.n_spikes = channels.shape[0]
191
+ self.spikes_per_batch = spikes_per_batch
192
+ self.radius = radius
193
+ self.n_channels_subset = n_channels_subset
194
+ self.localization_model = localization_model
195
+ self.logbarrier = logbarrier
196
+
197
+
198
+ _loc_context = threading.local()
199
+
200
+
201
+ def _h5_localize_init(
202
+ rank_queue,
203
+ hdf5_filename,
204
+ main_channels_dataset_name,
205
+ amplitude_vectors_dataset_name,
206
+ channel_index_dataset_name,
207
+ geometry_dataset_name,
208
+ radius,
209
+ n_channels_subset,
210
+ localization_model,
211
+ device,
212
+ spikes_per_batch,
213
+ logbarrier,
214
+ ):
215
+ my_rank = rank_queue.get()
216
+ if device is None:
217
+ device = "cuda" if torch.cuda.is_available() else "cpu"
218
+ device = torch.device(device)
219
+ if device.type == "cuda" and device.index is None:
220
+ if torch.cuda.device_count() > 1:
221
+ device = torch.device("cuda", index=my_rank % torch.cuda.device_count())
222
+
223
+ _loc_context.ctx = H5LocalizationContext(
224
+ hdf5_filename,
225
+ main_channels_dataset_name,
226
+ amplitude_vectors_dataset_name,
227
+ channel_index_dataset_name,
228
+ geometry_dataset_name,
229
+ device,
230
+ spikes_per_batch,
231
+ radius,
232
+ n_channels_subset,
233
+ localization_model,
234
+ logbarrier,
235
+ )
236
+
237
+
238
+ def _h5_localize_job(start_ix):
239
+ p = _loc_context.ctx
240
+
241
+ end_ix = min(p.n_spikes, start_ix + p.spikes_per_batch)
242
+ channels_batch = p.channels[start_ix:end_ix]
243
+ amp_vecs_batch = torch.tensor(p.amp_vecs[start_ix:end_ix], device=p.device)
244
+
245
+ with torch.enable_grad():
246
+ locs = localize_amplitude_vectors(
247
+ amp_vecs_batch,
248
+ p.geom,
249
+ channels_batch,
250
+ channel_index=p.channel_index,
251
+ radius=p.radius,
252
+ n_channels_subset=p.n_channels_subset,
253
+ model=p.localization_model,
254
+ logbarrier=p.logbarrier,
255
+ )
256
+ xyza_batch = np.c_[
257
+ locs["x"].cpu().numpy(),
258
+ locs["y"].cpu().numpy(),
259
+ locs["z_abs"].cpu().numpy(),
260
+ locs["alpha"].cpu().numpy(),
261
+ ]
262
+ return start_ix, end_ix, xyza_batch
263
+
264
+
265
+ def point_source_mse(locs, amp_vecs, channels, channel_index, geom):
266
+ is_torch = torch.is_tensor(locs)
267
+ locs = torch.asarray(locs)
268
+ amp_vecs = torch.asarray(amp_vecs)
269
+ channels = torch.asarray(channels)
270
+ channel_index = torch.asarray(channel_index)
271
+ pgeom = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
272
+ geom = torch.asarray(geom)
273
+ pgeom = torch.asarray(pgeom)
274
+
275
+ neighborhoods = channel_index[channels]
276
+ invalid = neighborhoods == len(geom)
277
+ dxz = pgeom[neighborhoods]
278
+
279
+ if locs.shape[1] == 3:
280
+ channel_mask = torch.logical_not(invalid).to(torch.float)
281
+ alpha = vmap_point_source_find_alpha(
282
+ torch.asarray(amp_vecs).nan_to_num(),
283
+ channel_mask,
284
+ *torch.asarray(locs).T,
285
+ torch.asarray(dxz).nan_to_num(),
286
+ )
287
+ alpha = alpha.numpy()
288
+ else:
289
+ alpha = locs[:, 3]
290
+ locs = locs[:, :3]
291
+
292
+ x, y, z = locs.T
293
+
294
+ dxz[:, :, 0] -= x[:, None]
295
+ dxz[:, :, 1] -= z[:, None]
296
+ dxz.square_()
297
+ pred = dxz.sum(2)
298
+ pred += y[:, None] ** 2
299
+ pred.sqrt_()
300
+ pred[invalid] = 1.0
301
+ pred.reciprocal_()
302
+ pred *= alpha[:, None]
303
+
304
+ mse = amp_vecs - pred
305
+ mse[invalid] = 0.0
306
+ mse.square_()
307
+ nnz = channel_index.shape[1] - invalid.sum(1)
308
+ mse = mse.sum(1) / nnz
309
+
310
+ if not is_torch:
311
+ mse = mse.numpy()
312
+
313
+ return mse