DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,771 @@
1
+ """Compute template waveforms, including accounting for drift and denoising
2
+
3
+ The class TemplateData in templates.py provides a friendlier interface,
4
+ where you can get templates using the TemplateConfig in config.py.
5
+ """
6
+
7
+ import threading
8
+ from typing import cast
9
+
10
+ import numpy as np
11
+ import torch
12
+ from scipy.spatial import KDTree
13
+ from scipy.spatial.distance import pdist
14
+ from sklearn.decomposition import TruncatedSVD, PCA
15
+ from tqdm.auto import tqdm
16
+
17
+ from ..util import spikeio
18
+ from ..util.data_util import DARTsortSorting, load_stored_tsvd
19
+ from ..util.drift_util import (
20
+ registered_template,
21
+ registered_geometry,
22
+ get_spike_pitch_shifts,
23
+ )
24
+ from ..util.multiprocessing_util import get_pool
25
+ from ..util.spiketorch import fast_nanmedian, ptp
26
+ from ..util.waveform_util import make_channel_index
27
+ from ..util.internal_config import RealignStrategy
28
+
29
+
30
+ def get_templates(
31
+ recording,
32
+ sorting,
33
+ trough_offset_samples=42,
34
+ spike_length_samples=121,
35
+ spikes_per_unit=500,
36
+ motion_est=None,
37
+ geom=None,
38
+ pitch_shifts=None,
39
+ registered_geom=None,
40
+ low_rank_denoising=True,
41
+ denoising_tsvd=None,
42
+ denoising_rank=5,
43
+ denoising_fit_radius=75.0,
44
+ denoising_spikes_fit=25_000,
45
+ recompute_tsvd=False,
46
+ denoising_snr_threshold=50.0,
47
+ min_fraction_at_shift=0.25,
48
+ min_count_at_shift=25,
49
+ with_raw_std_dev=False,
50
+ reducer=fast_nanmedian,
51
+ random_seed=0,
52
+ units_per_job=8,
53
+ n_jobs=0,
54
+ dtype=np.float32,
55
+ show_progress=True,
56
+ device=None,
57
+ localizations_dataset_name="point_source_localizations",
58
+ times_s_dataset_name="times_seconds",
59
+ ):
60
+ """Raw, denoised, and shifted templates
61
+
62
+ Low-level helper function which does the work of template computation for
63
+ the template classes elsewhere in this folder
64
+
65
+ Arguments
66
+ ---------
67
+ times, channels, labels : arrays of shape (n_spikes,)
68
+ The trough (or peak) times, main channels, and unit labels
69
+ geom : array of shape (n_channels, 2)
70
+ Probe channel geometry, needed to subsample channels when fitting
71
+ the low-rank denoising model, and also needed if the shifting
72
+ arguments are specified
73
+ pitch_shifts : int array of shape (n_spikes,)
74
+ When computing extended templates, these shifts are applied
75
+ before averaging
76
+ registered_geom : array of shape (n_channels_extended, 2)
77
+ Required if pitch_shifts is supplied. See drift_util.registered_geometry.
78
+ realign_peaks : bool
79
+ If True, a first round of raw templates are computed and used to shift
80
+ the spike times such that their peaks/troughs land on trough_offset_samples
81
+ trough_offset_samples, spike_length_samples : int
82
+ Waveform snippets will be loaded from times[i] - trough_offset_samples
83
+ to times[i] - trough_offset_samples + spike_length_samples
84
+ spikes_per_unit : int
85
+ Load at most this many randomly selected spikes per unit
86
+ low_rank_denoising : bool
87
+ Should we compute denoised templates? If not, raw averages.
88
+ denoising_model : sklearn Transformer
89
+ Pre-fit denoising model, in which case the next args are ignored
90
+ denoising_rank, denoising_fit_radius, denoising_spikes_fit
91
+ Parameters for the low rank model fit for denoising
92
+ denoising_snr_threshold : int
93
+ The SNR (=amplitude*sqrt(n_spikes)) threshold at which the
94
+ denoising is ignored and we just use the usual template
95
+ output_hdf5_filename : str or Path
96
+ Denoised and/or raw templates will be saved here under the dataset
97
+ names "raw_templates" and "denoised_templates"
98
+ keep_waveforms_in_hdf5 : bool
99
+ If True and output_hdf5_filename is supplied, waveforms extracted
100
+ for template computation are retained in the output hdf5. Else,
101
+ deleted to save disk space.
102
+ scratch_dir : str or Path
103
+ This is where a temporary directory will be made for intermediate
104
+ computations, if output_hdf5_filename is None. If it's left blank,
105
+ the tempfile default directory is used. If output_hdf5_file is not
106
+ None, that hdf5 file is used and this argument is ignored.
107
+
108
+ Returns
109
+ -------
110
+ dict whose keys vary based on the above arguments
111
+
112
+ """
113
+ # validate arguments
114
+ raw_only = not low_rank_denoising
115
+
116
+ # use geometry and motion estimate to get pitch shifts and reg geom
117
+ if pitch_shifts is None and motion_est is not None:
118
+ assert geom is not None
119
+ registered_geom = registered_geometry(geom, motion_est=motion_est)
120
+ try:
121
+ spike_depths_um = getattr(sorting, localizations_dataset_name)[:, 2]
122
+ except AttributeError:
123
+ raise ValueError(
124
+ "Sorting must contain localizations in the attribute "
125
+ f"{localizations_dataset_name=} when computing registered templates."
126
+ )
127
+ spike_times_s = getattr(sorting, times_s_dataset_name)
128
+ pitch_shifts = get_spike_pitch_shifts(
129
+ spike_depths_um, geom, times_s=spike_times_s, motion_est=motion_est
130
+ )
131
+ if pitch_shifts is not None:
132
+ assert registered_geom is not None
133
+ geom_kw = dict(registered_geom=registered_geom)
134
+ else:
135
+ geom_kw = dict(registered_geom=geom)
136
+
137
+ # fit tsvd
138
+ need_denoiser = denoising_tsvd is None and (low_rank_denoising)
139
+ if need_denoiser:
140
+ denoising_tsvd = fit_tsvd(
141
+ recording,
142
+ sorting,
143
+ dtype=dtype,
144
+ denoising_rank=denoising_rank,
145
+ denoising_fit_radius=denoising_fit_radius,
146
+ denoising_spikes_fit=denoising_spikes_fit,
147
+ trough_offset_samples=trough_offset_samples,
148
+ spike_length_samples=spike_length_samples,
149
+ recompute_tsvd=recompute_tsvd,
150
+ random_seed=random_seed,
151
+ )
152
+ elif not low_rank_denoising:
153
+ denoising_tsvd = None
154
+
155
+ # template logic
156
+ # for each unit, get shifted raw and denoised averages and channel SNRs
157
+ res = get_all_shifted_raw_and_low_rank_templates(
158
+ recording,
159
+ sorting,
160
+ registered_geom=registered_geom,
161
+ denoising_tsvd=denoising_tsvd,
162
+ pitch_shifts=pitch_shifts,
163
+ spikes_per_unit=spikes_per_unit,
164
+ reducer=reducer,
165
+ n_jobs=n_jobs,
166
+ units_per_job=units_per_job,
167
+ random_seed=random_seed,
168
+ show_progress=show_progress,
169
+ dtype=dtype,
170
+ trough_offset_samples=trough_offset_samples,
171
+ spike_length_samples=spike_length_samples,
172
+ min_fraction_at_shift=min_fraction_at_shift,
173
+ min_count_at_shift=min_count_at_shift,
174
+ with_raw_std_dev=with_raw_std_dev,
175
+ device=device,
176
+ )
177
+ (
178
+ unit_ids,
179
+ spike_counts,
180
+ raw_templates,
181
+ raw_std_devs,
182
+ low_rank_templates,
183
+ snrs_by_channel,
184
+ spike_counts_by_channel,
185
+ ) = res
186
+
187
+ if raw_only:
188
+ return dict(
189
+ sorting=sorting,
190
+ unit_ids=unit_ids,
191
+ spike_counts=spike_counts,
192
+ templates=raw_templates,
193
+ raw_templates=raw_templates,
194
+ raw_std_devs=raw_std_devs,
195
+ snrs_by_channel=snrs_by_channel,
196
+ spike_counts_by_channel=spike_counts_by_channel,
197
+ denoising_tsvd=denoising_tsvd,
198
+ **geom_kw,
199
+ )
200
+
201
+ weights = denoising_weights(
202
+ snrs_by_channel,
203
+ spike_length_samples=spike_length_samples,
204
+ trough_offset=trough_offset_samples,
205
+ snr_threshold=denoising_snr_threshold,
206
+ )
207
+ templates = weights * raw_templates + (1 - weights) * low_rank_templates
208
+ templates = templates.astype(dtype)
209
+
210
+ return dict(
211
+ sorting=sorting,
212
+ unit_ids=unit_ids,
213
+ spike_counts=spike_counts,
214
+ templates=templates,
215
+ raw_std_devs=raw_std_devs,
216
+ raw_templates=raw_templates,
217
+ low_rank_templates=low_rank_templates,
218
+ snrs_by_channel=snrs_by_channel,
219
+ spike_counts_by_channel=spike_counts_by_channel,
220
+ denoising_tsvd=denoising_tsvd,
221
+ weights=weights,
222
+ **geom_kw,
223
+ )
224
+
225
+
226
+ # -- helpers
227
+
228
+
229
+ def fit_tsvd(
230
+ recording,
231
+ sorting,
232
+ denoising_rank=5,
233
+ denoising_fit_radius=75.0,
234
+ denoising_spikes_fit=25_000,
235
+ trough_offset_samples=42,
236
+ spike_length_samples=121,
237
+ recompute_tsvd=False,
238
+ dtype=np.float32,
239
+ random_seed=0,
240
+ n_iter=15,
241
+ ) -> PCA | TruncatedSVD:
242
+ tsvd = None
243
+ if not recompute_tsvd:
244
+ tsvd = load_stored_tsvd(sorting)
245
+ assert isinstance(tsvd, (TruncatedSVD, PCA))
246
+ if tsvd is not None:
247
+ return tsvd
248
+
249
+ # read spikes on channel neighborhood
250
+ geom = recording.get_channel_locations()
251
+ tsvd_channel_index = make_channel_index(geom, denoising_fit_radius)
252
+
253
+ # subset spikes used to fit tsvd
254
+ rg = np.random.default_rng(random_seed)
255
+ max_time = recording.get_num_samples() - (
256
+ spike_length_samples - trough_offset_samples
257
+ )
258
+ t_clip = sorting.times_samples.clip(trough_offset_samples, max_time)
259
+ valid = np.logical_and(
260
+ sorting.labels >= 0,
261
+ sorting.times_samples == t_clip,
262
+ )
263
+ choices = np.flatnonzero(valid)
264
+ if choices.size > denoising_spikes_fit:
265
+ choices = rg.choice(choices, denoising_spikes_fit, replace=False)
266
+ choices.sort()
267
+ times = sorting.times_samples[choices]
268
+ channels = sorting.channels[choices]
269
+
270
+ # grab waveforms
271
+ waveforms = spikeio.read_waveforms_channel_index(
272
+ recording,
273
+ times,
274
+ tsvd_channel_index,
275
+ channels,
276
+ trough_offset_samples=trough_offset_samples,
277
+ spike_length_samples=spike_length_samples,
278
+ fill_value=0.0, # all-0 rows don't change SVD basis
279
+ )
280
+ waveforms = waveforms.astype(dtype)
281
+ waveforms = waveforms.transpose(0, 2, 1)
282
+ waveforms = waveforms.reshape(len(times) * tsvd_channel_index.shape[1], -1)
283
+
284
+ # reshape, fit tsvd, and done
285
+ tsvd = TruncatedSVD(
286
+ n_components=denoising_rank, random_state=random_seed, n_iter=n_iter
287
+ )
288
+ tsvd.fit(waveforms)
289
+
290
+ return tsvd
291
+
292
+
293
+ def denoising_weights(
294
+ snrs,
295
+ spike_length_samples,
296
+ trough_offset,
297
+ snr_threshold,
298
+ a=12.0,
299
+ b=12.0,
300
+ d=6.0,
301
+ edge_behavior="saturate",
302
+ ):
303
+ """Weights are applied to raw template, 1-weights to low rank"""
304
+ # v shaped function for time weighting
305
+ vt = np.abs(np.arange(spike_length_samples) - trough_offset, dtype=float)
306
+ if trough_offset < spike_length_samples:
307
+ vt[trough_offset:] /= vt[trough_offset:].max()
308
+ if trough_offset > 0:
309
+ vt[:trough_offset] /= vt[:trough_offset].max()
310
+
311
+ # snr weighting per channel
312
+ if edge_behavior == "saturate":
313
+ snc = np.minimum(snrs, snr_threshold) / snr_threshold
314
+ elif edge_behavior == "inf":
315
+ snc = np.minimum(snrs, snr_threshold) / snr_threshold
316
+ snc[snc >= 1.0] = np.inf
317
+ elif edge_behavior == "raw":
318
+ snc = snrs
319
+ else:
320
+ assert False
321
+
322
+ # pass it through a hand picked squashing function
323
+ wntc = 1.0 / (1.0 + np.exp(d + a * vt[None, :, None] - b * snc[:, None, :]))
324
+
325
+ return wntc
326
+
327
+
328
+ # -- main routine which does all the spike loading and computation
329
+
330
+
331
+ def get_all_shifted_raw_and_low_rank_templates(
332
+ recording,
333
+ sorting,
334
+ registered_geom=None,
335
+ denoising_tsvd=None,
336
+ pitch_shifts=None,
337
+ with_raw_std_dev=False,
338
+ spikes_per_unit=500,
339
+ reducer=fast_nanmedian,
340
+ n_jobs=0,
341
+ units_per_job=8,
342
+ random_seed=0,
343
+ show_progress=True,
344
+ trough_offset_samples=42,
345
+ spike_length_samples=121,
346
+ min_fraction_at_shift=0.1,
347
+ min_count_at_shift=5,
348
+ dtype=np.float32,
349
+ device=None,
350
+ ):
351
+ n_jobs, Executor, context, rank_queue = get_pool(n_jobs, with_rank_queue=True) # type: ignore
352
+ unit_ids, spike_counts = np.unique(sorting.labels, return_counts=True)
353
+ spike_counts = spike_counts[unit_ids >= 0]
354
+ unit_ids = unit_ids[unit_ids >= 0]
355
+ raw = denoising_tsvd is None
356
+ prefix = "Raw" if raw else "Denoised"
357
+
358
+ n_template_channels = recording.get_num_channels()
359
+ registered_kdtree = None
360
+ if registered_geom is not None:
361
+ n_template_channels = len(registered_geom)
362
+ registered_kdtree = KDTree(registered_geom)
363
+
364
+ n_units = unit_ids.size
365
+ raw_templates = np.zeros(
366
+ (n_units, spike_length_samples, n_template_channels),
367
+ dtype=dtype,
368
+ )
369
+ if with_raw_std_dev:
370
+ raw_square_templates = np.zeros(
371
+ (n_units, spike_length_samples, n_template_channels),
372
+ dtype=dtype,
373
+ )
374
+ low_rank_templates = None
375
+ if not raw:
376
+ low_rank_templates = np.zeros(
377
+ (n_units, spike_length_samples, n_template_channels),
378
+ dtype=dtype,
379
+ )
380
+ snrs_by_channel = np.zeros((n_units, n_template_channels), dtype=dtype)
381
+ spike_counts_by_channel = np.zeros((n_units, n_template_channels), dtype=dtype)
382
+
383
+ unit_id_chunks = [
384
+ unit_ids[i : i + units_per_job] for i in range(0, n_units, units_per_job)
385
+ ]
386
+
387
+ with Executor(
388
+ max_workers=n_jobs,
389
+ mp_context=context,
390
+ initializer=_template_process_init,
391
+ initargs=(
392
+ rank_queue,
393
+ random_seed,
394
+ recording,
395
+ sorting,
396
+ registered_kdtree,
397
+ denoising_tsvd,
398
+ pitch_shifts,
399
+ spikes_per_unit,
400
+ min_fraction_at_shift,
401
+ min_count_at_shift,
402
+ with_raw_std_dev,
403
+ reducer,
404
+ trough_offset_samples,
405
+ spike_length_samples,
406
+ device,
407
+ units_per_job,
408
+ dtype,
409
+ ),
410
+ ) as pool:
411
+ # launch the jobs and wrap in a progress bar
412
+ results = pool.map(_template_job, unit_id_chunks)
413
+ if show_progress:
414
+ pbar = tqdm(
415
+ smoothing=0.01,
416
+ desc=f"{prefix} templates",
417
+ total=unit_ids.size,
418
+ unit="template",
419
+ )
420
+ for res in results:
421
+ if res is None:
422
+ continue
423
+ (
424
+ units_chunk,
425
+ raw_temps_chunk,
426
+ raw_square_temps_chunk,
427
+ low_rank_temps_chunk,
428
+ snrs_chunk,
429
+ chancounts_chunk,
430
+ ) = res
431
+ ix_chunk = np.isin(unit_ids, units_chunk)
432
+ raw_templates[ix_chunk] = raw_temps_chunk
433
+ if with_raw_std_dev:
434
+ raw_square_templates[ix_chunk] = raw_square_temps_chunk # type: ignore
435
+ if not raw:
436
+ low_rank_templates[ix_chunk] = low_rank_temps_chunk # type: ignore
437
+ snrs_by_channel[ix_chunk] = snrs_chunk
438
+ spike_counts_by_channel[ix_chunk] = chancounts_chunk
439
+ if show_progress:
440
+ pbar.update(len(units_chunk)) # type: ignore
441
+ if show_progress:
442
+ pbar.close() # type: ignore
443
+
444
+ raw_std_dev = None
445
+ if with_raw_std_dev:
446
+ raw_std_dev = raw_square_templates # type: ignore
447
+ raw_std_dev -= raw_templates**2
448
+ np.abs(raw_std_dev, out=raw_std_dev)
449
+ raw_std_dev **= 0.5
450
+
451
+ return (
452
+ unit_ids,
453
+ spike_counts,
454
+ raw_templates,
455
+ raw_std_dev,
456
+ low_rank_templates,
457
+ snrs_by_channel,
458
+ spike_counts_by_channel,
459
+ )
460
+
461
+
462
+ class TemplateProcessContext:
463
+ def __init__(
464
+ self,
465
+ rg,
466
+ recording,
467
+ sorting,
468
+ registered_kdtree,
469
+ denoising_tsvd,
470
+ pitch_shifts,
471
+ spikes_per_unit,
472
+ min_fraction_at_shift,
473
+ min_count_at_shift,
474
+ with_raw_std_dev,
475
+ reducer,
476
+ trough_offset_samples,
477
+ spike_length_samples,
478
+ device,
479
+ units_per_job,
480
+ dtype,
481
+ ):
482
+ self.n_channels = recording.get_num_channels()
483
+ self.registered = registered_kdtree is not None
484
+
485
+ self.rg = rg
486
+ self.device = device
487
+ self.dtype = dtype
488
+ self.recording = recording
489
+ self.sorting = sorting
490
+ self.denoising_tsvd = denoising_tsvd
491
+ if denoising_tsvd is not None:
492
+ self.denoising_tsvd = TorchSVDProjector(
493
+ torch.from_numpy(denoising_tsvd.components_.astype(dtype))
494
+ )
495
+ self.denoising_tsvd.to(self.device)
496
+ self.spikes_per_unit = spikes_per_unit
497
+ self.reducer = reducer
498
+ self.trough_offset_samples = trough_offset_samples
499
+ self.spike_length_samples = spike_length_samples
500
+ self.max_spike_time = recording.get_num_samples() - (
501
+ spike_length_samples - trough_offset_samples
502
+ )
503
+ self.min_fraction_at_shift = min_fraction_at_shift
504
+ self.min_count_at_shift = min_count_at_shift
505
+ self.with_raw_std_dev = with_raw_std_dev
506
+
507
+ self.spike_buffer = torch.zeros(
508
+ (
509
+ spikes_per_unit * units_per_job,
510
+ spike_length_samples,
511
+ self.n_channels,
512
+ ),
513
+ device=device,
514
+ dtype=torch.from_numpy(np.zeros(1, dtype=dtype)).dtype,
515
+ )
516
+
517
+ self.n_template_channels = self.n_channels
518
+ if self.registered:
519
+ self.geom = recording.get_channel_locations()
520
+ self.match_distance = pdist(self.geom).min() / 2
521
+ self.registered_geom = registered_kdtree.data
522
+ self.registered_kdtree = registered_kdtree
523
+ self.pitch_shifts = pitch_shifts
524
+ self.n_template_channels = len(self.registered_geom)
525
+
526
+
527
+ _template_process_context = threading.local()
528
+
529
+
530
+ def _template_process_init(
531
+ rank_queue,
532
+ random_seed,
533
+ recording,
534
+ sorting,
535
+ registered_kdtree,
536
+ denoising_tsvd,
537
+ pitch_shifts,
538
+ spikes_per_unit,
539
+ min_fraction_at_shift,
540
+ min_count_at_shift,
541
+ with_raw_std_dev,
542
+ reducer,
543
+ trough_offset_samples,
544
+ spike_length_samples,
545
+ device,
546
+ units_per_job,
547
+ dtype,
548
+ ):
549
+ global _template_process_context
550
+
551
+ rank = rank_queue.get()
552
+ if device is None:
553
+ device = "cuda" if torch.cuda.is_available() else "cpu"
554
+ device = torch.device(device)
555
+ if device.type == "cuda" and device.index is None:
556
+ if torch.cuda.device_count() > 1:
557
+ device = torch.device("cuda", index=rank % torch.cuda.device_count())
558
+ torch.set_grad_enabled(False)
559
+
560
+ rg = np.random.default_rng(random_seed + rank)
561
+ _template_process_context.ctx = TemplateProcessContext(
562
+ rg,
563
+ recording,
564
+ sorting,
565
+ registered_kdtree,
566
+ denoising_tsvd,
567
+ pitch_shifts,
568
+ spikes_per_unit,
569
+ min_fraction_at_shift,
570
+ min_count_at_shift,
571
+ with_raw_std_dev,
572
+ reducer,
573
+ trough_offset_samples,
574
+ spike_length_samples,
575
+ device,
576
+ units_per_job,
577
+ dtype,
578
+ )
579
+
580
+
581
+ def _template_job(unit_ids):
582
+ p = _template_process_context.ctx
583
+
584
+ in_units_full = np.flatnonzero(np.isin(p.sorting.labels, unit_ids))
585
+ if not in_units_full.size:
586
+ return
587
+ labels_full = p.sorting.labels[in_units_full]
588
+
589
+ # only so many spikes per unit
590
+ uids, counts = np.unique(labels_full, return_counts=True)
591
+ n_spikes_grab = np.minimum(counts, p.spikes_per_unit).sum()
592
+ in_units = np.empty(n_spikes_grab, dtype=in_units_full.dtype)
593
+ labels = np.empty(n_spikes_grab, dtype=labels_full.dtype)
594
+ offset = 0
595
+ for u, c in zip(uids, counts):
596
+ if c > p.spikes_per_unit:
597
+ in_unit = p.rg.choice(
598
+ in_units_full[labels_full == u],
599
+ p.spikes_per_unit,
600
+ replace=False,
601
+ )
602
+ in_units[offset : offset + min(c, p.spikes_per_unit)] = in_unit
603
+ labels[offset : offset + min(c, p.spikes_per_unit)] = u
604
+ else:
605
+ in_units[offset : offset + c] = in_units_full[labels_full == u]
606
+ labels[offset : offset + c] = u
607
+ offset += min(c, p.spikes_per_unit)
608
+ order = np.argsort(in_units)
609
+ in_units = in_units[order]
610
+ labels = labels[order]
611
+
612
+ # read waveforms for all units
613
+ times = p.sorting.times_samples[in_units]
614
+ valid = np.flatnonzero(
615
+ (times >= p.trough_offset_samples) & (times <= p.max_spike_time)
616
+ )
617
+ if not valid.size:
618
+ return
619
+ in_units = in_units[valid]
620
+ labels = labels[valid]
621
+ times = times[valid]
622
+ waveforms = spikeio.read_full_waveforms(
623
+ p.recording,
624
+ times,
625
+ trough_offset_samples=p.trough_offset_samples,
626
+ spike_length_samples=p.spike_length_samples,
627
+ )
628
+ p.spike_buffer[: times.size] = torch.from_numpy(waveforms.astype(p.dtype))
629
+ waveforms = p.spike_buffer[: times.size]
630
+ n, t, c = waveforms.shape
631
+
632
+ # compute raw templates and spike counts per channel
633
+ raw_templates = []
634
+ raw_square_templates = []
635
+ counts = []
636
+ units_chunk = []
637
+ for u in uids:
638
+ in_unit = np.flatnonzero(labels == u)
639
+ if not in_unit.size:
640
+ continue
641
+ units_chunk.append(u)
642
+ in_unit_orig = in_units[labels == u]
643
+ unit_waveforms = waveforms[in_unit]
644
+ if p.registered:
645
+ raw_templates.append(
646
+ registered_template(
647
+ unit_waveforms,
648
+ p.pitch_shifts[in_unit_orig],
649
+ p.geom,
650
+ p.registered_geom,
651
+ min_fraction_at_shift=p.min_fraction_at_shift,
652
+ min_count_at_shift=p.min_count_at_shift,
653
+ registered_kdtree=p.registered_kdtree,
654
+ match_distance=p.match_distance,
655
+ reducer=p.reducer,
656
+ )
657
+ )
658
+ if p.with_raw_std_dev:
659
+ raw_square_templates.append(
660
+ registered_template(
661
+ unit_waveforms.square_(),
662
+ p.pitch_shifts[in_unit_orig],
663
+ p.geom,
664
+ p.registered_geom,
665
+ min_fraction_at_shift=p.min_fraction_at_shift,
666
+ min_count_at_shift=p.min_count_at_shift,
667
+ registered_kdtree=p.registered_kdtree,
668
+ match_distance=p.match_distance,
669
+ reducer=p.reducer,
670
+ )
671
+ )
672
+ counts.append(
673
+ registered_template(
674
+ np.ones((in_unit.size, p.n_channels)),
675
+ p.pitch_shifts[in_unit_orig],
676
+ p.geom,
677
+ p.registered_geom,
678
+ min_fraction_at_shift=p.min_fraction_at_shift,
679
+ min_count_at_shift=p.min_count_at_shift,
680
+ registered_kdtree=p.registered_kdtree,
681
+ match_distance=p.match_distance,
682
+ reducer=np.nansum,
683
+ )
684
+ )
685
+ else:
686
+ raw_templates.append(
687
+ p.reducer(waveforms[in_unit], axis=0).numpy(force=True)
688
+ )
689
+ if p.with_raw_std_dev:
690
+ raw_square_templates.append(
691
+ p.reducer(unit_waveforms.square_(), axis=0).numpy(force=True)
692
+ )
693
+ counts.append(in_unit.size)
694
+ snrs_by_chan = np.array(
695
+ [ptp(rt, 0) * np.sqrt(c) for rt, c in zip(raw_templates, counts)]
696
+ )
697
+ counts_by_chan = np.array(counts)
698
+ if counts_by_chan.ndim == 1:
699
+ counts_by_chan = np.broadcast_to(counts_by_chan[:, None], snrs_by_chan.shape)
700
+ raw_templates = np.array(raw_templates)
701
+ if p.with_raw_std_dev:
702
+ raw_square_templates = np.array(raw_square_templates)
703
+ else:
704
+ raw_square_templates = None
705
+
706
+ if p.denoising_tsvd is None:
707
+ return (
708
+ units_chunk,
709
+ raw_templates,
710
+ raw_square_templates,
711
+ None,
712
+ snrs_by_chan,
713
+ counts_by_chan,
714
+ )
715
+
716
+ # nt, t, ct = raw_templates.shape
717
+ # low_rank_templates = torch.tensor(raw_templates.transpose(0, 2, 1), device=p.device)
718
+ # low_rank_templates = low_rank_templates.reshape(nt * ct, t)
719
+ # low_rank_templates = p.denoising_tsvd(low_rank_templates, in_place=True)
720
+ # low_rank_templates = low_rank_templates.view(nt, ct, t).permute(0, 2, 1)
721
+ # low_rank_templates = low_rank_templates.numpy(force=True)
722
+
723
+ # apply denoising
724
+ waveforms = waveforms.permute(0, 2, 1).reshape(n * c, t)
725
+ waveforms = p.denoising_tsvd(waveforms, in_place=True)
726
+ waveforms = waveforms.reshape(n, c, t).permute(0, 2, 1)
727
+
728
+ # get low rank templates
729
+ low_rank_templates = []
730
+ for u in units_chunk:
731
+ in_unit = np.flatnonzero(labels == u)
732
+ in_unit_orig = in_units[labels == u]
733
+ if p.registered:
734
+ low_rank_templates.append(
735
+ registered_template(
736
+ waveforms[in_unit],
737
+ p.pitch_shifts[in_unit_orig],
738
+ p.geom,
739
+ p.registered_geom,
740
+ min_fraction_at_shift=p.min_fraction_at_shift,
741
+ min_count_at_shift=p.min_count_at_shift,
742
+ registered_kdtree=p.registered_kdtree,
743
+ match_distance=p.match_distance,
744
+ reducer=p.reducer,
745
+ )
746
+ )
747
+ else:
748
+ low_rank_templates.append(
749
+ p.reducer(waveforms[in_unit], axis=0).numpy(force=True)
750
+ )
751
+ low_rank_templates = np.array(low_rank_templates)
752
+
753
+ return (
754
+ units_chunk,
755
+ raw_templates,
756
+ raw_square_templates,
757
+ low_rank_templates,
758
+ snrs_by_chan,
759
+ counts_by_chan,
760
+ )
761
+
762
+
763
+ class TorchSVDProjector(torch.nn.Module):
764
+ def __init__(self, components):
765
+ super().__init__()
766
+ self.register_buffer("components", components)
767
+
768
+ def forward(self, x, in_place=False):
769
+ embed = x @ self.components.T
770
+ out = x if in_place else None
771
+ return torch.matmul(embed, cast(torch.Tensor, self.components), out=out)