DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,466 @@
1
+ import gc
2
+ from dataclasses import replace
3
+ import math
4
+
5
+ import numpy as np
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from spikeinterface.core import BaseRecording
9
+ from tqdm.auto import trange
10
+
11
+ from ..util.data_util import DARTsortSorting
12
+ from ..util.internal_config import (
13
+ ComputationConfig,
14
+ RealignStrategy,
15
+ TemplateRealignmentConfig,
16
+ WaveformConfig,
17
+ default_waveform_cfg,
18
+ )
19
+ from ..util.job_util import ensure_computation_config
20
+ from ..util.logging_util import get_logger
21
+ from ..util.spiketorch import ptp
22
+ from .templates import TemplateData
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ def realign(
28
+ *,
29
+ recording: BaseRecording,
30
+ sorting: DARTsortSorting,
31
+ realign_cfg: TemplateRealignmentConfig | None = None,
32
+ waveform_cfg: WaveformConfig = default_waveform_cfg,
33
+ computation_cfg: ComputationConfig | None = None,
34
+ motion_est=None,
35
+ ) -> tuple[DARTsortSorting, TemplateData | None]:
36
+ if realign_cfg is None:
37
+ return sorting, None
38
+ if not realign_cfg.realign_peaks:
39
+ return sorting, None
40
+ if not realign_cfg.realign_shift_ms:
41
+ return sorting, None
42
+
43
+ realignment_waveform_cfg = WaveformConfig(
44
+ ms_before=waveform_cfg.ms_before + realign_cfg.realign_shift_ms,
45
+ ms_after=waveform_cfg.ms_after + realign_cfg.realign_shift_ms,
46
+ )
47
+ templates = TemplateData.from_config(
48
+ recording=recording,
49
+ sorting=sorting,
50
+ template_cfg=realign_cfg.template_cfg,
51
+ waveform_cfg=realignment_waveform_cfg,
52
+ computation_cfg=computation_cfg,
53
+ motion_est=motion_est,
54
+ )
55
+ trough_offset_samples = waveform_cfg.trough_offset_samples(
56
+ sampling_frequency=recording.sampling_frequency
57
+ )
58
+ spike_length_samples = waveform_cfg.spike_length_samples(
59
+ sampling_frequency=recording.sampling_frequency
60
+ )
61
+
62
+ template_shifts, aligned_templates = realign_templates(
63
+ templates=templates.templates,
64
+ snrs_by_channel=templates.snrs_by_channel(),
65
+ unit_ids=templates.unit_ids,
66
+ padded_trough_offset_samples=realignment_waveform_cfg.trough_offset_samples(
67
+ sampling_frequency=recording.sampling_frequency
68
+ ),
69
+ trough_offset_samples=trough_offset_samples,
70
+ realign_strategy=realign_cfg.realign_strategy,
71
+ trough_factor=realign_cfg.trough_factor,
72
+ min_pair_corr=realign_cfg.min_pair_corr,
73
+ computation_cfg=computation_cfg,
74
+ )
75
+ templates = replace(
76
+ templates,
77
+ templates=aligned_templates,
78
+ trough_offset_samples=trough_offset_samples,
79
+ )
80
+ aligned_sorting = apply_time_shifts(
81
+ sorting,
82
+ template_shifts=template_shifts,
83
+ trough_offset_samples=trough_offset_samples,
84
+ spike_length_samples=spike_length_samples,
85
+ recording_length_samples=recording.get_num_samples(),
86
+ )
87
+ return aligned_sorting, templates
88
+
89
+
90
+ @torch.jit.script
91
+ def singlechan_alignments(
92
+ traces: torch.Tensor, trough_factor: float = 3.0, dim: int = 1
93
+ ) -> torch.Tensor:
94
+ aligner_traces = torch.where(traces < 0, trough_factor * traces, -traces)
95
+ offsets = aligner_traces.argmin(dim=dim)
96
+ return offsets
97
+
98
+
99
+ def get_main_channels_and_alignments(
100
+ template_data=None, trough_factor=3.0, templates=None, main_channels=None
101
+ ):
102
+ if templates is None:
103
+ assert template_data is not None
104
+ templates = template_data.templates
105
+ if main_channels is None:
106
+ main_channels = np.ptp(templates, axis=1).argmax(1)
107
+ mc_traces = np.take_along_axis(templates, main_channels[:, None, None], axis=2)[
108
+ :, :, 0
109
+ ]
110
+ aligner_traces = np.where(mc_traces < 0, trough_factor * mc_traces, -mc_traces)
111
+ aligner_traces = np.abs(aligner_traces, out=aligner_traces)
112
+ # break ties late
113
+ tmax = aligner_traces.shape[1] - 1
114
+ offsets = tmax - aligner_traces[:, ::-1].argmax(1)
115
+ return main_channels, mc_traces, offsets
116
+
117
+
118
+ def estimate_offset(
119
+ *,
120
+ templates,
121
+ computation_cfg: ComputationConfig,
122
+ snrs_by_channel=None,
123
+ strategy="mainchan_trough_factor",
124
+ trough_factor=3.0,
125
+ min_weight=0.75,
126
+ min_pair_corr=0.75,
127
+ main_channels=None,
128
+ padded_trough_offset_samples=42,
129
+ trough_offset_samples=42,
130
+ ) -> torch.Tensor:
131
+ templates = torch.asarray(templates)
132
+
133
+ if strategy == "mainchan_trough_factor":
134
+ _, _, offsets = get_main_channels_and_alignments(
135
+ None,
136
+ trough_factor=trough_factor,
137
+ templates=templates.numpy(force=True),
138
+ main_channels=main_channels,
139
+ )
140
+ return offsets - padded_trough_offset_samples
141
+
142
+ if strategy == "dredge":
143
+ offsets = dredge_realign(
144
+ templates=templates,
145
+ main_channels=main_channels,
146
+ snrs_by_channel=snrs_by_channel,
147
+ trough_factor=trough_factor,
148
+ padded_trough_offset_samples=padded_trough_offset_samples,
149
+ trough_offset_samples=trough_offset_samples,
150
+ min_corr=min_pair_corr,
151
+ computation_cfg=computation_cfg,
152
+ )
153
+ gc.collect()
154
+ torch.cuda.empty_cache()
155
+ return offsets
156
+
157
+ if strategy == "snr_weighted_trough_factor":
158
+ assert snrs_by_channel is not None
159
+ weights = torch.asarray(snrs_by_channel).clone()
160
+ weights /= weights.amax(dim=1, keepdim=True)
161
+ weights[weights < min_weight] = 0.0
162
+ weights /= weights.sum(dim=1, keepdim=True)
163
+ tmp = templates.clone()
164
+ tmp[tmp < 0] *= trough_factor
165
+ tmp.abs_()
166
+ offsets = tmp.argmax(dim=1).double()
167
+ offsets = torch.sum(offsets * weights, dim=1)
168
+ offsets = torch.round(offsets).long()
169
+ return offsets - padded_trough_offset_samples
170
+
171
+ if strategy == "normsq_weighted_trough_factor":
172
+ tmp = templates.square()
173
+ weights = tmp.sum(dim=1)
174
+ weights /= weights.amax(dim=1, keepdim=True)
175
+ weights[weights < min_weight] = 0.0
176
+ weights /= weights.sum(dim=1, keepdim=True)
177
+ tmp[:] = templates
178
+ tmp[tmp < 0] *= trough_factor
179
+ tmp.abs_()
180
+ offsets = tmp.argmax(dim=1).double()
181
+ offsets = torch.sum(offsets * weights, dim=1)
182
+ offsets = torch.round(offsets).long()
183
+ return offsets - padded_trough_offset_samples
184
+
185
+ if strategy == "ampsq_weighted_trough_factor":
186
+ weights = ptp(templates).square()
187
+ weights /= weights.amax(dim=1, keepdim=True)
188
+ weights[weights < min_weight] = 0.0
189
+ weights /= weights.sum(dim=1, keepdim=True)
190
+ tmp = templates.clone()
191
+ tmp[tmp < 0] *= trough_factor
192
+ tmp.abs_()
193
+ offsets = tmp.argmax(dim=1).double()
194
+ offsets = torch.sum(offsets * weights, dim=1)
195
+ offsets = torch.round(offsets).long()
196
+ return offsets - padded_trough_offset_samples
197
+
198
+ assert False
199
+
200
+
201
+ def realign_templates(
202
+ *,
203
+ templates,
204
+ snrs_by_channel=None,
205
+ unit_ids=None,
206
+ main_channels=None,
207
+ padded_trough_offset_samples=42,
208
+ trough_offset_samples=42,
209
+ realign_strategy: RealignStrategy = "mainchan_trough_factor",
210
+ min_pair_corr=0.75,
211
+ trough_factor=3.0,
212
+ computation_cfg: ComputationConfig | None = None,
213
+ ):
214
+ computation_cfg = ensure_computation_config(computation_cfg)
215
+ if main_channels is None:
216
+ if realign_strategy in ("mainchan_trough_factor", "dredge"):
217
+ if snrs_by_channel is not None:
218
+ main_channels = snrs_by_channel.argmax(1)
219
+ else:
220
+ main_channels = np.ptp(templates, axis=1).argmax(1)
221
+ elif realign_strategy.startswith("snr_weighted"):
222
+ assert snrs_by_channel is not None
223
+ main_channels = snrs_by_channel.argmax(1)
224
+ elif realign_strategy.startswith("normsq_weighted"):
225
+ main_channels = np.square(templates).sum(1).argmax(1)
226
+ elif realign_strategy.startswith("ampsq_weighted"):
227
+ main_channels = np.ptp(templates).max(1).argmax(1)
228
+ else:
229
+ assert False
230
+ assert main_channels is not None
231
+
232
+ # find template peak time
233
+ max_shift = padded_trough_offset_samples - trough_offset_samples
234
+ template_shifts__ = estimate_offset(
235
+ computation_cfg=computation_cfg,
236
+ templates=templates,
237
+ snrs_by_channel=snrs_by_channel,
238
+ strategy=realign_strategy,
239
+ trough_factor=trough_factor,
240
+ main_channels=main_channels,
241
+ min_pair_corr=min_pair_corr,
242
+ padded_trough_offset_samples=padded_trough_offset_samples,
243
+ trough_offset_samples=trough_offset_samples,
244
+ )
245
+ if torch.is_tensor(template_shifts__):
246
+ template_shifts_ = template_shifts__.numpy(force=True)
247
+ else:
248
+ template_shifts_ = template_shifts__
249
+
250
+ # clip if needed
251
+ template_shifts_[np.abs(template_shifts_) > max_shift] = 0
252
+
253
+ # find unit sample time shifts
254
+ if unit_ids is None:
255
+ template_shifts = template_shifts_
256
+ else:
257
+ template_shifts = np.zeros(unit_ids.max() + 1, dtype=np.int64)
258
+ template_shifts[unit_ids] = template_shifts_
259
+
260
+ aligned_templates = trim_templates_to_shift(
261
+ templates=templates,
262
+ max_shift=max_shift,
263
+ template_shifts=template_shifts,
264
+ unit_ids=unit_ids,
265
+ )
266
+
267
+ return template_shifts, aligned_templates
268
+
269
+
270
+ def trim_templates_to_shift(
271
+ templates, max_shift=0, template_shifts=None, unit_ids=None
272
+ ):
273
+ if not max_shift or template_shifts is None:
274
+ return templates
275
+
276
+ n, t, c = templates.shape
277
+
278
+ # trim templates
279
+ aligned_spike_len = t - 2 * max_shift
280
+ aligned_templates = np.empty((n, aligned_spike_len, c), dtype=templates.dtype)
281
+ if unit_ids is None:
282
+ assert len(templates) == len(template_shifts) == n
283
+ else:
284
+ template_shifts = template_shifts[unit_ids]
285
+ for i, dt in enumerate(template_shifts):
286
+ aligned_templates[i] = templates[
287
+ i, max_shift + dt : max_shift + dt + aligned_spike_len
288
+ ]
289
+
290
+ return aligned_templates
291
+
292
+
293
+ def apply_time_shifts(
294
+ sorting,
295
+ template_shifts=None,
296
+ template_data=None,
297
+ trough_offset_samples=None,
298
+ spike_length_samples=None,
299
+ recording_length_samples=None,
300
+ ) -> DARTsortSorting:
301
+ if template_shifts is not None:
302
+ unit_ids = None
303
+ elif template_data is not None and template_data.properties:
304
+ if "template_time_shifts" in template_data.properties:
305
+ template_shifts = template_data.properties["template_time_shifts"]
306
+ elif "time_shifts" in template_data.properties:
307
+ template_shifts = template_data.properties["time_shifts"]
308
+ unit_ids = template_data.unit_ids
309
+ else:
310
+ unit_ids = None
311
+
312
+ if template_shifts is None:
313
+ return sorting
314
+
315
+ ixs = sorting.labels
316
+ valid = np.flatnonzero(ixs >= 0)
317
+ ixsv = ixs[valid]
318
+ if unit_ids is not None and not np.array_equal(unit_ids, np.arange(len(unit_ids))):
319
+ ixsv = np.searchsorted(unit_ids, ixsv)
320
+ assert np.array_equal(unit_ids[ixsv], sorting.labels[valid])
321
+
322
+ new_times = sorting.times_samples.copy()
323
+ new_times[valid] += template_shifts[ixsv]
324
+
325
+ if recording_length_samples is not None:
326
+ assert spike_length_samples is not None
327
+ assert trough_offset_samples is not None
328
+ labels = sorting.labels.copy()
329
+ tail_samples = spike_length_samples - trough_offset_samples
330
+ highlim = recording_length_samples - tail_samples
331
+ labels[new_times < trough_offset_samples] = -1
332
+ labels[new_times >= highlim] = -1
333
+ else:
334
+ labels = sorting.labels
335
+
336
+ return sorting.ephemeral_replace(labels=labels, times_samples=new_times)
337
+
338
+
339
+ def dredge_realign(
340
+ *,
341
+ templates: torch.Tensor,
342
+ main_channels: torch.Tensor | None,
343
+ snrs_by_channel: torch.Tensor | None,
344
+ min_spatial_cosine: float = 0.75,
345
+ min_corr: float = 0.75,
346
+ trough_factor: float = 3.0,
347
+ padded_trough_offset_samples: int,
348
+ trough_offset_samples: int,
349
+ computation_cfg: ComputationConfig,
350
+ dredge_window_fraction: float = 0.5,
351
+ eps=1e-3,
352
+ ) -> torch.Tensor:
353
+ from dredge.dredgelib import newton_solve_rigid
354
+
355
+ max_shift = padded_trough_offset_samples - trough_offset_samples
356
+
357
+ if main_channels is None and snrs_by_channel is not None:
358
+ main_channels = snrs_by_channel.argmax(1)
359
+
360
+ # start with trough alignment
361
+ main_channels, main_channel_traces, offsets = get_main_channels_and_alignments(
362
+ None,
363
+ trough_factor=trough_factor,
364
+ templates=templates.numpy(force=True),
365
+ main_channels=main_channels,
366
+ )
367
+ offsets = offsets - padded_trough_offset_samples
368
+ offsets[np.abs(offsets) > max_shift] = 0
369
+
370
+ # trim to alignment
371
+ templates_trim0 = trim_templates_to_shift(
372
+ templates=templates.numpy(force=True),
373
+ max_shift=max_shift,
374
+ template_shifts=offsets,
375
+ )
376
+
377
+ # which pairs to convolve?
378
+ sp = snrs_by_channel if snrs_by_channel is not None else ptp(templates_trim0)
379
+ sp = torch.asarray(sp)
380
+ sp = sp / torch.linalg.norm(sp, dim=1)[:, None]
381
+ spcos = sp @ sp.T
382
+ pair_mask = spcos >= min_spatial_cosine
383
+ ii, jj = pair_mask.nonzero(as_tuple=True)
384
+ triu = ii < jj
385
+ ii = ii[triu]
386
+ jj = jj[triu]
387
+ logger.dartsortdebug(
388
+ f"DREDge align: convolving {ii.numel()} pairs (mincos={min_spatial_cosine:.2f})."
389
+ )
390
+
391
+ # get best lags and correlations
392
+ templates_trim0 = torch.asarray(
393
+ templates_trim0, dtype=torch.float, device=computation_cfg.actual_device()
394
+ )
395
+ dredge_max_shift = math.floor(max_shift * dredge_window_fraction)
396
+ lags, corrs = _pairwise_correlate_templates(
397
+ templates=templates_trim0, ii=ii, jj=jj, max_shift=dredge_max_shift
398
+ )
399
+ lags = lags.numpy(force=True)
400
+ corrs = corrs.numpy(force=True)
401
+ logger.dartsortdebug(
402
+ f"DREDge align: {100 * (corrs >= min_corr).mean():.1f}% "
403
+ "of pairs were correlated enough."
404
+ )
405
+
406
+ # densify
407
+ k = len(templates)
408
+ D = np.zeros((k, k))
409
+ C = D.copy()
410
+ D[ii, jj] = lags
411
+ D[jj, ii] = -lags
412
+ C[ii, jj] = corrs
413
+ C[jj, ii] = corrs
414
+ np.fill_diagonal(C, 1.0)
415
+
416
+ # corrs->weights
417
+ C[C < min_corr] = 0.0
418
+
419
+ # call out to dredge
420
+ Sigma0inv = np.eye(C.shape[0]) * eps
421
+ p, *_ = newton_solve_rigid(D, C, Sigma0inv)
422
+ p = np.rint(p).astype(offsets.dtype)
423
+
424
+ # adjust offsets
425
+ offsets = torch.asarray(offsets) - torch.asarray(p)
426
+
427
+ return offsets
428
+
429
+
430
+ def _pairwise_correlate_templates(templates, ii, jj, max_shift: int, batch_size=256):
431
+ lags = torch.arange(-max_shift, max_shift + 1).to(templates.device)
432
+ k = len(templates)
433
+ npair = len(ii)
434
+
435
+ # compute norms...
436
+ normsa = templates.new_zeros((k, lags.shape[0]))
437
+ normsb = templates.new_zeros((k, lags.shape[0]))
438
+ for i0 in range(0, k, batch_size):
439
+ i1 = min(k, i0 + batch_size)
440
+
441
+ filt = torch.square(templates[i0:i1])[:, None]
442
+ ones = torch.ones_like(templates[i0:i1])[None, :]
443
+ conv = F.conv2d(ones, filt, padding=(max_shift, 0), groups=i1 - i0)
444
+ assert conv.shape == (1, i1 - i0, lags.shape[0], 1)
445
+ conv = conv[0, :, :, 0].sqrt_()
446
+ normsa[i0:i1] = conv
447
+ normsb[i0:i1] = torch.flip(conv, dims=(1,))
448
+
449
+ best_lag = templates.new_zeros(ii.shape)
450
+ best_corr = templates.new_zeros(ii.shape)
451
+
452
+ for i0 in trange(0, npair, batch_size, desc="DREDge template alignment"):
453
+ i1 = min(npair, i0 + batch_size)
454
+
455
+ iib = ii[i0:i1]
456
+ jjb = jj[i0:i1]
457
+
458
+ filt = templates[iib][:, None]
459
+ inpt = templates[jjb][None, :]
460
+ conv = F.conv2d(inpt, filt, padding=(max_shift, 0), groups=i1 - i0)
461
+ assert conv.shape == (1, i1 - i0, lags.shape[0], 1)
462
+ conv = conv[0, :, :, 0].div_(normsa[iib] * normsb[jjb])
463
+ best_corr[i0:i1], lag_ix = conv.max(dim=1)
464
+ best_lag[i0:i1] = lags[lag_ix]
465
+
466
+ return best_lag, best_corr
@@ -0,0 +1,204 @@
1
+ from dataclasses import replace
2
+ from logging import getLogger
3
+
4
+ import numpy as np
5
+ from dartsort.util import drift_util
6
+
7
+
8
+ logger = getLogger(__name__)
9
+
10
+
11
+ def superres_sorting(
12
+ sorting,
13
+ geom,
14
+ motion_est=None,
15
+ strategy="motion_estimate",
16
+ localizations_dataset_name="point_source_localizations",
17
+ superres_bin_size_um=10.0,
18
+ min_spikes_per_bin=5,
19
+ probe_margin_um=200.0,
20
+ ):
21
+ """Construct the spatially superresolved spike train
22
+
23
+ The superres spike train assigns new labels to each unit
24
+ to capture shape variation due to vertical drift. Each
25
+ unit is broken into several sub-units, and spikes are
26
+ assigned into sub-units (superres units) depending on
27
+ their position and estimated displacement.
28
+
29
+ This function just needs to compute superres bin labels;
30
+ the pitch shifting is taken care of during template
31
+ computation.
32
+
33
+ There is a strategy flag here to allow experiments
34
+ with how this behaves.
35
+
36
+ Strategies:
37
+ - motion_estimate : bin spikes purely based on the drift
38
+ estimate. if the motion estimate is entirely correct,
39
+ then that means we should bin by the displacement estimate
40
+ mod the pitch.
41
+ - drift_pitch_loc_bin : spikes are coarsely motion corrected
42
+ by integer multiples of the pitch using the motion estimate,
43
+ and then these coarsely corrected positions are used to bin
44
+ for superres
45
+
46
+
47
+ Returns
48
+ -------
49
+ dict with keys : values
50
+ group_ids : np.array
51
+ Int array such that group_ids[superres_id] is the original id of
52
+ superres unit superres_id
53
+ sorting : DARTsortSorting
54
+ properties : dict of np.array
55
+ Each array in here is of shape (n_superres_units,) and contains
56
+ bookkeeping information about the superres process.
57
+ For instance, for motion_estimate binning, we store the centers
58
+ of the bins (which are modulo pitch).
59
+ """
60
+ pitch = drift_util.get_pitch(geom)
61
+ full_labels = sorting.labels.copy()
62
+
63
+ # load spike depths
64
+ # TODO: relying on this index feels wrong
65
+ spike_times_s = sorting.times_seconds
66
+ spike_depths_um = getattr(sorting, localizations_dataset_name)[:, 2]
67
+
68
+ # remove spikes far away from the probe
69
+ if probe_margin_um is not None:
70
+ valid = spike_depths_um == np.clip(
71
+ spike_depths_um,
72
+ geom[:, 1].min() - probe_margin_um,
73
+ geom[:, 1].max() + probe_margin_um,
74
+ )
75
+ full_labels[~valid] = -1
76
+
77
+ # handle triaging
78
+ kept = np.flatnonzero(full_labels >= 0)
79
+ labels = full_labels[kept]
80
+ spike_times_s = spike_times_s[kept]
81
+ spike_depths_um = spike_depths_um[kept]
82
+
83
+ # make superres spike train
84
+ properties = {}
85
+ if strategy in ("none", None):
86
+ superres_to_original = np.arange(labels.max() + 1)
87
+ superres_sorting = sorting
88
+ elif strategy == "motion_estimate":
89
+ superres_labels, superres_to_original, bin_centers = motion_estimate_strategy(
90
+ labels,
91
+ spike_times_s,
92
+ spike_depths_um,
93
+ pitch,
94
+ motion_est,
95
+ superres_bin_size_um=superres_bin_size_um,
96
+ )
97
+ properties["motion_estimate_bin_centers"] = bin_centers
98
+ elif strategy == "drift_pitch_loc_bin":
99
+ superres_labels, superres_to_original = drift_pitch_loc_bin_strategy(
100
+ labels,
101
+ spike_times_s,
102
+ spike_depths_um,
103
+ pitch,
104
+ motion_est,
105
+ superres_bin_size_um=superres_bin_size_um,
106
+ )
107
+ else:
108
+ raise ValueError(f"Unknown superres {strategy=}")
109
+
110
+ # handle too-small units
111
+ superres_labels, superres_to_original = remove_small_superres_units(
112
+ superres_labels, superres_to_original, min_spikes_per_bin=min_spikes_per_bin # type: ignore
113
+ )
114
+
115
+ # back to un-triaged label space
116
+ full_labels[kept] = superres_labels
117
+ superres_sorting = sorting.ephemeral_replace(labels=full_labels)
118
+ return dict(
119
+ group_ids=superres_to_original,
120
+ sorting=superres_sorting,
121
+ properties=properties,
122
+ )
123
+
124
+
125
+ def motion_estimate_strategy(
126
+ original_labels,
127
+ spike_times_s,
128
+ spike_depths_um,
129
+ pitch,
130
+ motion_est,
131
+ superres_bin_size_um=10.0,
132
+ bin_round_atol=0.1,
133
+ ):
134
+ """ """
135
+ # reg_pos = pos - disp, pos = reg_pos + disp
136
+ # so, disp is the motion of spikes relative to fixed probe
137
+ if motion_est is None:
138
+ displacements = np.zeros_like(spike_depths_um)
139
+ else:
140
+ displacements = motion_est.disp_at_s(spike_times_s, spike_depths_um)
141
+
142
+ n_bins = pitch // superres_bin_size_um
143
+ remainder = pitch - n_bins * superres_bin_size_um
144
+ if not np.isclose(remainder, 0.0):
145
+ n_bins = n_bins + (remainder > bin_round_atol)
146
+ logger.info(
147
+ f"Superres bin size didn't divide the pitch. Rounding it down from "
148
+ f"{superres_bin_size_um} to {pitch / n_bins}, for {n_bins} bins."
149
+ )
150
+ superres_bin_size_um = pitch / n_bins
151
+
152
+ mod_positions = displacements % pitch
153
+ bin_centers = np.arange(n_bins) * superres_bin_size_um + superres_bin_size_um / 2
154
+ bin_ids = mod_positions // superres_bin_size_um
155
+ bin_ids = bin_ids.astype(original_labels.dtype)
156
+ orig_label_and_bin, superres_labels = np.unique(
157
+ np.c_[original_labels, bin_ids], axis=0, return_inverse=True
158
+ )
159
+ superres_to_original = orig_label_and_bin[:, 0]
160
+ bin_centers = bin_centers[orig_label_and_bin[:, 1]]
161
+ return superres_labels, superres_to_original, bin_centers
162
+
163
+
164
+ def drift_pitch_loc_bin_strategy(
165
+ original_labels,
166
+ spike_times_s,
167
+ spike_depths_um,
168
+ pitch,
169
+ motion_est,
170
+ superres_bin_size_um=10.0,
171
+ ):
172
+ n_pitches_shift = drift_util.get_spike_pitch_shifts(
173
+ spike_depths_um, pitch=pitch, times_s=spike_times_s, motion_est=motion_est
174
+ )
175
+ coarse_reg_depths = spike_depths_um + n_pitches_shift * pitch
176
+
177
+ bin_ids = coarse_reg_depths // superres_bin_size_um
178
+ bin_ids = bin_ids.astype(int)
179
+ orig_label_and_bin, superres_labels = np.unique(
180
+ np.c_[original_labels, bin_ids], axis=0, return_inverse=True
181
+ )
182
+ superres_to_original = orig_label_and_bin[:, 0]
183
+ return superres_labels, superres_to_original
184
+
185
+
186
+ def remove_small_superres_units(
187
+ superres_labels, superres_to_original, min_spikes_per_bin
188
+ ):
189
+ if not min_spikes_per_bin:
190
+ return superres_labels, superres_to_original
191
+
192
+ slabels, scounts = np.unique(superres_labels, return_counts=True)
193
+
194
+ # new labels
195
+ kept = scounts >= min_spikes_per_bin
196
+ kept_labels = slabels[kept]
197
+ relabeling = np.full_like(slabels, -1)
198
+ relabeling[kept] = np.arange(kept_labels.size)
199
+
200
+ # relabel
201
+ superres_labels = relabeling[superres_labels]
202
+ superres_to_original = superres_to_original[kept_labels]
203
+
204
+ return superres_labels, superres_to_original