DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,842 @@
1
+ from tempfile import tempdir
2
+ from typing import Literal, Union
3
+
4
+ import numpy as np
5
+ import torch
6
+ import torch.nn.functional as F
7
+ from scipy.interpolate import griddata
8
+ from scipy.spatial import KDTree
9
+ from scipy.spatial.distance import cdist, pdist
10
+ from scipy.sparse.csgraph import connected_components
11
+ from scipy.sparse import coo_array
12
+
13
+ from ..templates.template_util import svd_compress_templates
14
+ from ..templates.templates import TemplateData
15
+ from ..templates.realignment import get_main_channels_and_alignments
16
+ from ..util.interpolation_util import (
17
+ InterpolationParams,
18
+ default_interpolation_params,
19
+ interp_precompute,
20
+ kernel_interpolate,
21
+ )
22
+ from ..util.waveform_util import (
23
+ make_channel_index,
24
+ upsample_multichan,
25
+ upsample_singlechan,
26
+ )
27
+
28
+
29
+ def get_template_simulator(
30
+ n_units,
31
+ templates_kind="3exp",
32
+ template_library=None,
33
+ geom=None,
34
+ sampling_frequency=30_000.0,
35
+ random_seed=0,
36
+ common_reference=False,
37
+ temporal_jitter=1,
38
+ **template_simulator_kwargs,
39
+ ) -> "TemplateSimulator":
40
+ if templates_kind == "3exp":
41
+ return PointSource3ExpSimulator(
42
+ geom=geom,
43
+ n_units=n_units,
44
+ sampling_frequency=sampling_frequency,
45
+ seed=random_seed,
46
+ common_reference=common_reference,
47
+ temporal_jitter=temporal_jitter,
48
+ **template_simulator_kwargs,
49
+ )
50
+ if templates_kind == "library":
51
+ assert template_library is not None
52
+ return TemplateLibrarySimulator.from_template_library(
53
+ geom=geom,
54
+ n_units=n_units,
55
+ templates=template_library,
56
+ seed=random_seed,
57
+ common_reference=common_reference,
58
+ temporal_jitter=temporal_jitter,
59
+ **template_simulator_kwargs,
60
+ )
61
+ if templates_kind == "static":
62
+ assert "template_data" in template_simulator_kwargs
63
+ template_data = template_simulator_kwargs.pop("template_data")
64
+ if geom is not None:
65
+ assert np.array_equal(geom, template_data.registered_geom)
66
+ assert not common_reference
67
+ assert n_units == len(template_data.unit_ids)
68
+ return StaticTemplateSimulator(
69
+ template_data=template_data,
70
+ temporal_jitter=temporal_jitter,
71
+ **template_simulator_kwargs,
72
+ )
73
+ assert False
74
+
75
+
76
+ def singlechan_to_probe(pos, alpha, waveforms, geom3, decay_model="squared"):
77
+ dtype = waveforms.dtype
78
+ if decay_model == "pointsource":
79
+ amp = alpha / cdist(pos, geom3).astype(dtype)
80
+ elif decay_model == "squared":
81
+ amp = alpha * (cdist(pos, geom3).astype(dtype) ** -2)
82
+ elif decay_model == "32":
83
+ amp = alpha * (cdist(pos, geom3).astype(dtype) ** -(3 / 2))
84
+ else:
85
+ assert False
86
+ n_dims_expand = waveforms.ndim - 1
87
+ templates = waveforms[..., :, None] * amp[..., *([None] * n_dims_expand), :]
88
+ return templates
89
+
90
+
91
+ # -- template sims
92
+
93
+
94
+ class BaseTemplateSimulator:
95
+ n_units: int
96
+ geom: np.ndarray
97
+
98
+ def trough_offset_samples(self) -> int:
99
+ raise NotImplementedError
100
+
101
+ def spike_length_samples(self) -> int:
102
+ raise NotImplementedError
103
+
104
+ def templates(
105
+ self,
106
+ drift: float = 0.0,
107
+ up: bool = False,
108
+ padded: bool = False,
109
+ pad_value: float = float("nan"),
110
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
111
+ """Returns (pos, templates, trough_offsets) which are (K, 3), (K, t, C), (K,) with optional up dim after K."""
112
+ raise NotImplementedError
113
+
114
+
115
+ class StaticTemplateSimulator(BaseTemplateSimulator):
116
+ """Super simple version."""
117
+
118
+ def __init__(self, template_data: TemplateData, temporal_jitter: int = 1):
119
+ assert template_data.registered_geom is not None
120
+ self.n_units = template_data.unit_ids.size
121
+ self.geom = template_data.registered_geom
122
+ self.template_data = template_data
123
+ self.temporal_jitter = temporal_jitter
124
+ self.templates_up = upsample_multichan(
125
+ self.template_data.templates, temporal_jitter=temporal_jitter
126
+ )
127
+ _, mct, a0 = get_main_channels_and_alignments(template_data)
128
+ self.offsets = a0 - template_data.trough_offset_samples
129
+ assert self.templates_up.shape[:2] == (self.n_units, temporal_jitter)
130
+ _, mct, a1 = get_main_channels_and_alignments(
131
+ templates=self.templates_up.reshape(
132
+ self.n_units * temporal_jitter, *self.templates_up.shape[2:]
133
+ )
134
+ )
135
+ a1 = a1 - template_data.trough_offset_samples
136
+ self.offsets_up = a1.reshape(self.n_units, temporal_jitter)
137
+
138
+ def trough_offset_samples(self) -> int:
139
+ return self.template_data.trough_offset_samples
140
+
141
+ def spike_length_samples(self) -> int:
142
+ return self.template_data.spike_length_samples
143
+
144
+ def templates(
145
+ self,
146
+ drift: float = 0.0,
147
+ up: bool = False,
148
+ padded: bool = False,
149
+ pad_value: float = float("nan"),
150
+ ):
151
+ assert not drift
152
+ loc = self.template_data.template_locations()
153
+ assert loc.shape[1] == 2
154
+ loc = np.c_[loc[:, 0], np.zeros_like(loc[:, 0]), loc[:, 1]]
155
+ temps = self.templates_up if up else self.template_data.templates
156
+ if padded:
157
+ zpads = [(0, 0)] * (temps.ndim - 1)
158
+ temps = np.pad(temps, [*zpads, (0, 1)], constant_values=pad_value)
159
+ off = self.offsets_up if up else self.offsets
160
+ return loc, temps, off
161
+
162
+
163
+ class PointSource3ExpSimulator(BaseTemplateSimulator):
164
+ def __init__(
165
+ self,
166
+ geom,
167
+ n_units,
168
+ common_reference=False,
169
+ temporal_jitter=1,
170
+ temporal_jitter_kind: Literal["exact", "cubic"] = "cubic",
171
+ min_rms_distance=0.0,
172
+ force_no_offset=True,
173
+ snr_adjustment=1.0,
174
+ # timing params
175
+ tip_before_min=0.1,
176
+ tip_before_max=0.5,
177
+ peak_after_min=0.2,
178
+ peak_after_max=0.8,
179
+ # width params
180
+ trough_width_min=0.005,
181
+ trough_width_max=0.025,
182
+ tip_width_min=0.01,
183
+ tip_width_max=0.05,
184
+ peak_width_min=0.05,
185
+ peak_width_max=0.2,
186
+ # rel height params
187
+ tip_rel_max=0.3,
188
+ peak_rel_max=0.5,
189
+ # pos/amplitude params
190
+ pos_margin_um_x=25.0,
191
+ pos_margin_um_z=25.0,
192
+ orthdist_min_um=25.0,
193
+ orthdist_max_um=50.0,
194
+ alpha_family="uniform",
195
+ alpha_min=5 * 25.0**2,
196
+ alpha_max=40 * 25.0**2,
197
+ alpha_mean=10.0 * np.square(25.0),
198
+ alpha_var=5.0 * np.square(25.0),
199
+ # config
200
+ ms_before=1.4,
201
+ ms_after=2.6,
202
+ sampling_frequency=30_000.0,
203
+ decay_model="squared",
204
+ seed: int | np.random.Generator = 0,
205
+ dtype=np.float32,
206
+ ):
207
+ self.rg = np.random.default_rng(seed)
208
+ self.dtype = dtype
209
+ self.n_units = n_units
210
+ self.temporal_jitter = temporal_jitter
211
+ self.common_reference = common_reference
212
+ self.snr_adjustment = snr_adjustment
213
+ self.decay_model = decay_model
214
+
215
+ self.geom = geom
216
+ self.geom3 = geom
217
+ if geom.shape[1] == 2:
218
+ self.geom3 = np.zeros((geom.shape[0], 3))
219
+ self.geom3[:, [0, 2]] = geom
220
+ self.geom3 = self.geom3.astype(self.dtype)
221
+ self.ms_before = ms_before
222
+ self.ms_after = ms_after
223
+ self.sampling_frequency = sampling_frequency
224
+
225
+ self.waveform_kw = dict(
226
+ snr_adjustment=snr_adjustment,
227
+ tip_before_min=tip_before_min,
228
+ tip_before_max=tip_before_max,
229
+ peak_after_min=peak_after_min,
230
+ peak_after_max=peak_after_max,
231
+ trough_width_min=trough_width_min,
232
+ trough_width_max=trough_width_max,
233
+ tip_width_min=tip_width_min,
234
+ tip_width_max=tip_width_max,
235
+ peak_width_min=peak_width_min,
236
+ peak_width_max=peak_width_max,
237
+ tip_rel_max=tip_rel_max,
238
+ peak_rel_max=peak_rel_max,
239
+ )
240
+ alpha_shape = alpha_var / alpha_mean
241
+ alpha_scale = alpha_mean / alpha_shape
242
+ self.location_kw = dict(
243
+ pos_margin_um_x=pos_margin_um_x,
244
+ pos_margin_um_z=pos_margin_um_z,
245
+ orthdist_min_um=orthdist_min_um,
246
+ orthdist_max_um=orthdist_max_um,
247
+ alpha_shape=alpha_shape,
248
+ alpha_scale=alpha_scale,
249
+ alpha_min=alpha_min,
250
+ alpha_max=alpha_max,
251
+ alpha_family=alpha_family,
252
+ )
253
+
254
+ t_up = self.time_domain_ms(up=True)
255
+ pos, alpha, sct, sct_up = simulate_point_source_templates(
256
+ n_units=n_units,
257
+ rg=self.rg,
258
+ temporal_jitter_kind=temporal_jitter_kind,
259
+ temporal_jitter=temporal_jitter,
260
+ trough_offset_samples=self.trough_offset_samples(),
261
+ time_domain=self.time_domain_ms(),
262
+ time_domain_up=t_up,
263
+ geom=self.geom3,
264
+ location_kw=self.location_kw,
265
+ singlechan_kw=self.waveform_kw,
266
+ decay_model=decay_model,
267
+ min_rms_distance=min_rms_distance,
268
+ force_no_offset=force_no_offset,
269
+ dtype=dtype,
270
+ )
271
+ self.template_pos = pos
272
+ self.template_alpha = alpha
273
+ self.offsets = sct[:, :, 0].argmin(1) - self.trough_offset_samples()
274
+ print(f"{self.offsets=}")
275
+ self.singlechan_templates = sct[..., 0]
276
+ self.singlechan_templates_up = sct_up[..., 0]
277
+
278
+ up_half = temporal_jitter // 2
279
+ off_up = (np.arange(temporal_jitter) > up_half).astype(np.int32)
280
+ off_up = self.offsets[:, None] - off_up[None, :]
281
+ self.offsets_up = off_up
282
+ assert (np.abs(sct.argmin(1) - self.trough_offset_samples()) <= 2).all()
283
+ np.testing.assert_allclose(sct, sct_up[:, 0], atol=1e-15)
284
+
285
+ def templates(self, drift=0, up=False, padded=False, pad_value=np.nan):
286
+ pos = self.template_pos
287
+ if drift:
288
+ pos = pos + [0, 0, drift]
289
+
290
+ geom3 = self.geom3
291
+ if padded:
292
+ geom3 = np.pad(geom3, [(0, 1), (0, 0)])
293
+
294
+ templates = singlechan_to_probe(
295
+ pos,
296
+ self.template_alpha,
297
+ self.singlechan_templates_up if up else self.singlechan_templates,
298
+ geom3,
299
+ decay_model=self.decay_model,
300
+ )
301
+ if self.common_reference and geom3.shape[0] > 1:
302
+ tunpad = templates[..., :-1] if padded else templates
303
+ tunpad -= np.median(tunpad, axis=-1, keepdims=True)
304
+ if padded:
305
+ templates[..., -1] = pad_value
306
+ off = self.offsets_up if up else self.offsets
307
+ return pos, templates, off
308
+
309
+ def trough_offset_samples(self):
310
+ return int(self.ms_before * (self.sampling_frequency / 1000))
311
+
312
+ def spike_length_samples(self):
313
+ spike_len_ms = self.ms_before + self.ms_after
314
+ length = int(spike_len_ms * (self.sampling_frequency / 1000))
315
+ length = 2 * (length // 2) + 1
316
+ return length
317
+
318
+ def time_domain_ms(self, up=False):
319
+ t1 = self.spike_length_samples()
320
+ if up:
321
+ nt = t1 * self.temporal_jitter
322
+ else:
323
+ nt = t1
324
+ t = np.linspace(0.0, t1, dtype=self.dtype, endpoint=False, num=nt)
325
+ t -= self.trough_offset_samples()
326
+ t /= self.sampling_frequency / 1000
327
+ return t
328
+
329
+
330
+ class TemplateLibrarySimulator(BaseTemplateSimulator):
331
+ def __init__(
332
+ self,
333
+ geom: np.ndarray,
334
+ templates_local,
335
+ pos_local,
336
+ radius=250.0,
337
+ temporal_jitter=1,
338
+ common_reference=False,
339
+ trough_offset_samples=42,
340
+ interp_method: Literal["griddata", "dart"] = "dart",
341
+ griddata_method: str = "cubic",
342
+ interp_params: InterpolationParams = default_interpolation_params,
343
+ ):
344
+ self.geom = geom
345
+ self.geom_kdt = KDTree(geom)
346
+ self.temporal_jitter = temporal_jitter
347
+ self.common_reference = common_reference
348
+ self.radius = radius
349
+ self.interp_params = interp_params.normalize()
350
+ self.griddata_method = griddata_method
351
+ self.channel_index = make_channel_index(
352
+ geom=geom, radius=radius, to_torch=False
353
+ )
354
+ self._trough_offset_samples = trough_offset_samples
355
+
356
+ self.n_units = len(templates_local)
357
+ self.templates_local = templates_local
358
+ self.low_rank_templates = svd_compress_templates(templates_local, allow_na=True)
359
+ self.temporal_up = upsample_multichan(
360
+ self.low_rank_templates.temporal_components, temporal_jitter=temporal_jitter
361
+ )
362
+ self.spatial_singular = (
363
+ self.low_rank_templates.spatial_components
364
+ * self.low_rank_templates.singular_values[..., None]
365
+ )
366
+
367
+ tpos = pos_local[
368
+ np.arange(self.n_units),
369
+ np.nan_to_num(np.abs(templates_local).max(1)).argmax(1),
370
+ ]
371
+ assert tpos.shape[1] == 2
372
+ self.template_pos = np.c_[tpos[:, 0], 0 * tpos[:, 0], tpos[:, 1]]
373
+ assert np.array_equal(
374
+ np.isfinite(self.templates_local[:, 0]),
375
+ np.isfinite(self.low_rank_templates.spatial_components[:, 0]),
376
+ )
377
+ self.pos_local = pos_local
378
+
379
+ # precompute interpolation data
380
+ self.interp_method = interp_method
381
+ if interp_method != "griddata":
382
+ self.precomputed_data = interp_precompute(
383
+ source_pos=pos_local, params=self.interp_params
384
+ )
385
+ else:
386
+ self.precomputed_data = None
387
+
388
+ _, _, offsets_up = self.templates(up=True)
389
+ offsets_up = offsets_up - self.trough_offset_samples()
390
+ self.offsets_up = offsets_up.reshape(self.n_units, temporal_jitter)
391
+
392
+ def trough_offset_samples(self):
393
+ return self._trough_offset_samples
394
+
395
+ def spike_length_samples(self):
396
+ return self.templates_local.shape[1]
397
+
398
+ @classmethod
399
+ def from_template_library(
400
+ cls,
401
+ geom,
402
+ n_units,
403
+ templates,
404
+ randomize_position=True,
405
+ common_reference=False,
406
+ temporal_jitter=1,
407
+ extract_radius=250.0,
408
+ inject_radius=500.0,
409
+ trough_offset_samples=42,
410
+ pos_margin_um_z=25.0,
411
+ seed=0,
412
+ dtype="float32",
413
+ interp_params: InterpolationParams = default_interpolation_params,
414
+ **kwargs,
415
+ ):
416
+ rg = np.random.default_rng(seed)
417
+ if templates.shape[0] > n_units:
418
+ choices = rg.choice(len(templates), size=n_units, replace=False)
419
+ templates = templates[choices]
420
+ templates = templates.astype(dtype)
421
+
422
+ assert np.isfinite(templates).all()
423
+ channel_index = make_channel_index(geom, extract_radius, to_torch=False)
424
+ main_channels = np.abs(templates).max(1).argmax(1)
425
+ template_channels = channel_index[main_channels]
426
+
427
+ geomp = np.pad(geom.astype(dtype), [(0, 1), (0, 0)], constant_values=np.nan)
428
+ templatesp = np.pad(templates, [(0, 0), (0, 0), (0, 1)], constant_values=np.nan)
429
+ pos_local = geomp[template_channels]
430
+ templates_local = np.take_along_axis(
431
+ templatesp, template_channels[:, None], axis=2
432
+ )
433
+
434
+ if randomize_position:
435
+ z_low = geom[:, 1].min() - pos_margin_um_z
436
+ z_high = geom[:, 1].max() + pos_margin_um_z
437
+ z = rg.uniform(z_low, z_high, size=n_units)
438
+ # z = rg.choice(np.unique(geom[:, 1]), size=n_units)
439
+ pos_local[..., 1] += z[:, None] - geom[main_channels, 1][:, None]
440
+
441
+ return cls(
442
+ geom=geom,
443
+ templates_local=templates_local,
444
+ pos_local=pos_local,
445
+ temporal_jitter=temporal_jitter,
446
+ common_reference=common_reference,
447
+ trough_offset_samples=trough_offset_samples,
448
+ radius=inject_radius,
449
+ interp_params=interp_params,
450
+ **kwargs,
451
+ )
452
+
453
+ def interpolate_templates(self, source_pos, target_pos, unit_ids, up=False):
454
+ # interpolate spatial components
455
+ spatial_singular = self.spatial_singular[unit_ids]
456
+ if self.interp_method == "dart":
457
+ assert self.precomputed_data is not None
458
+ out = kernel_interpolate(
459
+ spatial_singular,
460
+ source_pos,
461
+ target_pos,
462
+ params=self.interp_params,
463
+ precomputed_data=self.precomputed_data[unit_ids],
464
+ ).numpy(force=True)
465
+ elif self.interp_method == "griddata":
466
+ n, nct, _2 = target_pos.shape
467
+ n_, f, nc_ = spatial_singular.shape
468
+ assert _2 == 2
469
+ assert n == n_
470
+ assert nc_ == source_pos.shape[1]
471
+ if torch.is_tensor(spatial_singular):
472
+ spatial_singular = spatial_singular.numpy(force=True)
473
+ if torch.is_tensor(source_pos):
474
+ source_pos = source_pos.numpy(force=True)
475
+ if torch.is_tensor(target_pos):
476
+ target_pos = target_pos.numpy(force=True)
477
+ out = np.full((n, f, nct), np.nan, dtype=spatial_singular.dtype)
478
+ griddata_interp(
479
+ spatial_singular,
480
+ source_pos,
481
+ target_pos,
482
+ out,
483
+ method=self.griddata_method,
484
+ )
485
+ else:
486
+ assert False
487
+
488
+ # temporal part...
489
+ n, r, c = spatial_singular.shape
490
+ if up:
491
+ temporal = self.temporal_up[unit_ids].reshape(n, -1, r)
492
+ else:
493
+ temporal = self.low_rank_templates.temporal_components[unit_ids]
494
+
495
+ return np.einsum("nrc,ntr->ntc", out, temporal)
496
+
497
+ def templates(
498
+ self, drift=None, up=False, padded=False, pad_value=np.nan, unit_ids=None
499
+ ):
500
+ if unit_ids is None:
501
+ unit_ids = slice(None)
502
+
503
+ source_pos = self.pos_local[unit_ids]
504
+ true_template_pos = self.template_pos[unit_ids]
505
+ if drift is not None:
506
+ drift = self.templates_local.dtype.type(drift)
507
+ zero = self.templates_local.dtype.type(0.0)
508
+ source_pos = source_pos + [zero, drift]
509
+ true_template_pos = true_template_pos + [zero, zero, drift]
510
+
511
+ tgeom = self.geom.astype(self.templates_local.dtype)
512
+ _, target_chans = self.geom_kdt.query(true_template_pos[:, [0, 2]])
513
+ tgeom = np.pad(tgeom, [(0, 1), (0, 0)], constant_values=np.nan)
514
+ target_chans = self.channel_index[target_chans]
515
+ target_pos = torch.asarray(tgeom[target_chans])
516
+
517
+ templates = self.interpolate_templates(source_pos, target_pos, unit_ids, up=up)
518
+
519
+ nu = len(templates)
520
+ nc_out = len(self.geom) + 1
521
+ nt = self.templates_local.shape[1]
522
+ up_factor = self.temporal_jitter if up else 1
523
+ out = np.zeros((nu, up_factor * nt, nc_out), dtype=self.templates_local.dtype)
524
+ np.put_along_axis(out, target_chans[:, None, :], templates, axis=2)
525
+
526
+ if up:
527
+ out = out.reshape(nu, up_factor, nt, nc_out)
528
+
529
+ if padded:
530
+ out[..., -1] = pad_value
531
+
532
+ if self.common_reference:
533
+ tunpad = out[..., :-1]
534
+ tunpad -= np.median(tunpad, axis=-1, keepdims=True)
535
+
536
+ if not padded:
537
+ out = out[..., :-1]
538
+
539
+ out_flat = out.reshape(nu * up_factor, nt, nc_out - (not padded)) if up else out
540
+ _, _, offsets = get_main_channels_and_alignments(templates=out_flat)
541
+ if up:
542
+ offsets = offsets.reshape(nu, up_factor)
543
+
544
+ return true_template_pos, out, offsets
545
+
546
+
547
+ def griddata_interp(templates, source_pos, target_pos, out, method):
548
+ n = templates.shape[0]
549
+ for j in range(n):
550
+ valid_in = np.flatnonzero(np.isfinite(source_pos[j, :, 0]))
551
+ valid_out = np.flatnonzero(np.isfinite(target_pos[j, :, 0]))
552
+ out[j, :, valid_out] = griddata(
553
+ points=source_pos[j, valid_in],
554
+ values=templates[j, :, valid_in],
555
+ xi=target_pos[j, valid_out],
556
+ method=method,
557
+ fill_value=0.0,
558
+ )
559
+ return out
560
+
561
+
562
+ TemplateSimulator = Union[
563
+ StaticTemplateSimulator, PointSource3ExpSimulator, TemplateLibrarySimulator
564
+ ]
565
+
566
+
567
+ # point source library fns
568
+
569
+
570
+ def simulate_source_locations(
571
+ *,
572
+ size: int,
573
+ geom: np.ndarray,
574
+ pos_margin_um_x: float,
575
+ pos_margin_um_z: float,
576
+ orthdist_min_um: float,
577
+ orthdist_max_um: float,
578
+ alpha_shape: float,
579
+ alpha_scale: float,
580
+ alpha_min: float,
581
+ alpha_max: float,
582
+ alpha_family: str,
583
+ rg: np.random.Generator,
584
+ depth_order: bool = True,
585
+ dtype: np.typing.DTypeLike,
586
+ ):
587
+ x_low = geom[:, 0].min() - pos_margin_um_x
588
+ x_high = geom[:, 0].max() + pos_margin_um_x
589
+ z_low = geom[:, -1].min() - pos_margin_um_z
590
+ z_high = geom[:, -1].max() + pos_margin_um_z
591
+
592
+ x = rg.uniform(x_low, x_high, size=size)
593
+ z = rg.uniform(z_low, z_high, size=size)
594
+ if depth_order:
595
+ z.sort()
596
+
597
+ orth = rg.uniform(orthdist_min_um, orthdist_max_um, size=size)
598
+
599
+ if alpha_family == "gamma":
600
+ alpha = rg.gamma(shape=alpha_shape, scale=alpha_scale, size=size)
601
+ elif alpha_family == "uniform":
602
+ alpha = rg.uniform(alpha_min, alpha_max, size=size)
603
+ else:
604
+ assert False
605
+
606
+ pos = np.c_[x, orth, z].astype(dtype)
607
+ alpha = alpha.astype(dtype)[:, None]
608
+ return pos, alpha
609
+
610
+
611
+ def simulate_singlechan(
612
+ *,
613
+ size: int,
614
+ time_domain: np.ndarray,
615
+ time_domain_up: np.ndarray,
616
+ trough_offset_samples: int,
617
+ snr_adjustment: float = 1.0,
618
+ temporal_jitter: int = 1,
619
+ # timing params
620
+ tip_before_min=0.1,
621
+ tip_before_max=0.5,
622
+ peak_after_min=0.2,
623
+ peak_after_max=0.8,
624
+ # width params
625
+ trough_width_min=0.005,
626
+ trough_width_max=0.025,
627
+ tip_width_min=0.01,
628
+ tip_width_max=0.05,
629
+ peak_width_min=0.05,
630
+ peak_width_max=0.2,
631
+ # rel height params
632
+ tip_rel_max=0.3,
633
+ peak_rel_max=0.5,
634
+ rg: np.random.Generator,
635
+ dtype: np.typing.DTypeLike,
636
+ up=False,
637
+ ):
638
+ """Simulate a trough-normalized 3-exp action potential."""
639
+ t = time_domain_up if up else time_domain
640
+
641
+ size_pad = (size, 1)
642
+
643
+ tip = rg.uniform(-tip_before_max, -tip_before_min, size=size_pad)
644
+ peak = rg.uniform(peak_after_min, peak_after_max, size=size_pad)
645
+ tip_height = rg.uniform(high=tip_rel_max, size=size_pad)
646
+ peak_height = rg.uniform(high=peak_rel_max, size=size_pad)
647
+ trough_width = rg.uniform(trough_width_min, trough_width_max, size=size_pad)
648
+ tip_width = rg.uniform(tip_width_min, tip_width_max, size=size_pad)
649
+ peak_width = rg.uniform(peak_width_min, peak_width_max, size=size_pad)
650
+
651
+ trough = -np.exp(-np.square(t) / (2 * trough_width))
652
+ tip = np.exp(-np.square(t - tip) / (2 * tip_width))
653
+ peak = np.exp(-np.square(t - peak) / (2 * peak_width))
654
+
655
+ waveforms = trough + tip_height * tip + peak_height * peak
656
+ center = trough_offset_samples
657
+ if up:
658
+ center = center * temporal_jitter
659
+ waveforms /= -waveforms[..., center, None]
660
+
661
+ waveforms = waveforms * snr_adjustment
662
+
663
+ return t, waveforms.astype(dtype)
664
+
665
+
666
+ def _simulate_point_source_templates(
667
+ *,
668
+ n_units: int,
669
+ rg: np.random.Generator,
670
+ temporal_jitter_kind: str,
671
+ temporal_jitter: int,
672
+ trough_offset_samples: int,
673
+ time_domain: np.ndarray,
674
+ time_domain_up: np.ndarray,
675
+ geom: np.ndarray,
676
+ location_kw: dict,
677
+ singlechan_kw: dict,
678
+ dtype: np.typing.DTypeLike,
679
+ ):
680
+ pos, alpha = simulate_source_locations(
681
+ size=n_units,
682
+ geom=geom,
683
+ **location_kw,
684
+ rg=rg,
685
+ dtype=dtype,
686
+ )
687
+ if temporal_jitter_kind == "exact":
688
+ t, sct_up = simulate_singlechan(
689
+ size=n_units,
690
+ time_domain=time_domain,
691
+ time_domain_up=time_domain_up,
692
+ trough_offset_samples=trough_offset_samples,
693
+ temporal_jitter=temporal_jitter,
694
+ **singlechan_kw,
695
+ rg=rg,
696
+ dtype=dtype,
697
+ up=True,
698
+ )
699
+ sct_up = sct_up.reshape(n_units, -1, temporal_jitter, 1)
700
+ sct_up = sct_up.transpose(0, 2, 1, 3)
701
+ sct = sct_up[:, 0]
702
+ else:
703
+ t, sct = simulate_singlechan(
704
+ size=n_units,
705
+ time_domain=time_domain,
706
+ time_domain_up=time_domain_up,
707
+ trough_offset_samples=trough_offset_samples,
708
+ **singlechan_kw,
709
+ rg=rg,
710
+ dtype=dtype,
711
+ up=False,
712
+ )
713
+ sct_up = upsample_singlechan(sct, temporal_jitter=temporal_jitter)
714
+ sct = sct[..., None]
715
+ sct_up = sct_up[..., None]
716
+ return pos, alpha, sct, sct_up
717
+
718
+
719
+ def _get_separated_group(
720
+ *,
721
+ n_grab: int,
722
+ pos: np.ndarray,
723
+ alpha: np.ndarray,
724
+ singlechan_templates: np.ndarray,
725
+ threshold: float,
726
+ geom3: np.ndarray,
727
+ decay_model: str,
728
+ rg: np.random.Generator,
729
+ ):
730
+ n = len(pos)
731
+ x = singlechan_to_probe(
732
+ pos=pos,
733
+ alpha=alpha,
734
+ waveforms=singlechan_templates,
735
+ geom3=geom3,
736
+ decay_model=decay_model,
737
+ )
738
+ d = pdist(x.reshape(n, -1), metric="sqeuclidean")
739
+ d = np.sqrt(d / np.prod(x.shape[1:]))
740
+ ii, jj = np.triu_indices(n, k=1)
741
+ mask = np.flatnonzero(d >= threshold)
742
+ ii = ii[mask]
743
+ jj = jj[mask]
744
+
745
+ # largest connected component (connected=separated)
746
+ v = np.ones(ii.shape)
747
+ coo = coo_array(((v, (ii, jj))), shape=(n, n))
748
+ coo = coo + coo.T
749
+ _, labels = connected_components(coo, directed=False, connection="strong")
750
+ u, c = np.unique(labels, return_counts=True)
751
+ ibig = np.argmax(c)
752
+ if c[ibig] < n_grab:
753
+ return None
754
+
755
+ # grab n random ones in there
756
+ in_big = np.flatnonzero(labels == u[ibig])
757
+ if in_big.shape[0] > n_grab:
758
+ in_big = rg.choice(in_big, size=n_grab, replace=False)
759
+ in_big.sort()
760
+
761
+ return in_big
762
+
763
+
764
+ def simulate_point_source_templates(
765
+ *,
766
+ n_units: int,
767
+ rg: np.random.Generator,
768
+ temporal_jitter_kind: str,
769
+ temporal_jitter: int,
770
+ trough_offset_samples: int,
771
+ time_domain: np.ndarray,
772
+ time_domain_up: np.ndarray,
773
+ geom: np.ndarray,
774
+ location_kw: dict,
775
+ singlechan_kw: dict,
776
+ decay_model: str,
777
+ min_rms_distance: float = 0.0,
778
+ oversampling: int = 4,
779
+ sampling_growth: int = 2,
780
+ max_oversampling: int = 40,
781
+ max_tries: int = 512,
782
+ force_no_offset: bool = False,
783
+ dtype: np.typing.DTypeLike,
784
+ ):
785
+ if not min_rms_distance:
786
+ oversampling = max_oversampling = sampling_growth = 1
787
+ puff = oversampling
788
+
789
+ for _ in range(max_tries):
790
+ sample_n = min(n_units * max_oversampling, n_units * puff)
791
+ pos, alpha, singlechan_templates, singlechan_templates_up = (
792
+ _simulate_point_source_templates(
793
+ temporal_jitter_kind=temporal_jitter_kind,
794
+ temporal_jitter=temporal_jitter,
795
+ trough_offset_samples=trough_offset_samples,
796
+ time_domain=time_domain,
797
+ time_domain_up=time_domain_up,
798
+ geom=geom,
799
+ location_kw=location_kw,
800
+ singlechan_kw=singlechan_kw,
801
+ n_units=sample_n,
802
+ rg=rg,
803
+ dtype=dtype,
804
+ )
805
+ )
806
+
807
+ # reject misaligned ones
808
+ if force_no_offset:
809
+ valid = singlechan_templates[:, :, 0].argmin(1) == trough_offset_samples
810
+ valid = np.flatnonzero(valid)
811
+ if valid.size < n_units:
812
+ continue
813
+ pos = pos[valid]
814
+ alpha = alpha[valid]
815
+ singlechan_templates = singlechan_templates[valid]
816
+ singlechan_templates_up = singlechan_templates_up[valid]
817
+
818
+ # reject too-close pairs
819
+ if min_rms_distance:
820
+ choices = _get_separated_group(
821
+ n_grab=n_units,
822
+ pos=pos,
823
+ alpha=alpha,
824
+ singlechan_templates=singlechan_templates,
825
+ threshold=min_rms_distance,
826
+ geom3=geom,
827
+ decay_model=decay_model,
828
+ rg=rg,
829
+ )
830
+ if choices is None:
831
+ puff = min(max_oversampling, puff * sampling_growth)
832
+ continue
833
+ pos = pos[choices]
834
+ alpha = alpha[choices]
835
+ singlechan_templates = singlechan_templates[choices]
836
+ singlechan_templates_up = singlechan_templates_up[choices]
837
+
838
+ break
839
+ else:
840
+ raise ValueError(f"Hit max template tries for {min_rms_distance=}.")
841
+
842
+ return pos, alpha, singlechan_templates, singlechan_templates_up