DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,518 @@
1
+ import math
2
+ from dataclasses import dataclass, replace
3
+
4
+ import numpy as np
5
+ import torch
6
+ import torch.nn.functional as F
7
+ from scipy.interpolate import interp1d
8
+ from scipy.spatial import KDTree
9
+
10
+ from ..util import drift_util, waveform_util
11
+ from ..util.job_util import ensure_computation_config
12
+ from ..util.logging_util import DARTSORTVERBOSE, get_logger
13
+ from ..util.spiketorch import ptp
14
+ from ..util.waveform_util import full_channel_index
15
+
16
+ logger = get_logger(__name__)
17
+
18
+
19
+ def weighted_average(unit_ids, templates, weights):
20
+ n_out = unit_ids.max() + 1
21
+ n_in, t, c = templates.shape
22
+ out = np.zeros((n_out, t, c), dtype=templates.dtype)
23
+ weights = weights.astype(float)
24
+ for i in range(n_out):
25
+ which_in = np.flatnonzero(unit_ids == i)
26
+ if not which_in.size:
27
+ continue
28
+
29
+ w = weights[which_in][:, None, None]
30
+ if w.sum() == 0:
31
+ continue
32
+ w /= w.sum()
33
+ out[i] = (w * templates[which_in]).sum(0)
34
+
35
+ return out
36
+
37
+
38
+ # -- template drift handling
39
+
40
+
41
+ def templates_at_time(
42
+ t_s,
43
+ registered_templates,
44
+ geom,
45
+ registered_template_depths_um=None,
46
+ registered_geom=None,
47
+ motion_est=None,
48
+ return_pitch_shifts=False,
49
+ geom_kdtree=None,
50
+ match_distance=None,
51
+ return_padded=False,
52
+ fill_value=torch.nan,
53
+ ):
54
+ if registered_geom is None and not return_padded:
55
+ return registered_templates
56
+ if registered_geom is None and return_padded:
57
+ assert torch.is_tensor(registered_templates)
58
+ return F.pad(registered_templates, (0, 1), value=fill_value)
59
+ assert motion_est is not None
60
+ assert registered_template_depths_um is not None
61
+
62
+ # for each unit, extract relevant channels at time t_s
63
+ # how many pitches to shift each unit relative to registered pos at time t_s?
64
+ unregistered_depths_um = drift_util.invert_motion_estimate(
65
+ motion_est, t_s, registered_template_depths_um
66
+ )
67
+ # reverse arguments to pitch shifts since we are going the other direction
68
+ pitch_shifts = drift_util.get_spike_pitch_shifts(
69
+ depths_um=registered_template_depths_um,
70
+ geom=geom,
71
+ registered_depths_um=unregistered_depths_um,
72
+ )
73
+ # extract relevant channel neighborhoods, also by reversing args to a drift helper
74
+ unregistered_templates = drift_util.get_waveforms_on_static_channels(
75
+ registered_templates,
76
+ registered_geom,
77
+ n_pitches_shift=pitch_shifts,
78
+ registered_geom=geom,
79
+ target_kdtree=geom_kdtree,
80
+ match_distance=match_distance,
81
+ fill_value=fill_value,
82
+ return_padded=return_padded,
83
+ )
84
+ if return_pitch_shifts:
85
+ return pitch_shifts, unregistered_templates
86
+ return unregistered_templates
87
+
88
+
89
+ def spatially_mask_templates(template_data, radius_um=0.0):
90
+ if not radius_um:
91
+ return template_data
92
+
93
+ tt = template_data.templates.copy()
94
+ ci = waveform_util.make_channel_index(template_data.registered_geom, radius_um)
95
+ chans = np.arange(ci.shape[0])
96
+ for j, t in enumerate(tt):
97
+ mask = ~np.isin(chans, ci[np.ptp(t, 0).argmax()])
98
+ tt[j, :, mask] = 0.0
99
+
100
+ return replace(template_data, templates=tt)
101
+
102
+
103
+ # -- template numerical processing
104
+
105
+
106
+ @dataclass
107
+ class LowRankTemplates:
108
+ unit_ids: np.ndarray
109
+ temporal_components: np.ndarray
110
+ singular_values: np.ndarray
111
+ spatial_components: np.ndarray
112
+ spike_counts_by_channel: np.ndarray | None
113
+
114
+ def shift_to_best_channels(self, geom, registered_geom=None) -> "LowRankTemplates":
115
+ if registered_geom is None or registered_geom.shape == geom.shape:
116
+ return self
117
+
118
+ pitch = waveform_util.get_pitch(geom)
119
+ rgkdt = KDTree(registered_geom)
120
+ low = registered_geom[:, 1].min() - geom[:, 1].min()
121
+ high = registered_geom[:, 1].max() - geom[:, 1].max()
122
+ assert low <= 0 <= high
123
+ nlow = int(-low // pitch)
124
+ nhigh = int(high // pitch)
125
+
126
+ spatial_components = self.spatial_components
127
+ if torch.is_tensor(spatial_components):
128
+ spatial_components = spatial_components.numpy(force=True)
129
+ scores = np.full(len(self.spatial_components), -np.inf)
130
+ best_spatial = spatial_components[:, :, : len(geom)] + 0.0
131
+
132
+ for j in range(-nlow, nhigh + 1):
133
+ gshift = geom + [0, j * pitch]
134
+ d, regchans = rgkdt.query(gshift)
135
+ assert np.all(regchans < rgkdt.n)
136
+
137
+ spatial = spatial_components[:, :, regchans]
138
+ new_scores = np.square(spatial).sum((1, 2))
139
+ better = np.flatnonzero(new_scores > scores)
140
+ best_spatial[better] = spatial[better]
141
+ scores[better] = new_scores[better]
142
+
143
+ return replace(self, spatial_components=best_spatial)
144
+
145
+
146
+ def svd_compress_templates(
147
+ template_data,
148
+ min_channel_amplitude=0.0,
149
+ rank=5,
150
+ channel_sparse=True,
151
+ allow_na=False,
152
+ computation_cfg=None,
153
+ batch_size=32,
154
+ ):
155
+ """
156
+ Returns:
157
+ temporal_components: n_units, spike_length_samples, rank
158
+ singular_values: n_units, rank
159
+ spatial_components: n_units, rank, n_channels
160
+ """
161
+ computation_cfg = ensure_computation_config(computation_cfg)
162
+ dev = computation_cfg.actual_device()
163
+ if hasattr(template_data, "templates"):
164
+ unit_ids = template_data.unit_ids
165
+ templates = template_data.templates
166
+ counts = template_data.spike_counts_by_channel
167
+ else:
168
+ templates = template_data
169
+ counts = None
170
+ unit_ids = np.arange(len(templates))
171
+
172
+ rank = min(rank, *templates.shape[1:])
173
+
174
+ amp_vecs = ptp(templates, dim=1, keepdims=True)
175
+ isna = np.isnan(amp_vecs)
176
+ if not allow_na:
177
+ assert not isna.any()
178
+ else:
179
+ amp_vecs = np.nan_to_num(amp_vecs, copy=False)
180
+ templates = np.nan_to_num(templates)
181
+ vis_mask = amp_vecs > min_channel_amplitude
182
+ dtype = templates.dtype
183
+
184
+ if not channel_sparse:
185
+ vis_templates = torch.as_tensor(templates * vis_mask)
186
+ U, S, Vh = _svd_helper(vis_templates)
187
+ # s is descending.
188
+ temporal_components = U[:, :, :rank].to(dtype).numpy(force=True)
189
+ singular_values = S[:, :rank].to(dtype).numpy(force=True)
190
+ spatial_components = Vh[:, :rank, :].to(dtype).numpy(force=True)
191
+ else:
192
+ # channel sparse: only SVD the nonzero channels
193
+ # this encodes the same exact subspace as above, and the reconstruction
194
+ # error is the same as above as a function of rank. it's just that
195
+ # we can zero out some spatial components, which is a useful property
196
+ # (used in pairwise convolutions for instance)
197
+
198
+ vis_mask = torch.as_tensor(vis_mask)
199
+ n, t, c = templates.shape
200
+ assert amp_vecs.shape == (n, 1, c)
201
+ templates = torch.as_tensor(templates)
202
+ uvec, groups = vis_mask[:, 0].unique(dim=0, return_inverse=True)
203
+ assert uvec.shape[1] == c
204
+
205
+ temporal_components = np.zeros((n, t, rank), dtype=dtype)
206
+ singular_values = np.zeros((n, rank), dtype=dtype)
207
+ spatial_components = np.zeros((n, rank, c), dtype=dtype)
208
+
209
+ for j in range(len(uvec)):
210
+ (in_j,) = (groups == j).nonzero(as_tuple=True)
211
+ nj = in_j.numel()
212
+ (mask,) = uvec[j].nonzero(as_tuple=True)
213
+ rankj = min(rank, mask.numel())
214
+ if not rankj:
215
+ continue
216
+
217
+ for bs in range(0, nj, batch_size):
218
+ be = min(bs + batch_size, nj)
219
+ binds = in_j[bs:be]
220
+
221
+ batch_x = templates[binds[:, None], :, mask[None, :]].to(dev)
222
+ # fancy ix comes to the front
223
+ assert batch_x.shape == (be - bs, mask.numel(), t)
224
+ U, S, Vh = _svd_helper(batch_x)
225
+ spatial_components[binds[:, None], :rankj, mask[None, :]] = U[
226
+ :, :, :rankj
227
+ ].numpy(force=True)
228
+ singular_values[binds, :rankj] = S[:, :rankj].numpy(force=True)
229
+ assert np.isfinite(singular_values[binds, :rankj]).all()
230
+ temporal_components[binds, :, :rankj] = Vh[:, :rankj, :].mT.numpy(
231
+ force=True
232
+ )
233
+ assert np.isfinite(temporal_components[binds, :, :rankj]).all()
234
+
235
+ if logger.isEnabledFor(DARTSORTVERBOSE):
236
+ assert np.isfinite(temporal_components).all()
237
+ assert np.isfinite(singular_values).all()
238
+ assert np.isfinite(spatial_components).all()
239
+ recon = np.einsum(
240
+ "ntr,nr,nrc->ntc", temporal_components, singular_values, spatial_components
241
+ )
242
+ tp = templates.numpy(force=True)
243
+ err = np.square(recon - tp).sum(axis=(1, 2))
244
+ ss = np.square(tp).sum(axis=(1, 2))
245
+ r2 = 1.0 - err / ss
246
+ logger.dartsortverbose(
247
+ "Individual basis reconstructed templates with min, mean, max R^2s: "
248
+ f"{r2.min().item():0.3f}, {r2.mean().item():0.3f}, {r2.max().item():0.3f}."
249
+ )
250
+
251
+ if allow_na:
252
+ isna = np.broadcast_to(isna, spatial_components.shape)
253
+ spatial_components[isna] = np.nan
254
+
255
+ return LowRankTemplates(
256
+ unit_ids=unit_ids,
257
+ temporal_components=temporal_components,
258
+ singular_values=singular_values,
259
+ spatial_components=spatial_components,
260
+ spike_counts_by_channel=counts,
261
+ )
262
+
263
+
264
+ @dataclass
265
+ class SharedBasisTemplates:
266
+ unit_ids: np.ndarray
267
+ # rank, time
268
+ temporal_components: np.ndarray
269
+ # n, rank, chans
270
+ spatial_singular: np.ndarray
271
+ spike_counts_by_channel: np.ndarray | None
272
+
273
+
274
+ def shared_basis_compress_templates(
275
+ template_data, min_channel_amplitude=1.0, rank=5, computation_cfg=None
276
+ ):
277
+ computation_cfg = ensure_computation_config(computation_cfg)
278
+ dev = computation_cfg.actual_device()
279
+ if hasattr(template_data, "templates"):
280
+ unit_ids = template_data.unit_ids
281
+ templates = template_data.templates
282
+ counts = template_data.spike_counts_by_channel
283
+ else:
284
+ templates = template_data
285
+ counts = None
286
+ unit_ids = np.arange(len(templates))
287
+ n, t, c = templates.shape
288
+ rank = min(rank, t)
289
+ amp_vecs = ptp(templates, dim=1)
290
+ assert np.isfinite(amp_vecs).all()
291
+ visible = amp_vecs > min_channel_amplitude
292
+ uu, cc = np.nonzero(visible)
293
+ nvis = uu.shape[0]
294
+
295
+ # put time on the first axis
296
+ if nvis == visible.size:
297
+ to_compress = templates.transpose(1, 0, 2)
298
+ to_compress = to_compress.reshape(t, n * c)
299
+ else:
300
+ to_compress = templates[uu, :, cc]
301
+ assert to_compress.shape == (nvis, t)
302
+ to_compress = to_compress.T
303
+
304
+ # get the temporal basis
305
+ # we'll use cov->eigh here since we'd project onto U anyway
306
+ to_compress = torch.asarray(to_compress, device=dev, dtype=torch.double)
307
+ # U, S, Vh = _svd_helper(to_compress)
308
+ # assert U.shape == (t, min(t, nvis))
309
+ cov = torch.cov(to_compress, correction=0)
310
+ m = to_compress.mean(dim=1, keepdim=True)
311
+ cov += m * m.T
312
+ if nvis < t:
313
+ logger.warning(f"Had {nvis=} smaller than {t=} in shared basis compression.")
314
+ cov.diagonal().add_(1e-5)
315
+ vals, U = torch.linalg.eigh(cov)
316
+ temporal_comps = U[:, -rank:]
317
+ # to rank-major
318
+ temporal_comps = temporal_comps.T.contiguous()
319
+ temporal_comps = temporal_comps.numpy(force=True)
320
+ temporal_comps = temporal_comps[::-1]
321
+ temporal_comps = np.ascontiguousarray(temporal_comps, dtype=templates.dtype)
322
+
323
+ # project templates onto temporal comps (no sparsity here.)
324
+ # spatial_sing = np.einsum("ntc,rt->nrc", templates, temporal_comps)
325
+ templates_t = templates.transpose(0, 2, 1).reshape(n * c, t)
326
+ spatial_sing = templates_t @ (temporal_comps).T
327
+ spatial_sing = spatial_sing.reshape(n, c, rank).transpose(0, 2, 1)
328
+ spatial_sing = np.ascontiguousarray(spatial_sing)
329
+
330
+ assert np.isfinite(temporal_comps).all()
331
+ assert np.isfinite(spatial_sing).all()
332
+
333
+ if logger.isEnabledFor(DARTSORTVERBOSE):
334
+ recon = torch.asarray(np.einsum("rt,nrc->ntc", temporal_comps, spatial_sing))
335
+ templates = torch.asarray(templates)
336
+ err = recon.sub_(templates).square_().sum(dim=(1, 2)).cpu()
337
+ ss = templates.square().sum(dim=(1, 2)).cpu()
338
+ r2 = 1.0 - err / ss
339
+ logger.dartsortverbose(
340
+ "Shared basis reconstructed templates with min, mean, max R^2s: "
341
+ f"{r2.min().item():0.3f}, {r2.mean().item():0.3f}, {r2.max().item():0.3f}."
342
+ )
343
+
344
+ return SharedBasisTemplates(
345
+ unit_ids=unit_ids,
346
+ temporal_components=temporal_comps,
347
+ spatial_singular=spatial_sing,
348
+ spike_counts_by_channel=counts,
349
+ )
350
+
351
+
352
+ def temporally_upsample_templates(
353
+ templates, temporal_upsampling_factor=8, kind="cubic"
354
+ ):
355
+ """Note, also works on temporal components thanks to compatible shape."""
356
+ n, t, c = templates.shape
357
+ tp = np.arange(t).astype(float)
358
+ erp = interp1d(tp, templates, axis=1, bounds_error=True, kind=kind)
359
+ tup = np.arange(t, step=1.0 / temporal_upsampling_factor) # type: ignore[reportCallIssue]
360
+ tup.clip(0, t - 1, out=tup)
361
+ upsampled_templates = erp(tup)
362
+ upsampled_templates = upsampled_templates.reshape(
363
+ n, t, temporal_upsampling_factor, c
364
+ )
365
+ upsampled_templates = upsampled_templates.astype(templates.dtype)
366
+ return upsampled_templates
367
+
368
+
369
+ @dataclass
370
+ class CompressedUpsampledTemplates:
371
+ n_compressed_upsampled_templates: int
372
+ compressed_upsampled_templates: np.ndarray
373
+ compressed_upsampling_map: np.ndarray
374
+ compressed_upsampling_index: np.ndarray
375
+ compressed_index_to_template_index: np.ndarray
376
+ compressed_index_to_upsampling_index: np.ndarray
377
+
378
+
379
+ def default_n_upsamples_map(ptps, max_upsample=8):
380
+ # avoid overflow in 4 ** by trimming ptp range in advance
381
+ max_ptp = 1 + 2 * math.log(max_upsample, 4)
382
+ ptps = np.minimum(ptps, max_ptp)
383
+ upsamples = 4 ** (ptps // 2)
384
+ upsamples = upsamples.astype(int)
385
+ np.clip(upsamples, 1, max_upsample, out=upsamples)
386
+ return upsamples
387
+
388
+
389
+ def compressed_upsampled_templates(
390
+ templates,
391
+ *,
392
+ ptps=None,
393
+ max_upsample=8,
394
+ n_upsamples_map=default_n_upsamples_map,
395
+ kind="cubic",
396
+ ):
397
+ """compressedly store fewer temporally upsampled copies of lower amplitude templates
398
+
399
+ Returns
400
+ -------
401
+ A CompressedUpsampledTemplates object with fields:
402
+ compressed_upsampled_templates : array (n_compressed_upsampled_templates, spike_length_samples, n_channels)
403
+ compressed_upsampling_map : array (n_templates, max_upsample)
404
+ compressed_upsampled_templates[compressed_upsampling_map[unit, j]] is an approximation
405
+ of the jth upsampled template for this unit. for low-amplitude units,
406
+ compressed_upsampling_map[unit] will have fewer unique entries, corresponding
407
+ to fewer saved upsampled copies for that unit.
408
+ compressed_upsampling_index : array (n_templates, max_upsample)
409
+ A n_compressed_upsampled_templates-padded ragged array mapping each
410
+ template index to its compressed upsampled indices
411
+ compressed_index_to_template_index
412
+ compressed_index_to_upsampling_index
413
+ """
414
+ n_templates = templates.shape[0]
415
+ assert templates.ndim == 3
416
+ if max_upsample == 1:
417
+ return CompressedUpsampledTemplates(
418
+ n_templates,
419
+ templates,
420
+ np.arange(n_templates)[:, None],
421
+ np.arange(n_templates)[:, None],
422
+ np.arange(n_templates),
423
+ np.zeros(n_templates, dtype=np.int64),
424
+ )
425
+
426
+ # how many copies should each unit get?
427
+ # sometimes users may pass temporal SVD components in instead of templates,
428
+ # so we allow them to pass in the amplitudes of the actual templates
429
+ if ptps is None:
430
+ ptps = np.ptp(templates, 1).max(1)
431
+ assert ptps.shape == (n_templates,)
432
+
433
+ if n_upsamples_map == "yass":
434
+ n_upsamples_map = default_n_upsamples_map
435
+ elif n_upsamples_map == "none":
436
+ n_upsamples_map = None
437
+
438
+ if n_upsamples_map is None:
439
+ n_upsamples = np.full(n_templates, max_upsample)
440
+ else:
441
+ n_upsamples = n_upsamples_map(ptps, max_upsample=max_upsample)
442
+
443
+ # build the compressed upsampling map
444
+ compressed_upsampling_map = np.full((n_templates, max_upsample), -1, dtype=np.int64)
445
+ compressed_upsampling_index = np.full(
446
+ (n_templates, max_upsample), -1, dtype=np.int64
447
+ )
448
+ template_indices = []
449
+ upsampling_indices = []
450
+ current_compressed_index = 0
451
+ for i, nup in enumerate(n_upsamples):
452
+ compression = max_upsample // nup
453
+ nup = max_upsample // compression # handle divisibility failure
454
+
455
+ # new compressed indices
456
+ compressed_upsampling_map[i] = current_compressed_index + np.arange(nup).repeat(
457
+ compression
458
+ )
459
+ compressed_upsampling_index[i, :nup] = current_compressed_index + np.arange(nup)
460
+ current_compressed_index += nup
461
+
462
+ # indices of the templates to keep in the full array of upsampled templates
463
+ template_indices.extend([i] * nup)
464
+ upsampling_indices.extend(compression * np.arange(nup))
465
+ assert (compressed_upsampling_map >= 0).all()
466
+ assert (
467
+ np.unique(compressed_upsampling_map).size
468
+ == (compressed_upsampling_index >= 0).sum()
469
+ == compressed_upsampling_map.max() + 1
470
+ == compressed_upsampling_index.max() + 1
471
+ == current_compressed_index
472
+ )
473
+ template_indices = np.array(template_indices)
474
+ upsampling_indices = np.array(upsampling_indices)
475
+ compressed_upsampling_index[compressed_upsampling_index < 0] = (
476
+ current_compressed_index
477
+ )
478
+
479
+ # get the upsampled templates
480
+ all_upsampled_templates = temporally_upsample_templates(
481
+ templates, temporal_upsampling_factor=max_upsample, kind=kind
482
+ )
483
+ # n, up, t, c
484
+ all_upsampled_templates = all_upsampled_templates.transpose(0, 2, 1, 3)
485
+ rix = np.ravel_multi_index(
486
+ (template_indices, upsampling_indices), all_upsampled_templates.shape[:2]
487
+ )
488
+ all_upsampled_templates = all_upsampled_templates.reshape(
489
+ n_templates * max_upsample, templates.shape[1], templates.shape[2]
490
+ )
491
+ compressed_upsampled_templates = all_upsampled_templates[rix]
492
+
493
+ return CompressedUpsampledTemplates(
494
+ current_compressed_index,
495
+ compressed_upsampled_templates,
496
+ compressed_upsampling_map,
497
+ compressed_upsampling_index,
498
+ template_indices,
499
+ upsampling_indices,
500
+ )
501
+
502
+
503
+ def _svd_helper(x, svd_dtype=torch.double):
504
+ """This matches numpy's behavior in tests/test_matching.py."""
505
+ orig_dtype = x.dtype
506
+ x = x.to(dtype=svd_dtype)
507
+ if x.device.type == "cuda":
508
+ U, S, Vh = torch.linalg.svd(x, full_matrices=False, driver="gesvd")
509
+ U = U.to(dtype=orig_dtype)
510
+ S = S.to(dtype=orig_dtype)
511
+ Vh = Vh.to(dtype=orig_dtype)
512
+ return U, S, Vh
513
+ else:
514
+ U, S, Vh = np.linalg.svd(x.numpy(), full_matrices=False)
515
+ U = torch.asarray(U, dtype=orig_dtype)
516
+ S = torch.asarray(S, dtype=orig_dtype)
517
+ Vh = torch.asarray(Vh, dtype=orig_dtype)
518
+ return U, S, Vh