DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,809 @@
1
+ """
2
+ A collection of helper functions for dealing with which channels
3
+ waveforms are extracted on, things like that.
4
+ """
5
+
6
+ from typing import overload, Literal
7
+
8
+ import numpy as np
9
+ import torch
10
+ import torch.nn.functional as F
11
+ from scipy.interpolate import CubicSpline
12
+ from scipy.spatial import KDTree
13
+ from scipy.spatial.distance import cdist, pdist, squareform
14
+ from tqdm.auto import trange
15
+
16
+ # -- geometry utils
17
+
18
+
19
+ def get_pitch(geom, direction=1, allow_horizontal=False):
20
+ """Guess the pitch, even for probes with gaps or channels missing at random
21
+
22
+ This is the unit at which the probe repeats itself, computed as the
23
+ vertical distance between electrodes in the same column. Of course, this
24
+ makes little sense for probes with non-lattice layouts, or even worse,
25
+ horizontal probes (i.e., x positions are unique) where it cannot be defined.
26
+
27
+ So for NP1, it's not every row, but every 2 rows! And for a probe with a
28
+ zig-zag arrangement, it would be also 2 vertical distances between channels.
29
+ """
30
+ other_dims = [i for i in range(geom.shape[1]) if i != direction]
31
+ other_dims_uniq = np.unique(geom[:, other_dims], axis=0)
32
+
33
+ # case 1: lattice layouts
34
+ # the pitch is determined as the minimal spacing between
35
+ # contacts whose position is the same along directions other
36
+ # than `direction`. this handles dense Neuropixels layouts.
37
+ pitch = np.inf
38
+ for pos in other_dims_uniq:
39
+ at_x = np.all(geom[:, other_dims] == pos, axis=1)
40
+ y_uniq_at_x = np.unique(geom[at_x, direction])
41
+ if y_uniq_at_x.size > 1:
42
+ pitch = min(pitch, np.diff(y_uniq_at_x).min())
43
+
44
+ # case 1 succeeded
45
+ if np.isfinite(pitch):
46
+ return pitch
47
+
48
+ # case 2: fallback for zig-zag layouts
49
+ # there are two main types of zig-zag layouts I have seen. one
50
+ # is a regular zig-zag. The other has a stagger half-way down
51
+ # (a certain choice of NP1 banks for a long configuration).
52
+ all_unique_y = np.unique(geom[:, direction])
53
+
54
+ # there is 1 unique y position... call it 0 pitch.
55
+ if all_unique_y.size == 1:
56
+ if allow_horizontal:
57
+ return 0
58
+ else:
59
+ raise ValueError("Horizontal probe.")
60
+
61
+ assert all_unique_y.size > 1
62
+ return np.diff(all_unique_y).min()
63
+
64
+
65
+ def get_orders(geom):
66
+ orders = np.ones(geom.shape[1])
67
+ for dim in range(geom.shape[1]):
68
+ other_dims = [i for i in range(geom.shape[1]) if i != dim]
69
+ other_dims_uniq = np.unique(geom[:, other_dims], axis=0)
70
+
71
+ sgns = set()
72
+ for pos in other_dims_uniq:
73
+ at_x = np.all(geom[:, other_dims] == pos, axis=1)
74
+ sgns.update(np.unique(np.sign(np.diff(geom[at_x, dim]))))
75
+
76
+ if len(sgns) == 1:
77
+ sign = list(sgns)[0]
78
+ orders[dim] = sign
79
+ return orders
80
+
81
+
82
+ def fill_geom_holes(geom):
83
+ pitch = get_pitch(geom)
84
+ pitches_pad = int(np.ceil(np.ptp(geom[:, 1]) / pitch))
85
+
86
+ # we have to be careful about floating point error here
87
+ # two sites may be different due to floating point error
88
+ # we know they are the same if their distance is smaller than:
89
+ min_distance = pdist(geom, metric="sqeuclidean").min() / 2
90
+
91
+ # find all shifted site positions
92
+ # TODO make this not quadratic
93
+ unique_shifted_positions = list(geom)
94
+ is_original = [True] * len(geom)
95
+ for shift in range(-pitches_pad, pitches_pad + 1):
96
+ shifted_geom = geom + [0, pitch * shift]
97
+ # sz = shifted_geom[:, 1]
98
+ # szinside = sz == sz.clip(
99
+ # geom[:, 1].min() - np.sqrt(min_distance),
100
+ # geom[:, 1].max() + np.sqrt(min_distance),
101
+ # )
102
+ # shifted_geom = shifted_geom[szinside]
103
+ dists = cdist(shifted_geom, unique_shifted_positions, metric="sqeuclidean")
104
+ for site, dists in zip(shifted_geom, dists):
105
+ if np.all(dists > min_distance):
106
+ unique_shifted_positions.append(site)
107
+ is_original.append(False)
108
+ unique_shifted_positions = np.array(unique_shifted_positions)
109
+ is_original = np.array(is_original)
110
+
111
+ # order by depth first, then horizontal position (unique goes the other way)
112
+ order = np.lexsort(unique_shifted_positions.T)
113
+ filled_geom = unique_shifted_positions[order]
114
+ is_original = is_original[order]
115
+ assert np.isclose(get_pitch(filled_geom), pitch)
116
+
117
+ return filled_geom, is_original
118
+
119
+
120
+ def regularize_geom(geom, radius=0, depth_only=False):
121
+ """Re-order, fill holes, and optionally expand geometry to make it 'regular'
122
+
123
+ Used in make_regular_channel_index. That docstring has some info about what's
124
+ going on here.
125
+ """
126
+ eps = pdist(geom).min() / 2.0
127
+ radius = np.atleast_1d(radius)
128
+ radius = np.broadcast_to(radius, geom.shape[1:])
129
+
130
+ if torch.is_tensor(geom):
131
+ geom = geom.numpy()
132
+
133
+ rgeom = geom.copy()
134
+ dims = [1] if depth_only else range(geom.shape[1])
135
+ for j in dims:
136
+ # skip empty dims
137
+ if np.ptp(geom[:, j]) < eps:
138
+ continue
139
+ rgeom = _regularize_1d(rgeom, radius=max(eps, radius[j]), eps=eps, dim=j)
140
+
141
+ # order regularized geom by depth and then x
142
+ order = np.lexsort(rgeom.T)
143
+ rgeom = rgeom[order]
144
+
145
+ return rgeom, eps
146
+
147
+
148
+ def _regularize_1d(geom, radius, eps, dim=1):
149
+ total = np.ptp(geom[:, dim])
150
+ dim_pitch = get_pitch(geom, direction=dim, allow_horizontal=True)
151
+ if dim_pitch == 0:
152
+ return geom
153
+ steps = int(np.ceil(total / dim_pitch))
154
+
155
+ min_pos = geom[:, dim].min() - radius - eps
156
+ max_pos = geom[:, dim].max() + radius + eps
157
+
158
+ all_positions = []
159
+ offset = np.zeros(geom.shape[1])
160
+ for step in range(-steps, steps + 1):
161
+ offset[dim] = step * dim_pitch
162
+ # add positions within the radius
163
+ offset_geom = geom + offset
164
+ keepers = offset_geom[:, dim].clip(min_pos, max_pos) == offset_geom[:, dim]
165
+ all_positions.append(offset_geom[keepers])
166
+
167
+ all_positions = np.concatenate(all_positions)
168
+ all_positions = np.unique(all_positions, axis=0)
169
+
170
+ # deal with fp tolerance
171
+ dists = squareform(pdist(all_positions))
172
+ A = dists < eps
173
+ n_neighbs = A.sum(0)
174
+ if n_neighbs.max() == 1:
175
+ return all_positions
176
+ else:
177
+ assert n_neighbs.max() > 1
178
+
179
+ from scipy.cluster.hierarchy import linkage, fcluster
180
+
181
+ Z = linkage(A.astype(np.float32))
182
+ labels = fcluster(Z, 1.1, criterion="distance")
183
+ labels -= 1
184
+ return all_positions[np.unique(labels)]
185
+
186
+
187
+ # -- channel index creation
188
+
189
+
190
+ @overload
191
+ def make_channel_index(
192
+ geom: np.ndarray | torch.Tensor,
193
+ radius: float,
194
+ *,
195
+ to_torch: Literal[True],
196
+ p: int | float = 2,
197
+ pad_val: int | None = None,
198
+ fill_holes: bool = False,
199
+ ) -> torch.LongTensor: ...
200
+
201
+
202
+ @overload
203
+ def make_channel_index(
204
+ geom: np.ndarray | torch.Tensor,
205
+ radius: float,
206
+ *,
207
+ to_torch: Literal[False],
208
+ p: int | float = 2,
209
+ pad_val: int | None = None,
210
+ fill_holes: bool = False,
211
+ ) -> np.ndarray: ...
212
+
213
+
214
+ @overload
215
+ def make_channel_index(
216
+ geom: np.ndarray | torch.Tensor,
217
+ radius: float,
218
+ *,
219
+ to_torch: bool = False,
220
+ p: int | float = 2,
221
+ pad_val: int | None = None,
222
+ fill_holes: bool = False,
223
+ ) -> np.ndarray | torch.Tensor: ...
224
+
225
+
226
+ def make_channel_index(
227
+ geom: np.ndarray | torch.Tensor,
228
+ radius: float,
229
+ *,
230
+ to_torch: bool = False,
231
+ p: int | float = 2,
232
+ pad_val: int | None = None,
233
+ fill_holes: bool = False,
234
+ ) -> np.ndarray | torch.Tensor:
235
+ """
236
+ Compute an array whose whose ith row contains the neighbors for the ith channel
237
+ """
238
+ C = geom.shape[0]
239
+ if not radius:
240
+ return single_channel_index(C, to_torch=to_torch)
241
+ if pad_val is None:
242
+ pad_val = C
243
+
244
+ if fill_holes:
245
+ return make_filled_channel_index(
246
+ geom, radius, p=p, pad_val=pad_val, to_torch=to_torch
247
+ )
248
+
249
+ # get neighbors matrix
250
+ if torch.is_tensor(geom):
251
+ geom = geom.numpy(force=True)
252
+ neighbors = squareform(pdist(geom, metric="minkowski", p=p)) <= radius
253
+
254
+ # max number of neighbors for all channels
255
+ n_neighbors = np.max(np.sum(neighbors, 0))
256
+
257
+ # initialize channel index
258
+ # entries for channels which don't have as many neighbors as
259
+ # others will be filled with the total number of channels
260
+ # (an invalid index into the recording, but this behavior
261
+ # is useful e.g. in the spatial max pooling for deduplication)
262
+ channel_index = np.full((C, n_neighbors), pad_val, dtype=np.int64)
263
+
264
+ # fill every row in the matrix (one per channel)
265
+ for c in range(C):
266
+ # indices of c's neighbors
267
+ ch_idx = np.flatnonzero(neighbors[c])
268
+ channel_index[c, : ch_idx.shape[0]] = ch_idx
269
+
270
+ if to_torch:
271
+ channel_index = torch.LongTensor(channel_index)
272
+
273
+ return channel_index
274
+
275
+
276
+ def get_channel_index_rel_inds(channel_index: torch.Tensor):
277
+ chans_arange = torch.arange(
278
+ len(channel_index), device=channel_index.device, dtype=channel_index.dtype
279
+ )
280
+ jj, index_inds = (channel_index == chans_arange[:, None]).nonzero(as_tuple=True)
281
+ assert torch.equal(jj, chans_arange)
282
+ return index_inds
283
+
284
+
285
+ def make_filled_channel_index(
286
+ geom, radius, p: int | float = 2, pad_val=None, to_torch=False
287
+ ):
288
+ C = geom.shape[0]
289
+ if not radius:
290
+ return single_channel_index(C, to_torch=to_torch)
291
+ if pad_val is None:
292
+ pad_val = C
293
+
294
+ filled_geom, is_original = fill_geom_holes(geom)
295
+ filled_kdt = KDTree(filled_geom)
296
+ _, original_inds = filled_kdt.query(geom)
297
+ assert np.array_equal(filled_geom[original_inds], geom)
298
+ neighbors = cdist(geom, filled_geom, metric="minkowski", p=p) <= radius
299
+ n_neighbors = np.max(np.sum(neighbors, 0))
300
+ channel_index = np.full((C, n_neighbors), pad_val, dtype=np.int64)
301
+
302
+ # fill every row in the matrix (one per channel)
303
+ for c in range(C):
304
+ # indices of c's neighbors
305
+ filled_ch_idx = np.flatnonzero(neighbors[c])
306
+ ch_idx = np.searchsorted(original_inds, filled_ch_idx)
307
+ ch_idx[np.logical_not(is_original[filled_ch_idx])] = pad_val
308
+ channel_index[c] = ch_idx
309
+
310
+ if to_torch:
311
+ channel_index = torch.LongTensor(channel_index)
312
+
313
+ return channel_index
314
+
315
+
316
+ def make_regular_channel_index(geom, radius, p=2, to_torch=False, depth_only=False):
317
+ """Channel index for multi-channel models
318
+
319
+ In this channel index, the layout of channels around the max channel is
320
+ always consistent -- but this is achieved by dummy channels. This makes
321
+ for a consistent layout to input into multi-channel feature learners, at
322
+ least relative to the detection channel. However, those learners have to
323
+ deal with masked out dummy channels.
324
+
325
+ Example:
326
+
327
+ Let's say the probe looks like:
328
+ o o
329
+ o o
330
+ o
331
+ o o
332
+ And, let's say that our channel index's radius is twice the vertical
333
+ spacing (and say this is the same as the horizontal spacing). Then
334
+ the top left channel might have (eyeballing it) 4 neighbors (excluding
335
+ itself), the second row left channel maybe 5, the second row right channel
336
+ only 4 due to the hole.
337
+
338
+ This is padded out to a dummy probe (masked chans as xs) with holes filled in:
339
+ x x x x x x
340
+ x x x x x x
341
+ x x o o x x
342
+ x x o o x x
343
+ x x o x x x
344
+ x x o o x x
345
+ x x x x x x
346
+ x x x x x x
347
+ Now, all of the real channels have the same number of neighbors and in fact
348
+ exist in the same spatial relationship with their channel neighborhood. It's
349
+ just that some of those neighbors are fake. But, extracting a radiul channel
350
+ index here will lead to a consistent layout -- as long as the channels are
351
+ ordered right (!!).
352
+
353
+ Note that for probes which don't have a regular layout, this just doesn't
354
+ make sense at all. I'm thinking of ones that have weird layouts where one channel
355
+ is way bigger than the others and serves as a reference, etc -- throw those
356
+ away!
357
+ """
358
+ rgeom, eps = regularize_geom(geom=geom, radius=radius, depth_only=depth_only)
359
+ if np.array(radius).size > 1:
360
+ radius = np.sqrt(np.square(radius).sum())
361
+
362
+ # determine original geom's position in the regularized one, and which
363
+ # channels are fake chans (they are unmatched in the query)
364
+ kdt = KDTree(geom)
365
+ dists, reg2orig = kdt.query(rgeom, k=1, distance_upper_bound=eps)
366
+ # the usual extra padding chan
367
+ reg2orig = np.concatenate((reg2orig, [kdt.n]))
368
+
369
+ # make regularized channel index...
370
+ rci = make_channel_index(rgeom, radius, p=p)
371
+
372
+ # subset it to non-fake chans and replace regularized channel indices with
373
+ # the corresponding original channel index (or len(geom) if fake)
374
+ real_reg = np.flatnonzero(reg2orig < kdt.n)
375
+ real_reg_ix = reg2orig[real_reg]
376
+ ordered_real_reg = real_reg[np.argsort(real_reg_ix)]
377
+ channel_index = reg2orig[rci[ordered_real_reg]]
378
+ channel_index = channel_index.astype(np.int64)
379
+
380
+ if to_torch:
381
+ channel_index = torch.from_numpy(channel_index)
382
+
383
+ return channel_index
384
+
385
+
386
+ def regularize_channel_index(
387
+ geom, channel_index, p=2, to_torch=False, depth_only=False
388
+ ):
389
+ """Convert a channel index to the "regular" format
390
+
391
+ Need to know the p used in the first place.
392
+ """
393
+ nchans = len(geom)
394
+
395
+ # current radius
396
+ radius = 0
397
+ for j, row in enumerate(channel_index):
398
+ row = row[row < nchans]
399
+ radius = max(radius, cdist(geom[row], geom[j][None]).max())
400
+
401
+ regular_channel_index = make_regular_channel_index(
402
+ geom, radius, p=p, depth_only=depth_only
403
+ )
404
+
405
+ for c in range(nchans):
406
+ regrow = regular_channel_index[c]
407
+ origrow = channel_index[c]
408
+ invalid = np.logical_not(np.isin(regrow, origrow))
409
+ regular_channel_index[c, invalid] = nchans
410
+
411
+ # check that this is a re-encoding of the same channels
412
+ newc = regular_channel_index[c]
413
+ newc = newc[newc < nchans]
414
+ oldc = origrow[origrow < nchans]
415
+ assert np.array_equal(np.sort(newc), np.sort(oldc))
416
+
417
+ if to_torch:
418
+ regular_channel_index = torch.from_numpy(regular_channel_index)
419
+
420
+ return regular_channel_index
421
+
422
+
423
+ def make_contiguous_channel_index(n_channels, n_neighbors=40):
424
+ """Channel index with linear neighborhoods in channel id space"""
425
+ channel_index = []
426
+ for c in range(n_channels):
427
+ low = max(0, c - n_neighbors // 2)
428
+ low = min(n_channels - n_neighbors, low)
429
+ channel_index.append(np.arange(low, low + n_neighbors))
430
+ channel_index = np.array(channel_index, dtype=np.int64)
431
+
432
+ return channel_index
433
+
434
+
435
+ def full_channel_index(n_channels, to_torch=False):
436
+ """Everyone is everone's neighbor"""
437
+ ci = (
438
+ np.arange(n_channels, dtype=np.int64)[None, :]
439
+ * np.ones(n_channels, dtype=np.int64)[:, None]
440
+ )
441
+ if to_torch:
442
+ ci = torch.tensor(ci)
443
+ return ci
444
+
445
+
446
+ def single_channel_index(n_channels, to_torch=False):
447
+ """Lonely islands"""
448
+ ci = np.arange(n_channels, dtype=np.int64)[:, None]
449
+ if to_torch:
450
+ ci = torch.tensor(ci)
451
+ return ci
452
+
453
+
454
+ # -- extracting single channels which were not part of padding
455
+
456
+
457
+ # This is used heavily in dartsort.transform, where for instance
458
+ # we might want to avoid the nans used as padding channels when
459
+ # fitting models.
460
+
461
+
462
+ def get_channels_in_probe(waveforms, max_channels, channel_index):
463
+ n, t, c = waveforms.shape
464
+ assert max_channels.shape == (n,)
465
+ assert channel_index.ndim == 2 and channel_index.shape[1] == c
466
+ waveforms = waveforms.permute(0, 2, 1)
467
+ in_probe_index = channel_index < channel_index.shape[0]
468
+ channels_in_probe = in_probe_index[max_channels]
469
+ waveforms_in_probe = waveforms[channels_in_probe]
470
+ return channels_in_probe, waveforms_in_probe
471
+
472
+
473
+ def set_channels_in_probe(
474
+ waveforms_in_probe_src,
475
+ waveforms_full_dest,
476
+ channels_in_probe,
477
+ in_place=False,
478
+ ):
479
+ waveforms_full_dest = waveforms_full_dest.permute(0, 2, 1)
480
+ if not in_place:
481
+ waveforms_full_dest = waveforms_full_dest.clone()
482
+ waveforms_full_dest[channels_in_probe] = waveforms_in_probe_src
483
+ return waveforms_full_dest.permute(0, 2, 1)
484
+
485
+
486
+ # -- channel subsetting
487
+
488
+
489
+ def channel_subset_by_radius(
490
+ waveforms,
491
+ max_channels,
492
+ channel_index,
493
+ geom,
494
+ radius=None,
495
+ n_channels_subset=None,
496
+ fill_value=torch.nan,
497
+ return_new_channel_index=True,
498
+ ):
499
+ """Restrict waveforms (or amplitude vectors) to channels inside a radius"""
500
+ channel_index_mask = get_channel_index_mask(
501
+ geom, channel_index, radius=radius, n_channels_subset=n_channels_subset
502
+ )
503
+ waveforms_subset = get_channel_subset(waveforms, max_channels, channel_index_mask)
504
+ if return_new_channel_index:
505
+ new_channel_index = mask_to_channel_index(channel_index, channel_index_mask)
506
+ return waveforms_subset, new_channel_index
507
+ return waveforms_subset
508
+
509
+
510
+ def channel_subset_mask(channel_index_full, channel_index_new, to_torch=True):
511
+ n_channels = channel_index_full.shape[0]
512
+ mask = np.stack(
513
+ [
514
+ np.isin(row_full, np.setdiff1d(row_new, [n_channels]))
515
+ for row_full, row_new in zip(channel_index_full, channel_index_new)
516
+ ],
517
+ axis=0,
518
+ )
519
+ if to_torch:
520
+ mask = torch.tensor(mask)
521
+ return mask
522
+
523
+
524
+ def channel_subset_by_index(
525
+ waveforms,
526
+ max_channels,
527
+ channel_index_full,
528
+ channel_index_new,
529
+ fill_value=torch.nan,
530
+ chunk_length=None,
531
+ ):
532
+ """Restrict waveforms to channels in new channel index"""
533
+ # boolean mask of same shape as channel_index_full
534
+ # figure out which channels in channel_index_full are still
535
+ # present in the form of a boolean mask
536
+ channel_index_mask = channel_subset_mask(
537
+ channel_index_full,
538
+ channel_index_new,
539
+ to_torch=torch.is_tensor(channel_index_new),
540
+ )
541
+ if torch.is_tensor(channel_index_full):
542
+ channel_index_mask = torch.from_numpy(channel_index_mask)
543
+ return get_channel_subset(
544
+ waveforms, max_channels, channel_index_mask, chunk_length=chunk_length
545
+ )
546
+
547
+
548
+ def get_channel_index_mask(geom, channel_index, radius=None, n_channels_subset=None):
549
+ """Get a boolean mask showing if channels are inside a radial/linear subset
550
+
551
+ Subsetting is controlled by a radius or by a number of channels. Radius
552
+ takes priority.
553
+ """
554
+ assert geom.ndim == channel_index.ndim == 2
555
+ assert geom.shape[0] == channel_index.shape[0]
556
+ is_tensor = torch.is_tensor(channel_index)
557
+ npx = torch if is_tensor else np
558
+
559
+ if is_tensor:
560
+ subset = torch.zeros(
561
+ size=channel_index.shape, device=channel_index.device, dtype=torch.bool
562
+ )
563
+ pgeom = F.pad(geom, (0, 0, 0, 1), value=torch.nan)
564
+ else:
565
+ subset = np.zeros(shape=channel_index.shape, dtype=bool)
566
+ pgeom = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
567
+
568
+ for c in range(len(geom)):
569
+ if radius is not None:
570
+ dists = npx.square(geom[c][None] - pgeom[channel_index[c]]).sum(1)
571
+ subset[c] = dists <= radius**2
572
+ elif n_channels_subset is not None:
573
+ low = max(0, c - n_channels_subset // 2)
574
+ low = min(len(geom) - n_channels_subset, low)
575
+ high = min(len(geom), low + n_channels_subset)
576
+ subset[c] = (low <= channel_index[c]) & (channel_index[c] < high)
577
+ else:
578
+ subset[c] = True
579
+
580
+ return subset
581
+
582
+
583
+ def mask_to_relative(channel_index_mask):
584
+ assert channel_index_mask.ndim == 2
585
+ max_sub_chans = channel_index_mask.sum(axis=1).max()
586
+ original_max_neighbs = channel_index_mask.shape[1]
587
+ n_channels_tot = channel_index_mask.shape[0]
588
+
589
+ is_tensor = torch.is_tensor(channel_index_mask)
590
+ if is_tensor:
591
+ rel_sub_channel_index = torch.full(
592
+ (n_channels_tot, max_sub_chans),
593
+ original_max_neighbs,
594
+ device=channel_index_mask.device,
595
+ dtype=torch.long,
596
+ )
597
+ else:
598
+ rel_sub_channel_index = np.full(
599
+ (n_channels_tot, max_sub_chans), original_max_neighbs, dtype=np.int64
600
+ )
601
+
602
+ for i, mask in enumerate(channel_index_mask):
603
+ if is_tensor:
604
+ nz = mask.nonzero().squeeze()
605
+ nnz = nz.numel()
606
+ else:
607
+ nz = np.flatnonzero(mask)
608
+ nnz = nz.size
609
+ if nnz:
610
+ rel_sub_channel_index[i, :nnz] = nz # type: ignore
611
+
612
+ return rel_sub_channel_index
613
+
614
+
615
+ def mask_to_channel_index(channel_index, channel_index_mask):
616
+ assert channel_index.shape == channel_index_mask.shape
617
+ max_sub_chans = channel_index_mask.sum(axis=1).max()
618
+ n_channels = channel_index.shape[0]
619
+
620
+ is_tensor = torch.is_tensor(channel_index_mask)
621
+ if is_tensor:
622
+ new_channel_index = torch.full(
623
+ (n_channels, max_sub_chans),
624
+ n_channels,
625
+ device=channel_index_mask.device,
626
+ dtype=torch.long,
627
+ )
628
+ else:
629
+ new_channel_index = np.full(
630
+ (n_channels, max_sub_chans), fill_value=n_channels, dtype=np.int64
631
+ )
632
+
633
+ for i, mask in enumerate(channel_index_mask):
634
+ if is_tensor:
635
+ which = mask.nonzero().squeeze()
636
+ nnz = which.numel()
637
+ else:
638
+ which = np.flatnonzero(mask)
639
+ nnz = which.size
640
+ if nnz:
641
+ new_channel_index[i, :nnz] = channel_index[i][which]
642
+ return new_channel_index
643
+
644
+
645
+ def get_channel_subset(
646
+ waveforms,
647
+ max_channels,
648
+ channel_index_mask,
649
+ fill_value=np.nan,
650
+ chunk_length=None,
651
+ in_place=False,
652
+ out=None,
653
+ rel_sub_channel_index=None,
654
+ show_progress=True,
655
+ ):
656
+ """Given a binary mask indicating which channels to keep, grab those channels"""
657
+ if waveforms.ndim == 3:
658
+ N, T, C = waveforms.shape
659
+ pads = [(0, 0), (0, 0)]
660
+ elif waveforms.ndim == 2:
661
+ # for instance, amplitudes
662
+ N, C = waveforms.shape
663
+ pads = [(0, 0)]
664
+ else:
665
+ assert False
666
+ n_channels, C_ = channel_index_mask.shape
667
+ assert C == C_
668
+ is_torch = torch.is_tensor(waveforms)
669
+ npx = np
670
+ if is_torch:
671
+ npx = torch
672
+
673
+ if rel_sub_channel_index is None:
674
+ rel_sub_channel_index = mask_to_relative(channel_index_mask)
675
+ if is_torch:
676
+ rel_sub_channel_index = torch.asarray(rel_sub_channel_index)
677
+ n_chan_sub = rel_sub_channel_index.shape[1]
678
+
679
+ if in_place and out is None:
680
+ out = waveforms[..., :n_chan_sub]
681
+
682
+ if chunk_length is not None:
683
+ if out is None:
684
+ out = npx.zeros_like(waveforms[..., :n_chan_sub])
685
+ xrange = trange if show_progress else range
686
+ for bs in xrange(0, len(out), chunk_length):
687
+ sl = slice(bs, min(len(out), bs + chunk_length))
688
+ get_channel_subset(
689
+ waveforms[sl],
690
+ max_channels[sl],
691
+ channel_index_mask,
692
+ fill_value=fill_value,
693
+ chunk_length=None,
694
+ out=out[sl],
695
+ rel_sub_channel_index=rel_sub_channel_index,
696
+ )
697
+ return out
698
+
699
+ # convert to relative channel offsets
700
+ if is_torch:
701
+ waveforms = F.pad(waveforms, (0, 1), value=fill_value)
702
+ take_along_dim = torch.take_along_dim
703
+ else:
704
+ waveforms = np.pad(waveforms, [*pads, (0, 1)], constant_values=fill_value)
705
+
706
+ def take_along_dim(input, indices, dim, *, out=None):
707
+ res = np.take_along_axis(input, indices, axis=dim)
708
+ if out is not None:
709
+ out[:] = res
710
+ res = out
711
+ return res
712
+
713
+ inds = rel_sub_channel_index[max_channels]
714
+ if waveforms.ndim == 3:
715
+ inds = inds[:, None, :]
716
+
717
+ return take_along_dim(
718
+ input=waveforms, # type: ignore
719
+ indices=inds, # type: ignore
720
+ dim=waveforms.ndim - 1,
721
+ out=out,
722
+ )
723
+
724
+
725
+ def relative_channel_subset_index(channel_index_full, channel_index_new, to_torch=True):
726
+ mask = channel_subset_mask(channel_index_full, channel_index_new, to_torch=True)
727
+ rel_sub_channel_index = mask_to_relative(mask)
728
+ return rel_sub_channel_index
729
+
730
+
731
+ def get_relative_subset(
732
+ waveforms, max_channels, rel_sub_channel_index, fill_value=torch.nan
733
+ ):
734
+ waveforms = F.pad(waveforms, (0, 1), value=fill_value)
735
+ index = rel_sub_channel_index[max_channels]
736
+ if waveforms.ndim == 3:
737
+ index = index[:, None, :].broadcast_to(
738
+ index.shape[0], waveforms.shape[1], index.shape[1]
739
+ )
740
+ return torch.gather(waveforms, waveforms.ndim - 1, index)
741
+
742
+
743
+ def grab_main_channels(waveforms, main_channels, channel_index, keepdim=False):
744
+ nc = len(channel_index)
745
+ _, relative_positions = np.nonzero((channel_index == np.arange(nc)[:, None]))
746
+ assert relative_positions.shape == (nc,)
747
+ inds = relative_positions[main_channels]
748
+ res = np.take_along_axis(waveforms, inds[:, None, None], axis=2)
749
+ if keepdim:
750
+ return res
751
+ return res[:, :, 0]
752
+
753
+
754
+ def grab_main_channels_torch(waveforms, channels, channel_index):
755
+ nc = len(channel_index)
756
+ chans_arange = torch.arange(nc, dtype=torch.long).to(channel_index)
757
+ _, rpos = (channel_index == chans_arange.unsqueeze(1)).nonzero(as_tuple=True)
758
+ assert rpos.shape == (nc,)
759
+ inds = rpos[channels]
760
+ return torch.take_along_dim(waveforms, inds[:, None, None], dim=2)
761
+
762
+
763
+ # -- temporal processing
764
+
765
+
766
+ def upsample_singlechan_torch(
767
+ singlechan_waveforms: torch.Tensor, temporal_jitter: int
768
+ ) -> torch.Tensor:
769
+ up = upsample_singlechan(
770
+ singlechan_waveforms.numpy(force=True), temporal_jitter=temporal_jitter
771
+ )
772
+ return torch.asarray(up).to(singlechan_waveforms)
773
+
774
+
775
+ def upsample_singlechan(singlechan_waveforms, temporal_jitter=1):
776
+ """nt -> nut"""
777
+ if temporal_jitter == 1:
778
+ return singlechan_waveforms[:, None]
779
+
780
+ n, t = singlechan_waveforms.shape
781
+ time_domain = np.linspace(0.0, float(t), num=t)
782
+ erp = CubicSpline(time_domain, singlechan_waveforms, axis=-1)
783
+ dt_ms = np.diff(time_domain).mean()
784
+ t_up = np.stack(
785
+ [time_domain + dt_ms * (j / temporal_jitter) for j in range(temporal_jitter)],
786
+ axis=1,
787
+ )
788
+ t_up = t_up.ravel()
789
+ erp_y_up = erp(t_up)
790
+ erp_y_up = erp_y_up.reshape(n, t, temporal_jitter)
791
+ singlechan_waveforms_up = erp_y_up.transpose(0, 2, 1)
792
+ return singlechan_waveforms_up
793
+
794
+
795
+ def upsample_multichan(waveforms, temporal_jitter=1):
796
+ """ntc -> nutc"""
797
+ if temporal_jitter == 1:
798
+ return waveforms[:, None]
799
+ n, t, c = waveforms.shape
800
+ dtype = waveforms.dtype
801
+ waveforms = waveforms.transpose(0, 2, 1).reshape((n * c, t))
802
+ invalid = np.flatnonzero(np.isnan(waveforms[:, 0]))
803
+ waveforms = np.nan_to_num(waveforms, copy=False)
804
+ waveforms = upsample_singlechan(waveforms, temporal_jitter=temporal_jitter)
805
+ waveforms[invalid] = np.nan
806
+ waveforms = waveforms.reshape(n, c, temporal_jitter, t)
807
+ waveforms = waveforms.transpose(0, 2, 3, 1)
808
+ waveforms = np.ascontiguousarray(waveforms, dtype=dtype)
809
+ return waveforms