DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,993 @@
1
+ from dataclasses import dataclass
2
+ from typing import Literal, Self, Sequence
3
+
4
+ import h5py
5
+ import numpy as np
6
+ import torch
7
+
8
+ from ...transform.temporal_pca import TemporalPCAFeaturizer
9
+ from ...util import drift_util, interpolation_util, spiketorch
10
+ from ...util.internal_config import (
11
+ InterpolationParams,
12
+ default_interpolation_params,
13
+ default_extrapolation_params,
14
+ )
15
+ from ...util.data_util import DARTsortSorting, get_tpca
16
+ from ...util.job_util import get_global_computation_config
17
+ from ...util.torch_util import BModule
18
+
19
+
20
+ # -- main class
21
+
22
+
23
+ class StableSpikeDataset(torch.nn.Module):
24
+ """
25
+ ## On SpikeNeighborhoods
26
+
27
+ Recall, we store spikes on two kinds of channel subsets: the larger
28
+ "extract" and the smaller "core". Extract is used for fitting (M step)
29
+ and core for likelihood computation (E step).
30
+
31
+ During fitting, each unit determines some set of channels that it's
32
+ "active" on. At E time, any spike whose core neighborhood is contained
33
+ in those channels gets a likelihood computed (the assumption being that
34
+ it would otherwise have smaller-than-noise likelihood under that unit
35
+ due to determinant).
36
+
37
+ It is not cheap to compute likelihoods on spikes from arbitrary channels
38
+ willy nilly. It's better to compute likelihoods for all spikes living
39
+ on the same exatch channel set at once. So, we cache info about what
40
+ neighborhood each spike belongs to, and ways to reverse-lookup spikes
41
+ by neighborhood. We even store the core spike features by neighborhood
42
+ to avoid doing a million index_selects from large arrays.
43
+
44
+ ## On splits
45
+
46
+ We make use of at least two splits during inference: train and val.
47
+ The train split is the one whose extract features we use during the M
48
+ step, and whose core features we use during the E step. The val split
49
+ is just used for model comparisons during split/merge.
50
+
51
+ I say at least two because if the number of spikes is large, we will
52
+ choose to completely ignore a random subset of spikes. The name of that
53
+ split doesn't matter because it's not really used, but let's call it the
54
+ "ignore" split.
55
+
56
+ After fitting a model using the train and val sets, we can still
57
+ compute likelihoods for all spikes, even the ignored ones.
58
+
59
+ This means that:
60
+ - For "extract" spikes, we only need to load the "train" split and
61
+ make a SpikeNeighborhoods structure for the train split.
62
+ - For "core" spikes, we need to load all of the spikes, and we need
63
+ to cache SpikeNeighborhoods for train, train+val, and all 3.
64
+
65
+ ## How are these used from the SpikeMixtureModel?
66
+
67
+ In the M step, indices for fitting are chosen from the train split.
68
+
69
+ The E step then also only needs to run on the train split most of the
70
+ time. There is a `split='train'` flag for `log_likelihoods()` and for
71
+ `e_step()`. This works by dispatching to the split in question's
72
+ SpikeNeighborhoods structure. The returned sparse array of likelihoods
73
+
74
+
75
+
76
+ """
77
+
78
+ def __init__(
79
+ self,
80
+ kept_indices: np.ndarray,
81
+ prgeom: torch.Tensor,
82
+ tpca: TemporalPCAFeaturizer,
83
+ extract_channels: torch.Tensor,
84
+ core_channels: torch.Tensor | None,
85
+ original_sorting: DARTsortSorting,
86
+ core_features: torch.Tensor | None,
87
+ train_extract_features: torch.Tensor,
88
+ features_on_device: bool = False,
89
+ split_names: Sequence[str] | None = None,
90
+ split_mask: torch.Tensor | None = None,
91
+ core_radius: float | Literal["extract"] | None = 35.0,
92
+ interp_params: InterpolationParams = default_interpolation_params,
93
+ extrap_params: InterpolationParams = default_extrapolation_params,
94
+ extract_neighborhoods=None,
95
+ extract_neighborhood_ids=None,
96
+ core_neighborhoods=None,
97
+ core_neighborhood_ids=None,
98
+ device=None,
99
+ _core_feature_splits=("train", "kept"),
100
+ ):
101
+ """Motion-corrected spike data on the registered probe"""
102
+ super().__init__()
103
+
104
+ if device is None:
105
+ device = "cuda" if torch.cuda.is_available() else "cpu"
106
+ device = torch.device(device)
107
+ self.original_sorting = original_sorting
108
+
109
+ self.features_on_device = features_on_device
110
+
111
+ # data shapes
112
+ self.rank = tpca.rank
113
+ self.n_channels = prgeom.shape[0] - 1
114
+ self.n_channels_extract = extract_channels.shape[1]
115
+ self.core_is_extract = core_radius == "extract"
116
+ if core_radius is not None:
117
+ assert core_channels is not None
118
+ self.n_channels_core = core_channels.shape[1]
119
+ self.n_spikes = len(original_sorting)
120
+ # train is modified below if there is a train split.
121
+ self.n_spikes_train = self.n_spikes_kept = len(kept_indices)
122
+ self.core_radius = core_radius
123
+ # assert extrap_method != "kriging"
124
+ self.interp_params = interp_params.normalize()
125
+ self.extrap_params = extrap_params.normalize()
126
+
127
+ # spike collection indices
128
+ self.kept_indices = torch.from_numpy(kept_indices)
129
+ # split indices are within kept_indices
130
+ self.has_splits = split_names is not None
131
+ self.split_mask = split_mask
132
+ self.split_indices = dict(full=slice(None), kept=self.kept_indices)
133
+ if self.has_splits:
134
+ assert split_names is not None
135
+ self.split_indices.update(
136
+ (k, (split_mask == j).nonzero().view(-1))
137
+ for j, k in enumerate(split_names)
138
+ )
139
+
140
+ # pca module, for reconstructing wfs for vis
141
+ self.tpca = tpca
142
+
143
+ # neighborhoods module, for querying spikes by channel group
144
+ self.not_train_indices = torch.tensor([], dtype=torch.long)
145
+ if self.has_splits and "train" in self.split_indices:
146
+ train_ixs = self.split_indices["train"]
147
+ self.n_spikes_train = len(train_ixs) # type: ignore
148
+ train_enids = None
149
+ if extract_neighborhood_ids is not None:
150
+ train_enids = extract_neighborhood_ids[train_ixs]
151
+ self._train_extract_neighborhoods = SpikeNeighborhoods.from_channels(
152
+ extract_channels[train_ixs],
153
+ n_channels=self.n_channels,
154
+ neighborhood_ids=train_enids,
155
+ neighborhoods=extract_neighborhoods,
156
+ deduplicate=not self.core_is_extract,
157
+ device=device,
158
+ name="extract",
159
+ )
160
+ self._train_extract_channels = extract_channels.cpu()[train_ixs]
161
+ assert torch.is_tensor(train_ixs)
162
+ self.not_train_indices = torch.asarray(
163
+ np.setdiff1d(np.arange(self.n_spikes), train_ixs),
164
+ dtype=torch.long,
165
+ )
166
+ else:
167
+ train_ixs = None
168
+
169
+ if core_radius is not None:
170
+ _core_neighborhoods = {
171
+ f"key_{k}": SpikeNeighborhoods.from_channels(
172
+ channels=core_channels[ix], # type: ignore
173
+ n_channels=self.n_channels,
174
+ neighborhood_ids=(
175
+ None
176
+ if core_neighborhood_ids is None
177
+ else core_neighborhood_ids[ix]
178
+ ),
179
+ neighborhoods=core_neighborhoods,
180
+ features=core_features[ix] if k in _core_feature_splits else None, # type: ignore
181
+ device=device,
182
+ name=k,
183
+ )
184
+ for k, ix in self.split_indices.items()
185
+ if (not self.core_is_extract or k != "train")
186
+ }
187
+ if self.core_is_extract and "train" in self.split_indices:
188
+ assert torch.is_tensor(train_ixs)
189
+ assert core_channels is not None
190
+ assert torch.equal(
191
+ core_channels[train_ixs], extract_channels[train_ixs]
192
+ )
193
+ _core_neighborhoods["train"] = self._train_extract_neighborhoods
194
+ self._core_neighborhoods = torch.nn.ModuleDict(_core_neighborhoods)
195
+ self.core_channels = core_channels.cpu() # type: ignore
196
+
197
+ # channel neighborhoods and features
198
+ # if not self.features_on_device, .spike_data() will .to(self.device)
199
+ times_s = torch.asarray(self.original_sorting.times_seconds) # type: ignore[reportAttributeAccessIssue]
200
+ self.core_features = core_features
201
+ if self.features_on_device:
202
+ self.register_buffer("_train_extract_features", train_extract_features)
203
+ self.register_buffer("times_seconds", times_s)
204
+ else:
205
+ self._train_extract_features = train_extract_features
206
+ self.times_seconds = times_s
207
+
208
+ # always on device
209
+ self.register_buffer("prgeom", prgeom)
210
+
211
+ @property
212
+ def device(self) -> torch.device:
213
+ return self.prgeom.device # type: ignore
214
+
215
+ @property
216
+ def dtype(self):
217
+ return self._train_extract_features.dtype
218
+
219
+ @classmethod
220
+ def from_config(
221
+ cls,
222
+ sorting,
223
+ refinement_cfg,
224
+ motion_est=None,
225
+ computation_cfg=None,
226
+ _core_feature_splits=("train", "kept"),
227
+ ):
228
+ if computation_cfg is None:
229
+ computation_cfg = get_global_computation_config()
230
+ return cls.from_sorting(
231
+ sorting,
232
+ motion_est=motion_est,
233
+ core_radius=refinement_cfg.core_radius,
234
+ max_n_spikes=refinement_cfg.max_n_spikes,
235
+ interp_params=refinement_cfg.interp_params.normalize(),
236
+ split_proportions=(
237
+ 1.0 - refinement_cfg.val_proportion,
238
+ refinement_cfg.val_proportion,
239
+ ),
240
+ device=computation_cfg.actual_device(),
241
+ _core_feature_splits=_core_feature_splits,
242
+ )
243
+
244
+ @classmethod
245
+ def from_sorting(
246
+ cls,
247
+ sorting,
248
+ motion_est=None,
249
+ core_radius: float | Literal["extract"] | None = 35.0,
250
+ max_n_spikes: float | int = np.inf,
251
+ kept_indices: np.ndarray | None = None,
252
+ discard_triaged: bool = False,
253
+ interp_params: InterpolationParams = default_interpolation_params,
254
+ features_dataset_name="collisioncleaned_tpca_features",
255
+ motion_depth_mode="channel",
256
+ split_names=("train", "val"),
257
+ split_proportions=(0.75, 0.25),
258
+ show_progress=True,
259
+ store_on_device=False,
260
+ workers=-1,
261
+ device=None,
262
+ feature_rank: int = 8,
263
+ random_seed: int | np.random.Generator = 0,
264
+ _core_feature_splits=("train", "kept"),
265
+ ):
266
+ if device is None:
267
+ device = "cuda" if torch.cuda.is_available() else "cpu"
268
+ device = torch.device(device)
269
+
270
+ # load information not stored directly on the sorting
271
+ with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
272
+ geom = h5["geom"][:] # type: ignore
273
+ extract_channel_index: np.ndarray = h5["channel_index"][:] # type: ignore
274
+ assert np.all(np.diff(extract_channel_index) >= 0)
275
+ if motion_est is None:
276
+ registered_geom = geom
277
+ else:
278
+ registered_geom = drift_util.registered_geometry(
279
+ geom, motion_est=motion_est
280
+ )
281
+ prgeom = interpolation_util.pad_geom(registered_geom)
282
+
283
+ res = drift_util.get_shift_info(
284
+ sorting, motion_est, geom, motion_depth_mode
285
+ )
286
+ channels, shifts, n_pitches_shift = res
287
+
288
+ # determine channel occupancy of core/extract stable features
289
+ res = drift_util.get_stable_channels(
290
+ geom,
291
+ channels,
292
+ extract_channel_index,
293
+ registered_geom,
294
+ n_pitches_shift,
295
+ core_radius=core_radius,
296
+ workers=workers,
297
+ device=device,
298
+ )
299
+
300
+ extract_channels, extract_neighborhoods, extract_neighborhood_ids = res[:3]
301
+ core_channels, core_neighborhoods, core_neighborhood_ids = res[3:]
302
+ if core_radius is not None:
303
+ assert core_channels is not None
304
+ assert core_neighborhoods is not None
305
+ assert core_neighborhood_ids is not None
306
+
307
+ if core_radius == "extract":
308
+ assert core_neighborhoods is not None
309
+ assert core_neighborhood_ids is not None
310
+ assert np.array_equal(extract_neighborhoods, core_neighborhoods)
311
+ assert np.array_equal(extract_neighborhood_ids, core_neighborhood_ids)
312
+
313
+ rg = np.random.default_rng(random_seed)
314
+
315
+ assert len(split_names) >= 1
316
+ assert "train" in split_names
317
+ if kept_indices is not None:
318
+ pass
319
+ elif discard_triaged:
320
+ kept_indices = np.flatnonzero(sorting.labels >= 0)
321
+ elif len(sorting) <= max_n_spikes:
322
+ kept_indices = np.arange(len(sorting))
323
+ else:
324
+ assert isinstance(max_n_spikes, int)
325
+ kept_indices = rg.choice(len(sorting), size=max_n_spikes, replace=False)
326
+ kept_indices.sort()
327
+
328
+ split_mask = torch.full((len(sorting),), -1, dtype=torch.int8)
329
+ # train set initialized to everything
330
+ assert split_names[0] == "train"
331
+ split_mask[kept_indices] = 0
332
+ # val takes from uncovered
333
+ if "val" in split_names:
334
+ assert "val" == split_names[1]
335
+ assert split_names == ("train", "val")
336
+ n_val = int(np.ceil(split_proportions[1] * len(kept_indices)))
337
+ val_candidates = rg.choice(kept_indices, size=n_val, replace=False)
338
+ val_candidates.sort()
339
+ split_mask[val_candidates] = 1
340
+ else:
341
+ assert split_names == ("train",)
342
+ train_mask = (split_mask == 0).numpy()
343
+
344
+ # for all spikes (not just kept), the ID of its shift/chan combo.
345
+ # this determines its channel neighborhood under any registered index.
346
+ # we also get the total counts in each combo, and the indices of
347
+ # the first spikes in each combo, allowing neighbs to be reconstructed
348
+ # as needed
349
+ extract_channels = torch.from_numpy(extract_channels)
350
+ if store_on_device:
351
+ extract_channels = extract_channels.to(device)
352
+
353
+ if core_radius is not None:
354
+ core_channels = torch.from_numpy(core_channels)
355
+ if store_on_device:
356
+ core_channels = core_channels.to(device)
357
+
358
+ # stabilize features with interpolation
359
+ # only load kept extract spikes
360
+ train_extract_features = interpolation_util.interpolate_by_chunk(
361
+ train_mask,
362
+ h5[features_dataset_name],
363
+ geom,
364
+ extract_channel_index,
365
+ sorting.channels[train_mask],
366
+ shifts[train_mask],
367
+ registered_geom,
368
+ extract_channels[train_mask],
369
+ trim_to_rank=feature_rank,
370
+ params=interp_params,
371
+ device=device,
372
+ store_on_device=store_on_device,
373
+ show_progress=show_progress,
374
+ )
375
+ # always load all core spikes
376
+ core_features = None
377
+ if core_radius is not None:
378
+ core_features = interpolation_util.interpolate_by_chunk(
379
+ np.ones(len(sorting), dtype=np.bool_),
380
+ h5[features_dataset_name],
381
+ geom,
382
+ extract_channel_index,
383
+ sorting.channels,
384
+ shifts,
385
+ registered_geom,
386
+ core_channels,
387
+ trim_to_rank=feature_rank,
388
+ params=interp_params,
389
+ device=device,
390
+ store_on_device=store_on_device,
391
+ show_progress=show_progress,
392
+ )
393
+
394
+ # load temporal PCA
395
+ tpca = get_tpca(sorting)
396
+
397
+ if core_radius == "extract":
398
+ core_radius = -np.inf
399
+ for j in range(len(extract_channel_index)):
400
+ chans = extract_channel_index[j]
401
+ assert isinstance(geom, np.ndarray)
402
+ chans = chans[chans < len(geom)]
403
+ tmp = np.subtract(geom[chans], geom[j])
404
+ np.square(tmp, out=tmp)
405
+ max_dist = tmp.sum(axis=1).max()
406
+ core_radius = max(core_radius, np.sqrt(max_dist))
407
+
408
+ self = cls(
409
+ kept_indices=kept_indices,
410
+ prgeom=prgeom,
411
+ tpca=tpca, # type: ignore
412
+ extract_channels=extract_channels,
413
+ extract_neighborhoods=extract_neighborhoods,
414
+ extract_neighborhood_ids=extract_neighborhood_ids,
415
+ core_channels=core_channels, # type: ignore
416
+ core_neighborhoods=core_neighborhoods,
417
+ core_neighborhood_ids=core_neighborhood_ids,
418
+ original_sorting=sorting,
419
+ core_features=core_features,
420
+ train_extract_features=train_extract_features,
421
+ features_on_device=store_on_device,
422
+ split_names=split_names,
423
+ split_mask=split_mask,
424
+ core_radius=core_radius,
425
+ device=device,
426
+ interp_params=interp_params,
427
+ _core_feature_splits=_core_feature_splits,
428
+ )
429
+ self.to(device)
430
+ return self
431
+
432
+ def spike_data(
433
+ self,
434
+ indices: torch.Tensor,
435
+ split_indices: torch.Tensor | None = None,
436
+ neighborhood: str = "extract",
437
+ with_channels=True,
438
+ with_reconstructions: bool = False,
439
+ with_neighborhood_ids: bool = False,
440
+ split: str | None = "train",
441
+ feature_buffer: torch.Tensor | None = None,
442
+ ) -> "SpikeFeatures":
443
+ withbuf = feature_buffer is not None
444
+ non_blocking = False
445
+ channels = neighborhood_ids = None
446
+
447
+ if neighborhood == "extract":
448
+ assert split == "train"
449
+ assert split_indices is not None
450
+
451
+ if withbuf:
452
+ features = feature_buffer[: len(indices)]
453
+ non_blocking = features.is_pinned()
454
+ torch.index_select(
455
+ self._train_extract_features, 0, split_indices, out=features
456
+ )
457
+ else:
458
+ features = self._train_extract_features[split_indices]
459
+ if with_channels:
460
+ channels = self._train_extract_channels[split_indices]
461
+ if with_neighborhood_ids:
462
+ neighborhood_ids = self._train_extract_neighborhoods.b.neighborhood_ids[
463
+ split_indices
464
+ ]
465
+ elif neighborhood == "core":
466
+ if withbuf:
467
+ features = feature_buffer[: len(indices)]
468
+ non_blocking = features.is_pinned()
469
+ torch.index_select(self.core_features, 0, indices, out=features) # type: ignore
470
+ else:
471
+ features = self.core_features[indices] # type: ignore
472
+ if with_channels:
473
+ channels = self.core_channels[indices]
474
+ if with_neighborhood_ids:
475
+ _, core_neighborhoods = self.neighborhoods(split="full")
476
+ neighborhood_ids = core_neighborhoods.b.neighborhood_ids[indices]
477
+ else:
478
+ assert False
479
+
480
+ if not self.features_on_device:
481
+ features = features.to(self.device, non_blocking=non_blocking)
482
+ if with_channels:
483
+ channels = channels
484
+
485
+ waveforms = None
486
+ if with_reconstructions:
487
+ n = len(features)
488
+ assert channels is not None
489
+ c = channels.shape[1]
490
+ waveforms = features.permute(0, 2, 1).reshape(n * c, -1)
491
+ waveforms = self.tpca._inverse_transform_in_probe(waveforms)
492
+ waveforms = waveforms.reshape(n, c, -1).permute(0, 2, 1)
493
+
494
+ return SpikeFeatures(
495
+ indices=indices,
496
+ split_indices=split_indices,
497
+ features=features,
498
+ channels=channels,
499
+ waveforms=waveforms,
500
+ neighborhood_ids=neighborhood_ids,
501
+ )
502
+
503
+ def neighborhoods(
504
+ self, neighborhood="core", split="train"
505
+ ) -> tuple[torch.Tensor | slice, "SpikeNeighborhoods"]:
506
+ split_ixs = self.split_indices[split]
507
+
508
+ if neighborhood == "extract":
509
+ assert split == "train"
510
+ return split_ixs, self._train_extract_neighborhoods
511
+
512
+ return split_ixs, self._core_neighborhoods[f"key_{split}"] # type: ignore
513
+
514
+ def interp_to_chans(self, spike_data, channels):
515
+ return interp_to_chans(
516
+ spike_data, channels, self.prgeom, params=self.extrap_params
517
+ )
518
+
519
+
520
+ # -- utility classes
521
+
522
+
523
+ @dataclass(frozen=True, slots=True, kw_only=True)
524
+ class SpikeFeatures:
525
+ """A 'messenger class' to hold onto batches of spike features
526
+
527
+ Return type of StableSpikeDataset's spike_data method.
528
+ """
529
+
530
+ # these are relative to the StableFeatures instance's subsampling (keepers)
531
+ indices: torch.Tensor
532
+ # n, r, c
533
+ features: torch.Tensor
534
+ # n, c
535
+ channels: torch.Tensor | None = None
536
+ # n, t, c
537
+ waveforms: torch.Tensor | None = None
538
+ # n
539
+ neighborhood_ids: torch.Tensor | None = None
540
+ split_indices: torch.Tensor | None = None
541
+
542
+ def __len__(self):
543
+ return len(self.features)
544
+
545
+ def __getitem__(self, ix):
546
+ """Subset the spikes in this collection with [subset]."""
547
+ waveforms = channels = neighborhood_ids = split_indices = None
548
+ if self.channels is not None:
549
+ channels = self.channels[ix]
550
+ if self.waveforms is not None:
551
+ waveforms = self.waveforms[ix]
552
+ if self.neighborhood_ids is not None:
553
+ neighborhood_ids = self.neighborhood_ids[ix]
554
+ if self.split_indices is not None:
555
+ split_indices = self.split_indices[ix]
556
+ return self.__class__(
557
+ indices=self.indices[ix],
558
+ features=self.features[ix],
559
+ channels=channels,
560
+ waveforms=waveforms,
561
+ neighborhood_ids=neighborhood_ids,
562
+ split_indices=split_indices,
563
+ )
564
+
565
+ def __repr__(self):
566
+ indstr = f"indices.shape={self.indices.shape},"
567
+ featstr = f"features.shape={self.features.shape},"
568
+ chanstr = wfstr = idstr = ""
569
+ if self.channels is not None:
570
+ chanstr = f"channels.shape={self.channels.shape},"
571
+ if self.waveforms is not None:
572
+ wfstr = f"waveforms.shape={self.waveforms.shape},"
573
+ if self.neighborhood_ids is not None:
574
+ idstr = f"neighborhood_ids.shape={self.neighborhood_ids.shape},"
575
+ pstr = f"{indstr}{featstr}{chanstr}{wfstr}{idstr}"
576
+ return f"{self.__class__.__name__}({pstr.rstrip(',')})"
577
+
578
+
579
+ class SpikeNeighborhoods(BModule):
580
+ def __init__(
581
+ self,
582
+ n_channels: int,
583
+ neighborhood_ids,
584
+ neighborhoods,
585
+ features=None,
586
+ neighborhood_members=None,
587
+ device=None,
588
+ name=None,
589
+ ):
590
+ """SpikeNeighborhoods
591
+
592
+ Sparsely keep track of which channels each spike lives on. Used to query
593
+ which core sets are overlapped completely by unit channel neighborhoods.
594
+
595
+ Arguments
596
+ ---------
597
+ neighborhood_ids : torch.Tensor
598
+ Size (n_spikes,), the neighborhood id for each spike
599
+ neighborhoods : list[torch.Tensor]
600
+ The channels in each neighborhood
601
+ neighborhood_members : list[torch.Tensor]
602
+ The indices of spikes in each neighborhood
603
+ """
604
+ super().__init__()
605
+ self.name = name
606
+ self.n_channels = n_channels
607
+ self.register_buffer("neighborhood_ids", neighborhood_ids.long())
608
+ self.register_buffer("chans_arange", torch.arange(n_channels, dtype=torch.long))
609
+ self.register_buffer("neighborhoods", neighborhoods.long())
610
+ self.n_neighborhoods = len(neighborhoods)
611
+
612
+ # store neighborhoods as an indicator matrix
613
+ # also store nonzero-d masks
614
+ indicators = torch.zeros((n_channels, len(neighborhoods)), device=device)
615
+ masks = []
616
+ mask_slices = []
617
+ offset = 0
618
+ for j, nhood in enumerate(neighborhoods):
619
+ jvalid = nhood < n_channels
620
+ indicators[nhood[jvalid], j] = 1.0
621
+ (jvalid,) = jvalid.nonzero(as_tuple=True)
622
+ jvalid = jvalid.long()
623
+ njvalid = jvalid.numel()
624
+ assert njvalid
625
+ masks.append(jvalid)
626
+ mask_slices.append(slice(offset, offset + njvalid))
627
+ offset += njvalid
628
+ self.register_buffer("indicators", indicators)
629
+ self.register_buffer("channel_counts", indicators.sum(0))
630
+ self.register_buffer("_masks", torch.concatenate(masks, dim=0))
631
+ self._mask_slices = mask_slices
632
+
633
+ if neighborhood_members is None:
634
+ # cache lookups
635
+ neighborhood_members = []
636
+ for j in range(len(neighborhoods)):
637
+ (in_nhood,) = torch.nonzero(neighborhood_ids == j, as_tuple=True)
638
+ in_nhood = in_nhood.long()
639
+ neighborhood_members.append(in_nhood.cpu())
640
+ assert len(neighborhood_members) == self.n_neighborhoods
641
+
642
+ # it's a pain to store dicts with register_buffer, so store offsets
643
+ _neighborhood_members = torch.empty(
644
+ sum(v.numel() for v in neighborhood_members), dtype=torch.long
645
+ )
646
+ self.neighborhood_members_slices = []
647
+ neighborhood_member_offset = 0
648
+ neighborhood_popcounts = []
649
+ for j in range(len(neighborhoods)):
650
+ nhoodmemsz = neighborhood_members[j].numel()
651
+ nhoodmemsl = slice(
652
+ neighborhood_member_offset, neighborhood_member_offset + nhoodmemsz
653
+ )
654
+ _neighborhood_members[nhoodmemsl] = neighborhood_members[j]
655
+ self.neighborhood_members_slices.append(nhoodmemsl)
656
+ neighborhood_member_offset += nhoodmemsz
657
+ neighborhood_popcounts.append(nhoodmemsz)
658
+ # self.register_buffer("_neighborhood_members", _neighborhood_members)
659
+ # seems that indices want to live on cpu.
660
+ self._neighborhood_members = _neighborhood_members.cpu()
661
+ self.register_buffer("popcounts", torch.tensor(neighborhood_popcounts))
662
+
663
+ if features is not None:
664
+ _features_valid = []
665
+ for j in range(len(neighborhoods)):
666
+ f = features[self.neighborhood_members(j)]
667
+ f = f[..., self.valid_mask(j).to(f.device)]
668
+ if device is not None and device.type == "cuda":
669
+ f = f.pin_memory()
670
+ _features_valid.append(f)
671
+ self._features_valid = _features_valid
672
+ self.to(device=device)
673
+
674
+ @classmethod
675
+ def from_channels(
676
+ cls,
677
+ channels,
678
+ n_channels,
679
+ neighborhood_ids=None,
680
+ neighborhoods=None,
681
+ device=None,
682
+ deduplicate=False,
683
+ features=None,
684
+ name=None,
685
+ ):
686
+ if neighborhood_ids is not None:
687
+ assert neighborhoods is not None
688
+ return cls.from_known_ids(
689
+ n_channels=n_channels,
690
+ neighborhood_ids=neighborhood_ids,
691
+ neighborhoods=neighborhoods,
692
+ device=device,
693
+ deduplicate=deduplicate,
694
+ features=features,
695
+ name=name,
696
+ )
697
+ if device is not None:
698
+ channels = channels.to(device)
699
+ neighborhoods, neighborhood_ids = torch.unique(
700
+ channels, dim=0, return_inverse=True
701
+ )
702
+ neighborhoods = neighborhoods.long()
703
+ neighborhood_ids = neighborhood_ids.long()
704
+ return cls(
705
+ n_channels=n_channels,
706
+ neighborhoods=neighborhoods,
707
+ neighborhood_ids=neighborhood_ids,
708
+ features=features,
709
+ device=channels.device,
710
+ name=name,
711
+ )
712
+
713
+ @classmethod
714
+ def from_known_ids(
715
+ cls,
716
+ *,
717
+ n_channels: int,
718
+ neighborhood_ids,
719
+ neighborhoods,
720
+ device=None,
721
+ deduplicate=False,
722
+ features=None,
723
+ name=None,
724
+ ):
725
+ neighborhoods = torch.asarray(neighborhoods, dtype=torch.long)
726
+ neighborhood_ids = torch.asarray(neighborhood_ids, dtype=torch.long)
727
+ if device is not None:
728
+ neighborhoods = neighborhoods.to(device)
729
+ neighborhood_ids = neighborhood_ids.to(device)
730
+ if deduplicate:
731
+ neighborhoods, old2new = torch.unique(
732
+ neighborhoods, dim=0, return_inverse=True
733
+ )
734
+ neighborhood_ids = old2new[neighborhood_ids]
735
+ kept_ids, neighborhood_ids = torch.unique(
736
+ neighborhood_ids, return_inverse=True
737
+ )
738
+ neighborhoods = neighborhoods[kept_ids]
739
+ return cls(
740
+ n_channels=n_channels,
741
+ neighborhoods=neighborhoods,
742
+ neighborhood_ids=neighborhood_ids,
743
+ features=features,
744
+ device=device,
745
+ name=name,
746
+ )
747
+
748
+ def slice(self, indices: torch.Tensor | slice) -> Self:
749
+ return self.__class__(
750
+ n_channels=self.n_channels,
751
+ neighborhood_ids=self.b.neighborhood_ids[indices],
752
+ neighborhoods=self.b.neighborhoods,
753
+ device=self.b.neighborhoods.device,
754
+ name=self.name,
755
+ )
756
+
757
+ def has_feature_cache(self):
758
+ return hasattr(self, "_features_valid")
759
+
760
+ def valid_mask(self, id):
761
+ return self._masks[self._mask_slices[id]] # type: ignore
762
+
763
+ def neighborhood_channels(self, id):
764
+ nhc = self.b.neighborhoods[id]
765
+ return nhc[nhc < self.n_channels]
766
+
767
+ def missing_channels(self, id):
768
+ return self.b.chans_arange[self.b.indicators[:, id] == 0]
769
+
770
+ def neighborhood_members(self, id):
771
+ return self._neighborhood_members[self.neighborhood_members_slices[id]]
772
+
773
+ def neighborhood_features(
774
+ self, id, batch_start=None, batch_size=None, batch_buffer=None
775
+ ):
776
+ f = self._features_valid[id]
777
+ if batch_start is not None:
778
+ f = f[batch_start : batch_start + batch_size]
779
+ if batch_buffer is not None:
780
+ batch_buffer[: len(f)] = f
781
+ return batch_buffer[: len(f)]
782
+ else:
783
+ return f
784
+
785
+ def subset_neighborhoods(self, channels, min_coverage=1.0, batch_size=None):
786
+ """Return info on neighborhoods which cover the channel set well enough
787
+
788
+ Define coverage for a neighborhood and a channel group as the intersection
789
+ size divided by the neighborhood's size.
790
+
791
+ Returns
792
+ -------
793
+ neighborhood_info : list of tuples
794
+ Each entry is, in order,
795
+ - neighborhood id
796
+ - neighborhood channels array
797
+ - neighborhood member indices
798
+ - optional batch start
799
+ representing a batch of spikes living on that neighborhood.
800
+ n_spikes : int
801
+ The total number of spikes in the neighborhood.
802
+ """
803
+ inds = self.b.indicators[channels]
804
+ coverage = inds.sum(0) / self.b.channel_counts
805
+ (covered_ids,) = torch.nonzero(coverage >= min_coverage, as_tuple=True)
806
+ n_spikes = self.b.popcounts[covered_ids].sum()
807
+
808
+ neighborhood_info = []
809
+ for j in covered_ids:
810
+ jneighb = self.b.neighborhoods[j]
811
+ jmems = self.neighborhood_members(j)
812
+ if batch_size is None or len(jmems) < batch_size:
813
+ neighborhood_info.append((j, jneighb, jmems, None))
814
+ else:
815
+ for bs in range(0, len(jmems), batch_size):
816
+ mem_batch = jmems[bs : bs + batch_size]
817
+ neighborhood_info.append((j, jneighb, mem_batch, bs))
818
+
819
+ return covered_ids, neighborhood_info, n_spikes
820
+
821
+ def spike_neighborhoods(
822
+ self, channels, neighborhood_ids=None, spike_indices=None, min_coverage=1.0
823
+ ):
824
+ """Like subset_neighborhoods, but for an already chosen collection of spikes
825
+
826
+ This is used when subsetting log likelihood calculations.
827
+ In this case, the returned neighborhood_member_indices keys are relative:
828
+ spike_indices[neighborhood_member_indices] are the actual indices.
829
+ """
830
+ if neighborhood_ids is None:
831
+ assert spike_indices is not None
832
+ neighborhood_ids = self.b.neighborhood_ids[spike_indices]
833
+ assert neighborhood_ids is not None
834
+
835
+ covered_ids = torch.unique(neighborhood_ids)
836
+ if min_coverage:
837
+ covered_ids = covered_ids.to(self.indicators.device)
838
+ inds = self.b.indicators[channels][:, covered_ids]
839
+ coverage = inds.sum(0) / self.b.channel_counts[covered_ids]
840
+ covered = coverage >= min_coverage
841
+ covered_ids = covered_ids[covered].cpu()
842
+ neighborhood_ids = neighborhood_ids.cpu()
843
+
844
+ neighborhood_info = [
845
+ (
846
+ j,
847
+ self.b.neighborhoods[j],
848
+ *(neighborhood_ids == j).nonzero(as_tuple=True),
849
+ None,
850
+ )
851
+ for j in covered_ids
852
+ ]
853
+ n_spikes = self.b.popcounts[covered_ids].sum()
854
+ return neighborhood_info, n_spikes
855
+
856
+ def adjacency(self, overlap=0.5):
857
+ overlaps = self.b.indicators.T @ self.b.indicators
858
+ assert overlaps.shape == (self.n_neighborhoods, self.n_neighborhoods)
859
+ counts = self.b.indicators.sum(0)
860
+ overlaps /= torch.minimum(counts[:, None], counts)
861
+ return (overlaps >= overlap - 1e-5).to(torch.float)
862
+
863
+ def partial_order(self):
864
+ """ret[i, j] == 1 iff neighb j subset neighb i"""
865
+ inds = self.b.indicators.T # nneighb x nc
866
+ po = (inds[:, None, :] >= inds[None, :, :]).all(2)
867
+ assert po.shape == (self.n_neighborhoods, self.n_neighborhoods)
868
+ return po
869
+
870
+
871
+ # -- helpers
872
+
873
+
874
+ def occupied_chans(
875
+ spike_data,
876
+ n_channels,
877
+ neighborhood_ids=None,
878
+ neighborhoods=None,
879
+ fuzz=0,
880
+ weights=None,
881
+ ):
882
+ if spike_data.neighborhood_ids is None:
883
+ chans = torch.unique(spike_data.channels)
884
+ return chans[chans < n_channels]
885
+ assert neighborhoods is not None
886
+ if neighborhood_ids is None:
887
+ neighborhood_ids = spike_data.neighborhood_ids
888
+ ids, inverse = torch.unique(neighborhood_ids, return_inverse=True)
889
+ if weights is None:
890
+ weights = torch.ones(inverse.shape, device=ids.device)
891
+ id_counts = torch.zeros(ids.shape, device=ids.device)
892
+ spiketorch.add_at_(id_counts, inverse, weights.to(ids.device))
893
+
894
+ chans0 = neighborhoods.neighborhoods[ids]
895
+ chans, inverse = torch.unique(chans0, return_inverse=True)
896
+ counts = torch.zeros(chans.shape, device=chans.device)
897
+ spiketorch.add_at_(
898
+ counts,
899
+ inverse.view(-1).to(counts.device),
900
+ id_counts[:, None].broadcast_to(chans0.shape).reshape(-1).to(counts.device),
901
+ )
902
+
903
+ counts = counts[chans < n_channels]
904
+ chans = chans[chans < n_channels]
905
+ for _ in range(fuzz):
906
+ chans = neighborhoods.channel_index[chans]
907
+ chans = torch.unique(chans)
908
+ chans = chans[chans < n_channels]
909
+
910
+ return chans, counts
911
+
912
+
913
+ def interp_to_chans(
914
+ spike_data,
915
+ channels,
916
+ prgeom,
917
+ params,
918
+ batch_size=256,
919
+ ):
920
+ source_pos = prgeom[spike_data.channels]
921
+ target_pos = prgeom[channels]
922
+ shape = len(spike_data), *target_pos.shape
923
+ output_shape = *spike_data.features.shape[:2], channels.numel()
924
+ output = spike_data.features.new_empty(output_shape)
925
+ if not channels.numel():
926
+ return output
927
+ for batch_start in range(0, len(spike_data), batch_size):
928
+ batch_end = min(batch_start + batch_size, len(spike_data))
929
+ sl = slice(batch_start, batch_end)
930
+ shape = batch_end - batch_start, *target_pos.shape
931
+ target_pos_ = target_pos[None].broadcast_to(shape)
932
+ interpolation_util.kernel_interpolate(
933
+ spike_data.features[sl],
934
+ source_pos[sl],
935
+ target_pos_,
936
+ params=params,
937
+ allow_destroy=False,
938
+ out=output[sl],
939
+ )
940
+ return output
941
+
942
+
943
+ def pad_to_chans(
944
+ spike_data, channels, n_channels, weights=None, target_padded=None, pad_value=0.0
945
+ ):
946
+ n, r, c = spike_data.features.shape
947
+ c_targ = channels.numel()
948
+
949
+ # determine channels to write to
950
+ reindexer = get_channel_reindexer(channels, n_channels)
951
+ target_ixs = reindexer[spike_data.channels].to(spike_data.features.device)
952
+
953
+ # scatter data
954
+ if target_padded is None:
955
+ if pad_value == 0.0:
956
+ target_padded = spike_data.features.new_zeros(n, r, c_targ + 1)
957
+ else:
958
+ target_padded = spike_data.features.new_full((n, r, c_targ + 1), pad_value)
959
+ scatter_ixs = target_ixs.unsqueeze(1).broadcast_to((n, r, c))
960
+ target_padded.scatter_(src=spike_data.features, dim=2, index=scatter_ixs)
961
+ target = target_padded[..., :-1]
962
+ if weights is None:
963
+ return target, None
964
+
965
+ # same for weights, if supplied
966
+ assert weights.shape == (n,)
967
+ weights_padded = weights.new_zeros((n, c_targ + 1))
968
+ weights = weights.unsqueeze(1).broadcast_to((n, c))
969
+ weights_padded.scatter_(src=weights, dim=1, index=target_ixs)
970
+ weights = weights_padded[:, :-1]
971
+ return target, weights
972
+
973
+
974
+ def get_channel_reindexer(channels, n_channels):
975
+ """
976
+ Arguments
977
+ ---------
978
+ channels : Tensor
979
+ Shape (n_chans_subset,)
980
+ n_channels : int
981
+
982
+ Returns
983
+ -------
984
+ reindexer : Tensor
985
+ Shape (n_channels + 1,)
986
+ reindexer[i] is the index of i in channels, if present.
987
+ Otherwise, it is n_chans_subset.
988
+ And the last entry is, of course, n_chans_subset.
989
+ """
990
+ reindexer = channels.new_full((n_channels + 1,), channels.numel())
991
+ (rel_ixs,) = torch.nonzero(channels < n_channels, as_tuple=True)
992
+ reindexer[channels[rel_ixs]] = rel_ixs
993
+ return reindexer