DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,993 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Literal, Self, Sequence
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from ...transform.temporal_pca import TemporalPCAFeaturizer
|
|
9
|
+
from ...util import drift_util, interpolation_util, spiketorch
|
|
10
|
+
from ...util.internal_config import (
|
|
11
|
+
InterpolationParams,
|
|
12
|
+
default_interpolation_params,
|
|
13
|
+
default_extrapolation_params,
|
|
14
|
+
)
|
|
15
|
+
from ...util.data_util import DARTsortSorting, get_tpca
|
|
16
|
+
from ...util.job_util import get_global_computation_config
|
|
17
|
+
from ...util.torch_util import BModule
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# -- main class
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class StableSpikeDataset(torch.nn.Module):
|
|
24
|
+
"""
|
|
25
|
+
## On SpikeNeighborhoods
|
|
26
|
+
|
|
27
|
+
Recall, we store spikes on two kinds of channel subsets: the larger
|
|
28
|
+
"extract" and the smaller "core". Extract is used for fitting (M step)
|
|
29
|
+
and core for likelihood computation (E step).
|
|
30
|
+
|
|
31
|
+
During fitting, each unit determines some set of channels that it's
|
|
32
|
+
"active" on. At E time, any spike whose core neighborhood is contained
|
|
33
|
+
in those channels gets a likelihood computed (the assumption being that
|
|
34
|
+
it would otherwise have smaller-than-noise likelihood under that unit
|
|
35
|
+
due to determinant).
|
|
36
|
+
|
|
37
|
+
It is not cheap to compute likelihoods on spikes from arbitrary channels
|
|
38
|
+
willy nilly. It's better to compute likelihoods for all spikes living
|
|
39
|
+
on the same exatch channel set at once. So, we cache info about what
|
|
40
|
+
neighborhood each spike belongs to, and ways to reverse-lookup spikes
|
|
41
|
+
by neighborhood. We even store the core spike features by neighborhood
|
|
42
|
+
to avoid doing a million index_selects from large arrays.
|
|
43
|
+
|
|
44
|
+
## On splits
|
|
45
|
+
|
|
46
|
+
We make use of at least two splits during inference: train and val.
|
|
47
|
+
The train split is the one whose extract features we use during the M
|
|
48
|
+
step, and whose core features we use during the E step. The val split
|
|
49
|
+
is just used for model comparisons during split/merge.
|
|
50
|
+
|
|
51
|
+
I say at least two because if the number of spikes is large, we will
|
|
52
|
+
choose to completely ignore a random subset of spikes. The name of that
|
|
53
|
+
split doesn't matter because it's not really used, but let's call it the
|
|
54
|
+
"ignore" split.
|
|
55
|
+
|
|
56
|
+
After fitting a model using the train and val sets, we can still
|
|
57
|
+
compute likelihoods for all spikes, even the ignored ones.
|
|
58
|
+
|
|
59
|
+
This means that:
|
|
60
|
+
- For "extract" spikes, we only need to load the "train" split and
|
|
61
|
+
make a SpikeNeighborhoods structure for the train split.
|
|
62
|
+
- For "core" spikes, we need to load all of the spikes, and we need
|
|
63
|
+
to cache SpikeNeighborhoods for train, train+val, and all 3.
|
|
64
|
+
|
|
65
|
+
## How are these used from the SpikeMixtureModel?
|
|
66
|
+
|
|
67
|
+
In the M step, indices for fitting are chosen from the train split.
|
|
68
|
+
|
|
69
|
+
The E step then also only needs to run on the train split most of the
|
|
70
|
+
time. There is a `split='train'` flag for `log_likelihoods()` and for
|
|
71
|
+
`e_step()`. This works by dispatching to the split in question's
|
|
72
|
+
SpikeNeighborhoods structure. The returned sparse array of likelihoods
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
kept_indices: np.ndarray,
|
|
81
|
+
prgeom: torch.Tensor,
|
|
82
|
+
tpca: TemporalPCAFeaturizer,
|
|
83
|
+
extract_channels: torch.Tensor,
|
|
84
|
+
core_channels: torch.Tensor | None,
|
|
85
|
+
original_sorting: DARTsortSorting,
|
|
86
|
+
core_features: torch.Tensor | None,
|
|
87
|
+
train_extract_features: torch.Tensor,
|
|
88
|
+
features_on_device: bool = False,
|
|
89
|
+
split_names: Sequence[str] | None = None,
|
|
90
|
+
split_mask: torch.Tensor | None = None,
|
|
91
|
+
core_radius: float | Literal["extract"] | None = 35.0,
|
|
92
|
+
interp_params: InterpolationParams = default_interpolation_params,
|
|
93
|
+
extrap_params: InterpolationParams = default_extrapolation_params,
|
|
94
|
+
extract_neighborhoods=None,
|
|
95
|
+
extract_neighborhood_ids=None,
|
|
96
|
+
core_neighborhoods=None,
|
|
97
|
+
core_neighborhood_ids=None,
|
|
98
|
+
device=None,
|
|
99
|
+
_core_feature_splits=("train", "kept"),
|
|
100
|
+
):
|
|
101
|
+
"""Motion-corrected spike data on the registered probe"""
|
|
102
|
+
super().__init__()
|
|
103
|
+
|
|
104
|
+
if device is None:
|
|
105
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
106
|
+
device = torch.device(device)
|
|
107
|
+
self.original_sorting = original_sorting
|
|
108
|
+
|
|
109
|
+
self.features_on_device = features_on_device
|
|
110
|
+
|
|
111
|
+
# data shapes
|
|
112
|
+
self.rank = tpca.rank
|
|
113
|
+
self.n_channels = prgeom.shape[0] - 1
|
|
114
|
+
self.n_channels_extract = extract_channels.shape[1]
|
|
115
|
+
self.core_is_extract = core_radius == "extract"
|
|
116
|
+
if core_radius is not None:
|
|
117
|
+
assert core_channels is not None
|
|
118
|
+
self.n_channels_core = core_channels.shape[1]
|
|
119
|
+
self.n_spikes = len(original_sorting)
|
|
120
|
+
# train is modified below if there is a train split.
|
|
121
|
+
self.n_spikes_train = self.n_spikes_kept = len(kept_indices)
|
|
122
|
+
self.core_radius = core_radius
|
|
123
|
+
# assert extrap_method != "kriging"
|
|
124
|
+
self.interp_params = interp_params.normalize()
|
|
125
|
+
self.extrap_params = extrap_params.normalize()
|
|
126
|
+
|
|
127
|
+
# spike collection indices
|
|
128
|
+
self.kept_indices = torch.from_numpy(kept_indices)
|
|
129
|
+
# split indices are within kept_indices
|
|
130
|
+
self.has_splits = split_names is not None
|
|
131
|
+
self.split_mask = split_mask
|
|
132
|
+
self.split_indices = dict(full=slice(None), kept=self.kept_indices)
|
|
133
|
+
if self.has_splits:
|
|
134
|
+
assert split_names is not None
|
|
135
|
+
self.split_indices.update(
|
|
136
|
+
(k, (split_mask == j).nonzero().view(-1))
|
|
137
|
+
for j, k in enumerate(split_names)
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# pca module, for reconstructing wfs for vis
|
|
141
|
+
self.tpca = tpca
|
|
142
|
+
|
|
143
|
+
# neighborhoods module, for querying spikes by channel group
|
|
144
|
+
self.not_train_indices = torch.tensor([], dtype=torch.long)
|
|
145
|
+
if self.has_splits and "train" in self.split_indices:
|
|
146
|
+
train_ixs = self.split_indices["train"]
|
|
147
|
+
self.n_spikes_train = len(train_ixs) # type: ignore
|
|
148
|
+
train_enids = None
|
|
149
|
+
if extract_neighborhood_ids is not None:
|
|
150
|
+
train_enids = extract_neighborhood_ids[train_ixs]
|
|
151
|
+
self._train_extract_neighborhoods = SpikeNeighborhoods.from_channels(
|
|
152
|
+
extract_channels[train_ixs],
|
|
153
|
+
n_channels=self.n_channels,
|
|
154
|
+
neighborhood_ids=train_enids,
|
|
155
|
+
neighborhoods=extract_neighborhoods,
|
|
156
|
+
deduplicate=not self.core_is_extract,
|
|
157
|
+
device=device,
|
|
158
|
+
name="extract",
|
|
159
|
+
)
|
|
160
|
+
self._train_extract_channels = extract_channels.cpu()[train_ixs]
|
|
161
|
+
assert torch.is_tensor(train_ixs)
|
|
162
|
+
self.not_train_indices = torch.asarray(
|
|
163
|
+
np.setdiff1d(np.arange(self.n_spikes), train_ixs),
|
|
164
|
+
dtype=torch.long,
|
|
165
|
+
)
|
|
166
|
+
else:
|
|
167
|
+
train_ixs = None
|
|
168
|
+
|
|
169
|
+
if core_radius is not None:
|
|
170
|
+
_core_neighborhoods = {
|
|
171
|
+
f"key_{k}": SpikeNeighborhoods.from_channels(
|
|
172
|
+
channels=core_channels[ix], # type: ignore
|
|
173
|
+
n_channels=self.n_channels,
|
|
174
|
+
neighborhood_ids=(
|
|
175
|
+
None
|
|
176
|
+
if core_neighborhood_ids is None
|
|
177
|
+
else core_neighborhood_ids[ix]
|
|
178
|
+
),
|
|
179
|
+
neighborhoods=core_neighborhoods,
|
|
180
|
+
features=core_features[ix] if k in _core_feature_splits else None, # type: ignore
|
|
181
|
+
device=device,
|
|
182
|
+
name=k,
|
|
183
|
+
)
|
|
184
|
+
for k, ix in self.split_indices.items()
|
|
185
|
+
if (not self.core_is_extract or k != "train")
|
|
186
|
+
}
|
|
187
|
+
if self.core_is_extract and "train" in self.split_indices:
|
|
188
|
+
assert torch.is_tensor(train_ixs)
|
|
189
|
+
assert core_channels is not None
|
|
190
|
+
assert torch.equal(
|
|
191
|
+
core_channels[train_ixs], extract_channels[train_ixs]
|
|
192
|
+
)
|
|
193
|
+
_core_neighborhoods["train"] = self._train_extract_neighborhoods
|
|
194
|
+
self._core_neighborhoods = torch.nn.ModuleDict(_core_neighborhoods)
|
|
195
|
+
self.core_channels = core_channels.cpu() # type: ignore
|
|
196
|
+
|
|
197
|
+
# channel neighborhoods and features
|
|
198
|
+
# if not self.features_on_device, .spike_data() will .to(self.device)
|
|
199
|
+
times_s = torch.asarray(self.original_sorting.times_seconds) # type: ignore[reportAttributeAccessIssue]
|
|
200
|
+
self.core_features = core_features
|
|
201
|
+
if self.features_on_device:
|
|
202
|
+
self.register_buffer("_train_extract_features", train_extract_features)
|
|
203
|
+
self.register_buffer("times_seconds", times_s)
|
|
204
|
+
else:
|
|
205
|
+
self._train_extract_features = train_extract_features
|
|
206
|
+
self.times_seconds = times_s
|
|
207
|
+
|
|
208
|
+
# always on device
|
|
209
|
+
self.register_buffer("prgeom", prgeom)
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def device(self) -> torch.device:
|
|
213
|
+
return self.prgeom.device # type: ignore
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def dtype(self):
|
|
217
|
+
return self._train_extract_features.dtype
|
|
218
|
+
|
|
219
|
+
@classmethod
|
|
220
|
+
def from_config(
|
|
221
|
+
cls,
|
|
222
|
+
sorting,
|
|
223
|
+
refinement_cfg,
|
|
224
|
+
motion_est=None,
|
|
225
|
+
computation_cfg=None,
|
|
226
|
+
_core_feature_splits=("train", "kept"),
|
|
227
|
+
):
|
|
228
|
+
if computation_cfg is None:
|
|
229
|
+
computation_cfg = get_global_computation_config()
|
|
230
|
+
return cls.from_sorting(
|
|
231
|
+
sorting,
|
|
232
|
+
motion_est=motion_est,
|
|
233
|
+
core_radius=refinement_cfg.core_radius,
|
|
234
|
+
max_n_spikes=refinement_cfg.max_n_spikes,
|
|
235
|
+
interp_params=refinement_cfg.interp_params.normalize(),
|
|
236
|
+
split_proportions=(
|
|
237
|
+
1.0 - refinement_cfg.val_proportion,
|
|
238
|
+
refinement_cfg.val_proportion,
|
|
239
|
+
),
|
|
240
|
+
device=computation_cfg.actual_device(),
|
|
241
|
+
_core_feature_splits=_core_feature_splits,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
@classmethod
|
|
245
|
+
def from_sorting(
|
|
246
|
+
cls,
|
|
247
|
+
sorting,
|
|
248
|
+
motion_est=None,
|
|
249
|
+
core_radius: float | Literal["extract"] | None = 35.0,
|
|
250
|
+
max_n_spikes: float | int = np.inf,
|
|
251
|
+
kept_indices: np.ndarray | None = None,
|
|
252
|
+
discard_triaged: bool = False,
|
|
253
|
+
interp_params: InterpolationParams = default_interpolation_params,
|
|
254
|
+
features_dataset_name="collisioncleaned_tpca_features",
|
|
255
|
+
motion_depth_mode="channel",
|
|
256
|
+
split_names=("train", "val"),
|
|
257
|
+
split_proportions=(0.75, 0.25),
|
|
258
|
+
show_progress=True,
|
|
259
|
+
store_on_device=False,
|
|
260
|
+
workers=-1,
|
|
261
|
+
device=None,
|
|
262
|
+
feature_rank: int = 8,
|
|
263
|
+
random_seed: int | np.random.Generator = 0,
|
|
264
|
+
_core_feature_splits=("train", "kept"),
|
|
265
|
+
):
|
|
266
|
+
if device is None:
|
|
267
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
268
|
+
device = torch.device(device)
|
|
269
|
+
|
|
270
|
+
# load information not stored directly on the sorting
|
|
271
|
+
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
272
|
+
geom = h5["geom"][:] # type: ignore
|
|
273
|
+
extract_channel_index: np.ndarray = h5["channel_index"][:] # type: ignore
|
|
274
|
+
assert np.all(np.diff(extract_channel_index) >= 0)
|
|
275
|
+
if motion_est is None:
|
|
276
|
+
registered_geom = geom
|
|
277
|
+
else:
|
|
278
|
+
registered_geom = drift_util.registered_geometry(
|
|
279
|
+
geom, motion_est=motion_est
|
|
280
|
+
)
|
|
281
|
+
prgeom = interpolation_util.pad_geom(registered_geom)
|
|
282
|
+
|
|
283
|
+
res = drift_util.get_shift_info(
|
|
284
|
+
sorting, motion_est, geom, motion_depth_mode
|
|
285
|
+
)
|
|
286
|
+
channels, shifts, n_pitches_shift = res
|
|
287
|
+
|
|
288
|
+
# determine channel occupancy of core/extract stable features
|
|
289
|
+
res = drift_util.get_stable_channels(
|
|
290
|
+
geom,
|
|
291
|
+
channels,
|
|
292
|
+
extract_channel_index,
|
|
293
|
+
registered_geom,
|
|
294
|
+
n_pitches_shift,
|
|
295
|
+
core_radius=core_radius,
|
|
296
|
+
workers=workers,
|
|
297
|
+
device=device,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
extract_channels, extract_neighborhoods, extract_neighborhood_ids = res[:3]
|
|
301
|
+
core_channels, core_neighborhoods, core_neighborhood_ids = res[3:]
|
|
302
|
+
if core_radius is not None:
|
|
303
|
+
assert core_channels is not None
|
|
304
|
+
assert core_neighborhoods is not None
|
|
305
|
+
assert core_neighborhood_ids is not None
|
|
306
|
+
|
|
307
|
+
if core_radius == "extract":
|
|
308
|
+
assert core_neighborhoods is not None
|
|
309
|
+
assert core_neighborhood_ids is not None
|
|
310
|
+
assert np.array_equal(extract_neighborhoods, core_neighborhoods)
|
|
311
|
+
assert np.array_equal(extract_neighborhood_ids, core_neighborhood_ids)
|
|
312
|
+
|
|
313
|
+
rg = np.random.default_rng(random_seed)
|
|
314
|
+
|
|
315
|
+
assert len(split_names) >= 1
|
|
316
|
+
assert "train" in split_names
|
|
317
|
+
if kept_indices is not None:
|
|
318
|
+
pass
|
|
319
|
+
elif discard_triaged:
|
|
320
|
+
kept_indices = np.flatnonzero(sorting.labels >= 0)
|
|
321
|
+
elif len(sorting) <= max_n_spikes:
|
|
322
|
+
kept_indices = np.arange(len(sorting))
|
|
323
|
+
else:
|
|
324
|
+
assert isinstance(max_n_spikes, int)
|
|
325
|
+
kept_indices = rg.choice(len(sorting), size=max_n_spikes, replace=False)
|
|
326
|
+
kept_indices.sort()
|
|
327
|
+
|
|
328
|
+
split_mask = torch.full((len(sorting),), -1, dtype=torch.int8)
|
|
329
|
+
# train set initialized to everything
|
|
330
|
+
assert split_names[0] == "train"
|
|
331
|
+
split_mask[kept_indices] = 0
|
|
332
|
+
# val takes from uncovered
|
|
333
|
+
if "val" in split_names:
|
|
334
|
+
assert "val" == split_names[1]
|
|
335
|
+
assert split_names == ("train", "val")
|
|
336
|
+
n_val = int(np.ceil(split_proportions[1] * len(kept_indices)))
|
|
337
|
+
val_candidates = rg.choice(kept_indices, size=n_val, replace=False)
|
|
338
|
+
val_candidates.sort()
|
|
339
|
+
split_mask[val_candidates] = 1
|
|
340
|
+
else:
|
|
341
|
+
assert split_names == ("train",)
|
|
342
|
+
train_mask = (split_mask == 0).numpy()
|
|
343
|
+
|
|
344
|
+
# for all spikes (not just kept), the ID of its shift/chan combo.
|
|
345
|
+
# this determines its channel neighborhood under any registered index.
|
|
346
|
+
# we also get the total counts in each combo, and the indices of
|
|
347
|
+
# the first spikes in each combo, allowing neighbs to be reconstructed
|
|
348
|
+
# as needed
|
|
349
|
+
extract_channels = torch.from_numpy(extract_channels)
|
|
350
|
+
if store_on_device:
|
|
351
|
+
extract_channels = extract_channels.to(device)
|
|
352
|
+
|
|
353
|
+
if core_radius is not None:
|
|
354
|
+
core_channels = torch.from_numpy(core_channels)
|
|
355
|
+
if store_on_device:
|
|
356
|
+
core_channels = core_channels.to(device)
|
|
357
|
+
|
|
358
|
+
# stabilize features with interpolation
|
|
359
|
+
# only load kept extract spikes
|
|
360
|
+
train_extract_features = interpolation_util.interpolate_by_chunk(
|
|
361
|
+
train_mask,
|
|
362
|
+
h5[features_dataset_name],
|
|
363
|
+
geom,
|
|
364
|
+
extract_channel_index,
|
|
365
|
+
sorting.channels[train_mask],
|
|
366
|
+
shifts[train_mask],
|
|
367
|
+
registered_geom,
|
|
368
|
+
extract_channels[train_mask],
|
|
369
|
+
trim_to_rank=feature_rank,
|
|
370
|
+
params=interp_params,
|
|
371
|
+
device=device,
|
|
372
|
+
store_on_device=store_on_device,
|
|
373
|
+
show_progress=show_progress,
|
|
374
|
+
)
|
|
375
|
+
# always load all core spikes
|
|
376
|
+
core_features = None
|
|
377
|
+
if core_radius is not None:
|
|
378
|
+
core_features = interpolation_util.interpolate_by_chunk(
|
|
379
|
+
np.ones(len(sorting), dtype=np.bool_),
|
|
380
|
+
h5[features_dataset_name],
|
|
381
|
+
geom,
|
|
382
|
+
extract_channel_index,
|
|
383
|
+
sorting.channels,
|
|
384
|
+
shifts,
|
|
385
|
+
registered_geom,
|
|
386
|
+
core_channels,
|
|
387
|
+
trim_to_rank=feature_rank,
|
|
388
|
+
params=interp_params,
|
|
389
|
+
device=device,
|
|
390
|
+
store_on_device=store_on_device,
|
|
391
|
+
show_progress=show_progress,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
# load temporal PCA
|
|
395
|
+
tpca = get_tpca(sorting)
|
|
396
|
+
|
|
397
|
+
if core_radius == "extract":
|
|
398
|
+
core_radius = -np.inf
|
|
399
|
+
for j in range(len(extract_channel_index)):
|
|
400
|
+
chans = extract_channel_index[j]
|
|
401
|
+
assert isinstance(geom, np.ndarray)
|
|
402
|
+
chans = chans[chans < len(geom)]
|
|
403
|
+
tmp = np.subtract(geom[chans], geom[j])
|
|
404
|
+
np.square(tmp, out=tmp)
|
|
405
|
+
max_dist = tmp.sum(axis=1).max()
|
|
406
|
+
core_radius = max(core_radius, np.sqrt(max_dist))
|
|
407
|
+
|
|
408
|
+
self = cls(
|
|
409
|
+
kept_indices=kept_indices,
|
|
410
|
+
prgeom=prgeom,
|
|
411
|
+
tpca=tpca, # type: ignore
|
|
412
|
+
extract_channels=extract_channels,
|
|
413
|
+
extract_neighborhoods=extract_neighborhoods,
|
|
414
|
+
extract_neighborhood_ids=extract_neighborhood_ids,
|
|
415
|
+
core_channels=core_channels, # type: ignore
|
|
416
|
+
core_neighborhoods=core_neighborhoods,
|
|
417
|
+
core_neighborhood_ids=core_neighborhood_ids,
|
|
418
|
+
original_sorting=sorting,
|
|
419
|
+
core_features=core_features,
|
|
420
|
+
train_extract_features=train_extract_features,
|
|
421
|
+
features_on_device=store_on_device,
|
|
422
|
+
split_names=split_names,
|
|
423
|
+
split_mask=split_mask,
|
|
424
|
+
core_radius=core_radius,
|
|
425
|
+
device=device,
|
|
426
|
+
interp_params=interp_params,
|
|
427
|
+
_core_feature_splits=_core_feature_splits,
|
|
428
|
+
)
|
|
429
|
+
self.to(device)
|
|
430
|
+
return self
|
|
431
|
+
|
|
432
|
+
def spike_data(
|
|
433
|
+
self,
|
|
434
|
+
indices: torch.Tensor,
|
|
435
|
+
split_indices: torch.Tensor | None = None,
|
|
436
|
+
neighborhood: str = "extract",
|
|
437
|
+
with_channels=True,
|
|
438
|
+
with_reconstructions: bool = False,
|
|
439
|
+
with_neighborhood_ids: bool = False,
|
|
440
|
+
split: str | None = "train",
|
|
441
|
+
feature_buffer: torch.Tensor | None = None,
|
|
442
|
+
) -> "SpikeFeatures":
|
|
443
|
+
withbuf = feature_buffer is not None
|
|
444
|
+
non_blocking = False
|
|
445
|
+
channels = neighborhood_ids = None
|
|
446
|
+
|
|
447
|
+
if neighborhood == "extract":
|
|
448
|
+
assert split == "train"
|
|
449
|
+
assert split_indices is not None
|
|
450
|
+
|
|
451
|
+
if withbuf:
|
|
452
|
+
features = feature_buffer[: len(indices)]
|
|
453
|
+
non_blocking = features.is_pinned()
|
|
454
|
+
torch.index_select(
|
|
455
|
+
self._train_extract_features, 0, split_indices, out=features
|
|
456
|
+
)
|
|
457
|
+
else:
|
|
458
|
+
features = self._train_extract_features[split_indices]
|
|
459
|
+
if with_channels:
|
|
460
|
+
channels = self._train_extract_channels[split_indices]
|
|
461
|
+
if with_neighborhood_ids:
|
|
462
|
+
neighborhood_ids = self._train_extract_neighborhoods.b.neighborhood_ids[
|
|
463
|
+
split_indices
|
|
464
|
+
]
|
|
465
|
+
elif neighborhood == "core":
|
|
466
|
+
if withbuf:
|
|
467
|
+
features = feature_buffer[: len(indices)]
|
|
468
|
+
non_blocking = features.is_pinned()
|
|
469
|
+
torch.index_select(self.core_features, 0, indices, out=features) # type: ignore
|
|
470
|
+
else:
|
|
471
|
+
features = self.core_features[indices] # type: ignore
|
|
472
|
+
if with_channels:
|
|
473
|
+
channels = self.core_channels[indices]
|
|
474
|
+
if with_neighborhood_ids:
|
|
475
|
+
_, core_neighborhoods = self.neighborhoods(split="full")
|
|
476
|
+
neighborhood_ids = core_neighborhoods.b.neighborhood_ids[indices]
|
|
477
|
+
else:
|
|
478
|
+
assert False
|
|
479
|
+
|
|
480
|
+
if not self.features_on_device:
|
|
481
|
+
features = features.to(self.device, non_blocking=non_blocking)
|
|
482
|
+
if with_channels:
|
|
483
|
+
channels = channels
|
|
484
|
+
|
|
485
|
+
waveforms = None
|
|
486
|
+
if with_reconstructions:
|
|
487
|
+
n = len(features)
|
|
488
|
+
assert channels is not None
|
|
489
|
+
c = channels.shape[1]
|
|
490
|
+
waveforms = features.permute(0, 2, 1).reshape(n * c, -1)
|
|
491
|
+
waveforms = self.tpca._inverse_transform_in_probe(waveforms)
|
|
492
|
+
waveforms = waveforms.reshape(n, c, -1).permute(0, 2, 1)
|
|
493
|
+
|
|
494
|
+
return SpikeFeatures(
|
|
495
|
+
indices=indices,
|
|
496
|
+
split_indices=split_indices,
|
|
497
|
+
features=features,
|
|
498
|
+
channels=channels,
|
|
499
|
+
waveforms=waveforms,
|
|
500
|
+
neighborhood_ids=neighborhood_ids,
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
def neighborhoods(
|
|
504
|
+
self, neighborhood="core", split="train"
|
|
505
|
+
) -> tuple[torch.Tensor | slice, "SpikeNeighborhoods"]:
|
|
506
|
+
split_ixs = self.split_indices[split]
|
|
507
|
+
|
|
508
|
+
if neighborhood == "extract":
|
|
509
|
+
assert split == "train"
|
|
510
|
+
return split_ixs, self._train_extract_neighborhoods
|
|
511
|
+
|
|
512
|
+
return split_ixs, self._core_neighborhoods[f"key_{split}"] # type: ignore
|
|
513
|
+
|
|
514
|
+
def interp_to_chans(self, spike_data, channels):
|
|
515
|
+
return interp_to_chans(
|
|
516
|
+
spike_data, channels, self.prgeom, params=self.extrap_params
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
# -- utility classes
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
524
|
+
class SpikeFeatures:
|
|
525
|
+
"""A 'messenger class' to hold onto batches of spike features
|
|
526
|
+
|
|
527
|
+
Return type of StableSpikeDataset's spike_data method.
|
|
528
|
+
"""
|
|
529
|
+
|
|
530
|
+
# these are relative to the StableFeatures instance's subsampling (keepers)
|
|
531
|
+
indices: torch.Tensor
|
|
532
|
+
# n, r, c
|
|
533
|
+
features: torch.Tensor
|
|
534
|
+
# n, c
|
|
535
|
+
channels: torch.Tensor | None = None
|
|
536
|
+
# n, t, c
|
|
537
|
+
waveforms: torch.Tensor | None = None
|
|
538
|
+
# n
|
|
539
|
+
neighborhood_ids: torch.Tensor | None = None
|
|
540
|
+
split_indices: torch.Tensor | None = None
|
|
541
|
+
|
|
542
|
+
def __len__(self):
|
|
543
|
+
return len(self.features)
|
|
544
|
+
|
|
545
|
+
def __getitem__(self, ix):
|
|
546
|
+
"""Subset the spikes in this collection with [subset]."""
|
|
547
|
+
waveforms = channels = neighborhood_ids = split_indices = None
|
|
548
|
+
if self.channels is not None:
|
|
549
|
+
channels = self.channels[ix]
|
|
550
|
+
if self.waveforms is not None:
|
|
551
|
+
waveforms = self.waveforms[ix]
|
|
552
|
+
if self.neighborhood_ids is not None:
|
|
553
|
+
neighborhood_ids = self.neighborhood_ids[ix]
|
|
554
|
+
if self.split_indices is not None:
|
|
555
|
+
split_indices = self.split_indices[ix]
|
|
556
|
+
return self.__class__(
|
|
557
|
+
indices=self.indices[ix],
|
|
558
|
+
features=self.features[ix],
|
|
559
|
+
channels=channels,
|
|
560
|
+
waveforms=waveforms,
|
|
561
|
+
neighborhood_ids=neighborhood_ids,
|
|
562
|
+
split_indices=split_indices,
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
def __repr__(self):
|
|
566
|
+
indstr = f"indices.shape={self.indices.shape},"
|
|
567
|
+
featstr = f"features.shape={self.features.shape},"
|
|
568
|
+
chanstr = wfstr = idstr = ""
|
|
569
|
+
if self.channels is not None:
|
|
570
|
+
chanstr = f"channels.shape={self.channels.shape},"
|
|
571
|
+
if self.waveforms is not None:
|
|
572
|
+
wfstr = f"waveforms.shape={self.waveforms.shape},"
|
|
573
|
+
if self.neighborhood_ids is not None:
|
|
574
|
+
idstr = f"neighborhood_ids.shape={self.neighborhood_ids.shape},"
|
|
575
|
+
pstr = f"{indstr}{featstr}{chanstr}{wfstr}{idstr}"
|
|
576
|
+
return f"{self.__class__.__name__}({pstr.rstrip(',')})"
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
class SpikeNeighborhoods(BModule):
|
|
580
|
+
def __init__(
|
|
581
|
+
self,
|
|
582
|
+
n_channels: int,
|
|
583
|
+
neighborhood_ids,
|
|
584
|
+
neighborhoods,
|
|
585
|
+
features=None,
|
|
586
|
+
neighborhood_members=None,
|
|
587
|
+
device=None,
|
|
588
|
+
name=None,
|
|
589
|
+
):
|
|
590
|
+
"""SpikeNeighborhoods
|
|
591
|
+
|
|
592
|
+
Sparsely keep track of which channels each spike lives on. Used to query
|
|
593
|
+
which core sets are overlapped completely by unit channel neighborhoods.
|
|
594
|
+
|
|
595
|
+
Arguments
|
|
596
|
+
---------
|
|
597
|
+
neighborhood_ids : torch.Tensor
|
|
598
|
+
Size (n_spikes,), the neighborhood id for each spike
|
|
599
|
+
neighborhoods : list[torch.Tensor]
|
|
600
|
+
The channels in each neighborhood
|
|
601
|
+
neighborhood_members : list[torch.Tensor]
|
|
602
|
+
The indices of spikes in each neighborhood
|
|
603
|
+
"""
|
|
604
|
+
super().__init__()
|
|
605
|
+
self.name = name
|
|
606
|
+
self.n_channels = n_channels
|
|
607
|
+
self.register_buffer("neighborhood_ids", neighborhood_ids.long())
|
|
608
|
+
self.register_buffer("chans_arange", torch.arange(n_channels, dtype=torch.long))
|
|
609
|
+
self.register_buffer("neighborhoods", neighborhoods.long())
|
|
610
|
+
self.n_neighborhoods = len(neighborhoods)
|
|
611
|
+
|
|
612
|
+
# store neighborhoods as an indicator matrix
|
|
613
|
+
# also store nonzero-d masks
|
|
614
|
+
indicators = torch.zeros((n_channels, len(neighborhoods)), device=device)
|
|
615
|
+
masks = []
|
|
616
|
+
mask_slices = []
|
|
617
|
+
offset = 0
|
|
618
|
+
for j, nhood in enumerate(neighborhoods):
|
|
619
|
+
jvalid = nhood < n_channels
|
|
620
|
+
indicators[nhood[jvalid], j] = 1.0
|
|
621
|
+
(jvalid,) = jvalid.nonzero(as_tuple=True)
|
|
622
|
+
jvalid = jvalid.long()
|
|
623
|
+
njvalid = jvalid.numel()
|
|
624
|
+
assert njvalid
|
|
625
|
+
masks.append(jvalid)
|
|
626
|
+
mask_slices.append(slice(offset, offset + njvalid))
|
|
627
|
+
offset += njvalid
|
|
628
|
+
self.register_buffer("indicators", indicators)
|
|
629
|
+
self.register_buffer("channel_counts", indicators.sum(0))
|
|
630
|
+
self.register_buffer("_masks", torch.concatenate(masks, dim=0))
|
|
631
|
+
self._mask_slices = mask_slices
|
|
632
|
+
|
|
633
|
+
if neighborhood_members is None:
|
|
634
|
+
# cache lookups
|
|
635
|
+
neighborhood_members = []
|
|
636
|
+
for j in range(len(neighborhoods)):
|
|
637
|
+
(in_nhood,) = torch.nonzero(neighborhood_ids == j, as_tuple=True)
|
|
638
|
+
in_nhood = in_nhood.long()
|
|
639
|
+
neighborhood_members.append(in_nhood.cpu())
|
|
640
|
+
assert len(neighborhood_members) == self.n_neighborhoods
|
|
641
|
+
|
|
642
|
+
# it's a pain to store dicts with register_buffer, so store offsets
|
|
643
|
+
_neighborhood_members = torch.empty(
|
|
644
|
+
sum(v.numel() for v in neighborhood_members), dtype=torch.long
|
|
645
|
+
)
|
|
646
|
+
self.neighborhood_members_slices = []
|
|
647
|
+
neighborhood_member_offset = 0
|
|
648
|
+
neighborhood_popcounts = []
|
|
649
|
+
for j in range(len(neighborhoods)):
|
|
650
|
+
nhoodmemsz = neighborhood_members[j].numel()
|
|
651
|
+
nhoodmemsl = slice(
|
|
652
|
+
neighborhood_member_offset, neighborhood_member_offset + nhoodmemsz
|
|
653
|
+
)
|
|
654
|
+
_neighborhood_members[nhoodmemsl] = neighborhood_members[j]
|
|
655
|
+
self.neighborhood_members_slices.append(nhoodmemsl)
|
|
656
|
+
neighborhood_member_offset += nhoodmemsz
|
|
657
|
+
neighborhood_popcounts.append(nhoodmemsz)
|
|
658
|
+
# self.register_buffer("_neighborhood_members", _neighborhood_members)
|
|
659
|
+
# seems that indices want to live on cpu.
|
|
660
|
+
self._neighborhood_members = _neighborhood_members.cpu()
|
|
661
|
+
self.register_buffer("popcounts", torch.tensor(neighborhood_popcounts))
|
|
662
|
+
|
|
663
|
+
if features is not None:
|
|
664
|
+
_features_valid = []
|
|
665
|
+
for j in range(len(neighborhoods)):
|
|
666
|
+
f = features[self.neighborhood_members(j)]
|
|
667
|
+
f = f[..., self.valid_mask(j).to(f.device)]
|
|
668
|
+
if device is not None and device.type == "cuda":
|
|
669
|
+
f = f.pin_memory()
|
|
670
|
+
_features_valid.append(f)
|
|
671
|
+
self._features_valid = _features_valid
|
|
672
|
+
self.to(device=device)
|
|
673
|
+
|
|
674
|
+
@classmethod
|
|
675
|
+
def from_channels(
|
|
676
|
+
cls,
|
|
677
|
+
channels,
|
|
678
|
+
n_channels,
|
|
679
|
+
neighborhood_ids=None,
|
|
680
|
+
neighborhoods=None,
|
|
681
|
+
device=None,
|
|
682
|
+
deduplicate=False,
|
|
683
|
+
features=None,
|
|
684
|
+
name=None,
|
|
685
|
+
):
|
|
686
|
+
if neighborhood_ids is not None:
|
|
687
|
+
assert neighborhoods is not None
|
|
688
|
+
return cls.from_known_ids(
|
|
689
|
+
n_channels=n_channels,
|
|
690
|
+
neighborhood_ids=neighborhood_ids,
|
|
691
|
+
neighborhoods=neighborhoods,
|
|
692
|
+
device=device,
|
|
693
|
+
deduplicate=deduplicate,
|
|
694
|
+
features=features,
|
|
695
|
+
name=name,
|
|
696
|
+
)
|
|
697
|
+
if device is not None:
|
|
698
|
+
channels = channels.to(device)
|
|
699
|
+
neighborhoods, neighborhood_ids = torch.unique(
|
|
700
|
+
channels, dim=0, return_inverse=True
|
|
701
|
+
)
|
|
702
|
+
neighborhoods = neighborhoods.long()
|
|
703
|
+
neighborhood_ids = neighborhood_ids.long()
|
|
704
|
+
return cls(
|
|
705
|
+
n_channels=n_channels,
|
|
706
|
+
neighborhoods=neighborhoods,
|
|
707
|
+
neighborhood_ids=neighborhood_ids,
|
|
708
|
+
features=features,
|
|
709
|
+
device=channels.device,
|
|
710
|
+
name=name,
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
@classmethod
|
|
714
|
+
def from_known_ids(
|
|
715
|
+
cls,
|
|
716
|
+
*,
|
|
717
|
+
n_channels: int,
|
|
718
|
+
neighborhood_ids,
|
|
719
|
+
neighborhoods,
|
|
720
|
+
device=None,
|
|
721
|
+
deduplicate=False,
|
|
722
|
+
features=None,
|
|
723
|
+
name=None,
|
|
724
|
+
):
|
|
725
|
+
neighborhoods = torch.asarray(neighborhoods, dtype=torch.long)
|
|
726
|
+
neighborhood_ids = torch.asarray(neighborhood_ids, dtype=torch.long)
|
|
727
|
+
if device is not None:
|
|
728
|
+
neighborhoods = neighborhoods.to(device)
|
|
729
|
+
neighborhood_ids = neighborhood_ids.to(device)
|
|
730
|
+
if deduplicate:
|
|
731
|
+
neighborhoods, old2new = torch.unique(
|
|
732
|
+
neighborhoods, dim=0, return_inverse=True
|
|
733
|
+
)
|
|
734
|
+
neighborhood_ids = old2new[neighborhood_ids]
|
|
735
|
+
kept_ids, neighborhood_ids = torch.unique(
|
|
736
|
+
neighborhood_ids, return_inverse=True
|
|
737
|
+
)
|
|
738
|
+
neighborhoods = neighborhoods[kept_ids]
|
|
739
|
+
return cls(
|
|
740
|
+
n_channels=n_channels,
|
|
741
|
+
neighborhoods=neighborhoods,
|
|
742
|
+
neighborhood_ids=neighborhood_ids,
|
|
743
|
+
features=features,
|
|
744
|
+
device=device,
|
|
745
|
+
name=name,
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
def slice(self, indices: torch.Tensor | slice) -> Self:
|
|
749
|
+
return self.__class__(
|
|
750
|
+
n_channels=self.n_channels,
|
|
751
|
+
neighborhood_ids=self.b.neighborhood_ids[indices],
|
|
752
|
+
neighborhoods=self.b.neighborhoods,
|
|
753
|
+
device=self.b.neighborhoods.device,
|
|
754
|
+
name=self.name,
|
|
755
|
+
)
|
|
756
|
+
|
|
757
|
+
def has_feature_cache(self):
|
|
758
|
+
return hasattr(self, "_features_valid")
|
|
759
|
+
|
|
760
|
+
def valid_mask(self, id):
|
|
761
|
+
return self._masks[self._mask_slices[id]] # type: ignore
|
|
762
|
+
|
|
763
|
+
def neighborhood_channels(self, id):
|
|
764
|
+
nhc = self.b.neighborhoods[id]
|
|
765
|
+
return nhc[nhc < self.n_channels]
|
|
766
|
+
|
|
767
|
+
def missing_channels(self, id):
|
|
768
|
+
return self.b.chans_arange[self.b.indicators[:, id] == 0]
|
|
769
|
+
|
|
770
|
+
def neighborhood_members(self, id):
|
|
771
|
+
return self._neighborhood_members[self.neighborhood_members_slices[id]]
|
|
772
|
+
|
|
773
|
+
def neighborhood_features(
|
|
774
|
+
self, id, batch_start=None, batch_size=None, batch_buffer=None
|
|
775
|
+
):
|
|
776
|
+
f = self._features_valid[id]
|
|
777
|
+
if batch_start is not None:
|
|
778
|
+
f = f[batch_start : batch_start + batch_size]
|
|
779
|
+
if batch_buffer is not None:
|
|
780
|
+
batch_buffer[: len(f)] = f
|
|
781
|
+
return batch_buffer[: len(f)]
|
|
782
|
+
else:
|
|
783
|
+
return f
|
|
784
|
+
|
|
785
|
+
def subset_neighborhoods(self, channels, min_coverage=1.0, batch_size=None):
|
|
786
|
+
"""Return info on neighborhoods which cover the channel set well enough
|
|
787
|
+
|
|
788
|
+
Define coverage for a neighborhood and a channel group as the intersection
|
|
789
|
+
size divided by the neighborhood's size.
|
|
790
|
+
|
|
791
|
+
Returns
|
|
792
|
+
-------
|
|
793
|
+
neighborhood_info : list of tuples
|
|
794
|
+
Each entry is, in order,
|
|
795
|
+
- neighborhood id
|
|
796
|
+
- neighborhood channels array
|
|
797
|
+
- neighborhood member indices
|
|
798
|
+
- optional batch start
|
|
799
|
+
representing a batch of spikes living on that neighborhood.
|
|
800
|
+
n_spikes : int
|
|
801
|
+
The total number of spikes in the neighborhood.
|
|
802
|
+
"""
|
|
803
|
+
inds = self.b.indicators[channels]
|
|
804
|
+
coverage = inds.sum(0) / self.b.channel_counts
|
|
805
|
+
(covered_ids,) = torch.nonzero(coverage >= min_coverage, as_tuple=True)
|
|
806
|
+
n_spikes = self.b.popcounts[covered_ids].sum()
|
|
807
|
+
|
|
808
|
+
neighborhood_info = []
|
|
809
|
+
for j in covered_ids:
|
|
810
|
+
jneighb = self.b.neighborhoods[j]
|
|
811
|
+
jmems = self.neighborhood_members(j)
|
|
812
|
+
if batch_size is None or len(jmems) < batch_size:
|
|
813
|
+
neighborhood_info.append((j, jneighb, jmems, None))
|
|
814
|
+
else:
|
|
815
|
+
for bs in range(0, len(jmems), batch_size):
|
|
816
|
+
mem_batch = jmems[bs : bs + batch_size]
|
|
817
|
+
neighborhood_info.append((j, jneighb, mem_batch, bs))
|
|
818
|
+
|
|
819
|
+
return covered_ids, neighborhood_info, n_spikes
|
|
820
|
+
|
|
821
|
+
def spike_neighborhoods(
|
|
822
|
+
self, channels, neighborhood_ids=None, spike_indices=None, min_coverage=1.0
|
|
823
|
+
):
|
|
824
|
+
"""Like subset_neighborhoods, but for an already chosen collection of spikes
|
|
825
|
+
|
|
826
|
+
This is used when subsetting log likelihood calculations.
|
|
827
|
+
In this case, the returned neighborhood_member_indices keys are relative:
|
|
828
|
+
spike_indices[neighborhood_member_indices] are the actual indices.
|
|
829
|
+
"""
|
|
830
|
+
if neighborhood_ids is None:
|
|
831
|
+
assert spike_indices is not None
|
|
832
|
+
neighborhood_ids = self.b.neighborhood_ids[spike_indices]
|
|
833
|
+
assert neighborhood_ids is not None
|
|
834
|
+
|
|
835
|
+
covered_ids = torch.unique(neighborhood_ids)
|
|
836
|
+
if min_coverage:
|
|
837
|
+
covered_ids = covered_ids.to(self.indicators.device)
|
|
838
|
+
inds = self.b.indicators[channels][:, covered_ids]
|
|
839
|
+
coverage = inds.sum(0) / self.b.channel_counts[covered_ids]
|
|
840
|
+
covered = coverage >= min_coverage
|
|
841
|
+
covered_ids = covered_ids[covered].cpu()
|
|
842
|
+
neighborhood_ids = neighborhood_ids.cpu()
|
|
843
|
+
|
|
844
|
+
neighborhood_info = [
|
|
845
|
+
(
|
|
846
|
+
j,
|
|
847
|
+
self.b.neighborhoods[j],
|
|
848
|
+
*(neighborhood_ids == j).nonzero(as_tuple=True),
|
|
849
|
+
None,
|
|
850
|
+
)
|
|
851
|
+
for j in covered_ids
|
|
852
|
+
]
|
|
853
|
+
n_spikes = self.b.popcounts[covered_ids].sum()
|
|
854
|
+
return neighborhood_info, n_spikes
|
|
855
|
+
|
|
856
|
+
def adjacency(self, overlap=0.5):
|
|
857
|
+
overlaps = self.b.indicators.T @ self.b.indicators
|
|
858
|
+
assert overlaps.shape == (self.n_neighborhoods, self.n_neighborhoods)
|
|
859
|
+
counts = self.b.indicators.sum(0)
|
|
860
|
+
overlaps /= torch.minimum(counts[:, None], counts)
|
|
861
|
+
return (overlaps >= overlap - 1e-5).to(torch.float)
|
|
862
|
+
|
|
863
|
+
def partial_order(self):
|
|
864
|
+
"""ret[i, j] == 1 iff neighb j subset neighb i"""
|
|
865
|
+
inds = self.b.indicators.T # nneighb x nc
|
|
866
|
+
po = (inds[:, None, :] >= inds[None, :, :]).all(2)
|
|
867
|
+
assert po.shape == (self.n_neighborhoods, self.n_neighborhoods)
|
|
868
|
+
return po
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
# -- helpers
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def occupied_chans(
|
|
875
|
+
spike_data,
|
|
876
|
+
n_channels,
|
|
877
|
+
neighborhood_ids=None,
|
|
878
|
+
neighborhoods=None,
|
|
879
|
+
fuzz=0,
|
|
880
|
+
weights=None,
|
|
881
|
+
):
|
|
882
|
+
if spike_data.neighborhood_ids is None:
|
|
883
|
+
chans = torch.unique(spike_data.channels)
|
|
884
|
+
return chans[chans < n_channels]
|
|
885
|
+
assert neighborhoods is not None
|
|
886
|
+
if neighborhood_ids is None:
|
|
887
|
+
neighborhood_ids = spike_data.neighborhood_ids
|
|
888
|
+
ids, inverse = torch.unique(neighborhood_ids, return_inverse=True)
|
|
889
|
+
if weights is None:
|
|
890
|
+
weights = torch.ones(inverse.shape, device=ids.device)
|
|
891
|
+
id_counts = torch.zeros(ids.shape, device=ids.device)
|
|
892
|
+
spiketorch.add_at_(id_counts, inverse, weights.to(ids.device))
|
|
893
|
+
|
|
894
|
+
chans0 = neighborhoods.neighborhoods[ids]
|
|
895
|
+
chans, inverse = torch.unique(chans0, return_inverse=True)
|
|
896
|
+
counts = torch.zeros(chans.shape, device=chans.device)
|
|
897
|
+
spiketorch.add_at_(
|
|
898
|
+
counts,
|
|
899
|
+
inverse.view(-1).to(counts.device),
|
|
900
|
+
id_counts[:, None].broadcast_to(chans0.shape).reshape(-1).to(counts.device),
|
|
901
|
+
)
|
|
902
|
+
|
|
903
|
+
counts = counts[chans < n_channels]
|
|
904
|
+
chans = chans[chans < n_channels]
|
|
905
|
+
for _ in range(fuzz):
|
|
906
|
+
chans = neighborhoods.channel_index[chans]
|
|
907
|
+
chans = torch.unique(chans)
|
|
908
|
+
chans = chans[chans < n_channels]
|
|
909
|
+
|
|
910
|
+
return chans, counts
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def interp_to_chans(
|
|
914
|
+
spike_data,
|
|
915
|
+
channels,
|
|
916
|
+
prgeom,
|
|
917
|
+
params,
|
|
918
|
+
batch_size=256,
|
|
919
|
+
):
|
|
920
|
+
source_pos = prgeom[spike_data.channels]
|
|
921
|
+
target_pos = prgeom[channels]
|
|
922
|
+
shape = len(spike_data), *target_pos.shape
|
|
923
|
+
output_shape = *spike_data.features.shape[:2], channels.numel()
|
|
924
|
+
output = spike_data.features.new_empty(output_shape)
|
|
925
|
+
if not channels.numel():
|
|
926
|
+
return output
|
|
927
|
+
for batch_start in range(0, len(spike_data), batch_size):
|
|
928
|
+
batch_end = min(batch_start + batch_size, len(spike_data))
|
|
929
|
+
sl = slice(batch_start, batch_end)
|
|
930
|
+
shape = batch_end - batch_start, *target_pos.shape
|
|
931
|
+
target_pos_ = target_pos[None].broadcast_to(shape)
|
|
932
|
+
interpolation_util.kernel_interpolate(
|
|
933
|
+
spike_data.features[sl],
|
|
934
|
+
source_pos[sl],
|
|
935
|
+
target_pos_,
|
|
936
|
+
params=params,
|
|
937
|
+
allow_destroy=False,
|
|
938
|
+
out=output[sl],
|
|
939
|
+
)
|
|
940
|
+
return output
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def pad_to_chans(
|
|
944
|
+
spike_data, channels, n_channels, weights=None, target_padded=None, pad_value=0.0
|
|
945
|
+
):
|
|
946
|
+
n, r, c = spike_data.features.shape
|
|
947
|
+
c_targ = channels.numel()
|
|
948
|
+
|
|
949
|
+
# determine channels to write to
|
|
950
|
+
reindexer = get_channel_reindexer(channels, n_channels)
|
|
951
|
+
target_ixs = reindexer[spike_data.channels].to(spike_data.features.device)
|
|
952
|
+
|
|
953
|
+
# scatter data
|
|
954
|
+
if target_padded is None:
|
|
955
|
+
if pad_value == 0.0:
|
|
956
|
+
target_padded = spike_data.features.new_zeros(n, r, c_targ + 1)
|
|
957
|
+
else:
|
|
958
|
+
target_padded = spike_data.features.new_full((n, r, c_targ + 1), pad_value)
|
|
959
|
+
scatter_ixs = target_ixs.unsqueeze(1).broadcast_to((n, r, c))
|
|
960
|
+
target_padded.scatter_(src=spike_data.features, dim=2, index=scatter_ixs)
|
|
961
|
+
target = target_padded[..., :-1]
|
|
962
|
+
if weights is None:
|
|
963
|
+
return target, None
|
|
964
|
+
|
|
965
|
+
# same for weights, if supplied
|
|
966
|
+
assert weights.shape == (n,)
|
|
967
|
+
weights_padded = weights.new_zeros((n, c_targ + 1))
|
|
968
|
+
weights = weights.unsqueeze(1).broadcast_to((n, c))
|
|
969
|
+
weights_padded.scatter_(src=weights, dim=1, index=target_ixs)
|
|
970
|
+
weights = weights_padded[:, :-1]
|
|
971
|
+
return target, weights
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def get_channel_reindexer(channels, n_channels):
|
|
975
|
+
"""
|
|
976
|
+
Arguments
|
|
977
|
+
---------
|
|
978
|
+
channels : Tensor
|
|
979
|
+
Shape (n_chans_subset,)
|
|
980
|
+
n_channels : int
|
|
981
|
+
|
|
982
|
+
Returns
|
|
983
|
+
-------
|
|
984
|
+
reindexer : Tensor
|
|
985
|
+
Shape (n_channels + 1,)
|
|
986
|
+
reindexer[i] is the index of i in channels, if present.
|
|
987
|
+
Otherwise, it is n_chans_subset.
|
|
988
|
+
And the last entry is, of course, n_chans_subset.
|
|
989
|
+
"""
|
|
990
|
+
reindexer = channels.new_full((n_channels + 1,), channels.numel())
|
|
991
|
+
(rel_ixs,) = torch.nonzero(channels < n_channels, as_tuple=True)
|
|
992
|
+
reindexer[channels[rel_ixs]] = rel_ixs
|
|
993
|
+
return reindexer
|