DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1128 @@
|
|
|
1
|
+
from collections import namedtuple
|
|
2
|
+
from copy import copy
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Generator, Sequence, cast, Self, Literal
|
|
6
|
+
import warnings
|
|
7
|
+
|
|
8
|
+
import h5py
|
|
9
|
+
import numpy as np
|
|
10
|
+
import torch
|
|
11
|
+
from spikeinterface.core import NumpySorting, get_random_data_chunks
|
|
12
|
+
from tqdm.auto import tqdm
|
|
13
|
+
|
|
14
|
+
from ..detect import detect_and_deduplicate
|
|
15
|
+
from ..util.logging_util import get_logger
|
|
16
|
+
from ..util.py_util import resolve_path
|
|
17
|
+
from .waveform_util import make_channel_index
|
|
18
|
+
|
|
19
|
+
logger = get_logger(__name__)
|
|
20
|
+
|
|
21
|
+
# this is a data type used in the peeling code to store info about
|
|
22
|
+
# the datasets which are being computed
|
|
23
|
+
# the featurizers in transform have a .spike_dataset property which
|
|
24
|
+
# is this type
|
|
25
|
+
SpikeDataset = namedtuple("SpikeDataset", ["name", "shape_per_spike", "dtype"])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DARTsortSorting:
|
|
29
|
+
"""Class which holds spike times, channels, and labels
|
|
30
|
+
|
|
31
|
+
This class holds our algorithm state.
|
|
32
|
+
Initially the sorter doesn't have unit labels, so these are optional.
|
|
33
|
+
Export me to a SpikeInterface NumpySorting with .to_numpy_sorting()
|
|
34
|
+
|
|
35
|
+
When you instantiate this with from_peeling_hdf5, if the
|
|
36
|
+
flag load_simple_features is True (default), then additional
|
|
37
|
+
features of spikes will be loaded into memory -- like localizations,
|
|
38
|
+
which you can access like `sorting.point_source_localizations[...]`.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
*,
|
|
44
|
+
times_samples: np.ndarray,
|
|
45
|
+
channels: np.ndarray,
|
|
46
|
+
labels: np.ndarray | None,
|
|
47
|
+
parent_h5_path: str | Path | None = None,
|
|
48
|
+
sampling_frequency: float | int = 30000.0,
|
|
49
|
+
persistent_features: dict[str, np.ndarray] | None = None,
|
|
50
|
+
ephemeral_features: dict[str, np.ndarray] | None = None,
|
|
51
|
+
):
|
|
52
|
+
self.n_spikes = times_samples.shape[0]
|
|
53
|
+
if parent_h5_path is not None:
|
|
54
|
+
parent_h5_path = resolve_path(parent_h5_path)
|
|
55
|
+
self.parent_h5_path = parent_h5_path
|
|
56
|
+
self.sampling_frequency = float(sampling_frequency)
|
|
57
|
+
|
|
58
|
+
assert times_samples.dtype.kind == "i"
|
|
59
|
+
self.times_samples = times_samples
|
|
60
|
+
assert channels.dtype.kind == "i"
|
|
61
|
+
assert channels.shape == (self.n_spikes,)
|
|
62
|
+
self.channels = channels
|
|
63
|
+
if labels is not None:
|
|
64
|
+
assert labels.shape == (self.n_spikes,)
|
|
65
|
+
assert labels.dtype.kind == "i"
|
|
66
|
+
self.labels = labels
|
|
67
|
+
|
|
68
|
+
self._ephemeral_feature_names = []
|
|
69
|
+
if ephemeral_features is not None:
|
|
70
|
+
for k, v in ephemeral_features.items():
|
|
71
|
+
check_shape = not self._no_check_needed(k)
|
|
72
|
+
self.add_ephemeral_feature(k, v, check_shape=check_shape)
|
|
73
|
+
|
|
74
|
+
self._loaded_persistent_features = []
|
|
75
|
+
if persistent_features is not None:
|
|
76
|
+
for k, v in persistent_features.items():
|
|
77
|
+
check_shape = not self._no_check_needed(k)
|
|
78
|
+
self._register_persistent_feature(k, v, check_shape=check_shape)
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def unit_ids(self) -> np.ndarray:
|
|
82
|
+
if self.labels is None:
|
|
83
|
+
return np.array([], dtype=np.int64)
|
|
84
|
+
u = np.unique(self.labels)
|
|
85
|
+
return u[u >= 0]
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def n_units(self) -> int:
|
|
89
|
+
return self.unit_ids.shape[0]
|
|
90
|
+
|
|
91
|
+
def copy(self) -> Self:
|
|
92
|
+
"""Deep-enough copy."""
|
|
93
|
+
other = copy(self)
|
|
94
|
+
other._ephemeral_feature_names = self._ephemeral_feature_names.copy()
|
|
95
|
+
other._loaded_persistent_features = self._loaded_persistent_features.copy()
|
|
96
|
+
return other
|
|
97
|
+
|
|
98
|
+
def ephemeral_replace(
|
|
99
|
+
self, *, check_shapes=True, **new_features: np.ndarray
|
|
100
|
+
) -> Self:
|
|
101
|
+
other = self.copy()
|
|
102
|
+
for k, v in new_features.items():
|
|
103
|
+
if k in ("times_samples", "channels", "labels"):
|
|
104
|
+
if check_shapes:
|
|
105
|
+
other._check_shape(k, v)
|
|
106
|
+
setattr(other, k, v)
|
|
107
|
+
else:
|
|
108
|
+
check = check_shapes and not self._no_check_needed(k)
|
|
109
|
+
other.add_ephemeral_feature(k, v, check_shape=check, overwrite=True)
|
|
110
|
+
return other
|
|
111
|
+
|
|
112
|
+
def has_persistent_labels(self) -> bool:
|
|
113
|
+
"""Are my .labels those from the hdf5 file?"""
|
|
114
|
+
if self.parent_h5_path is None:
|
|
115
|
+
return False
|
|
116
|
+
if self.labels is None:
|
|
117
|
+
return False
|
|
118
|
+
if "labels" in self._ephemeral_feature_names:
|
|
119
|
+
assert "labels" not in self._loaded_persistent_features
|
|
120
|
+
return False
|
|
121
|
+
try:
|
|
122
|
+
return np.array_equal(self.labels, self._load_dataset("labels"))
|
|
123
|
+
except KeyError:
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
# interface for setting features
|
|
127
|
+
|
|
128
|
+
def add_ephemeral_feature(
|
|
129
|
+
self,
|
|
130
|
+
feature_name: str,
|
|
131
|
+
feature: np.ndarray,
|
|
132
|
+
check_shape: bool | None = None,
|
|
133
|
+
overwrite=False,
|
|
134
|
+
):
|
|
135
|
+
"""
|
|
136
|
+
Ephemeral features are accessible as properties and persisted to/from .npz,
|
|
137
|
+
but not saved in the .h5.
|
|
138
|
+
"""
|
|
139
|
+
if check_shape is None:
|
|
140
|
+
check_shape = not self._no_check_needed(feature_name)
|
|
141
|
+
if check_shape:
|
|
142
|
+
self._check_shape(feature_name, feature)
|
|
143
|
+
|
|
144
|
+
already_ephemeral = feature_name in self._ephemeral_feature_names
|
|
145
|
+
already_attr = hasattr(self, feature_name)
|
|
146
|
+
if already_ephemeral:
|
|
147
|
+
assert already_attr
|
|
148
|
+
if already_attr and not overwrite:
|
|
149
|
+
raise ValueError(
|
|
150
|
+
f"Can't add feature {feature_name}, since it already exists."
|
|
151
|
+
)
|
|
152
|
+
if not already_ephemeral:
|
|
153
|
+
self._ephemeral_feature_names.append(feature_name)
|
|
154
|
+
setattr(self, feature_name, feature)
|
|
155
|
+
|
|
156
|
+
def remove_ephemeral_feature(self, feature_name: str):
|
|
157
|
+
assert feature_name in self._ephemeral_feature_names
|
|
158
|
+
assert hasattr(self, feature_name)
|
|
159
|
+
self._ephemeral_feature_names = [
|
|
160
|
+
k for k in self._ephemeral_feature_names if k != feature_name
|
|
161
|
+
]
|
|
162
|
+
delattr(self, feature_name)
|
|
163
|
+
|
|
164
|
+
def unload_persistent_feature(self, feature_name: str):
|
|
165
|
+
assert feature_name in self._loaded_persistent_features
|
|
166
|
+
assert hasattr(self, feature_name)
|
|
167
|
+
self._loaded_persistent_features = [
|
|
168
|
+
k for k in self._loaded_persistent_features if k != feature_name
|
|
169
|
+
]
|
|
170
|
+
delattr(self, feature_name)
|
|
171
|
+
|
|
172
|
+
def add_feature(self, feature_name: str, feature: np.ndarray, check_shape=True):
|
|
173
|
+
"""Try to save a feature to h5, else register as ephemeral."""
|
|
174
|
+
if self.parent_h5_path is None:
|
|
175
|
+
self.add_ephemeral_feature(feature_name, feature, check_shape)
|
|
176
|
+
else:
|
|
177
|
+
self._register_persistent_feature(feature_name, feature, check_shape)
|
|
178
|
+
|
|
179
|
+
def remove_feature(self, feature_name: str):
|
|
180
|
+
if feature_name in self._loaded_persistent_features:
|
|
181
|
+
self.unload_persistent_feature(feature_name)
|
|
182
|
+
elif feature_name in self._ephemeral_feature_names:
|
|
183
|
+
self.remove_ephemeral_feature(feature_name)
|
|
184
|
+
else:
|
|
185
|
+
raise ValueError(f"Sorting doesn't have {feature_name}.")
|
|
186
|
+
|
|
187
|
+
def _register_persistent_feature(
|
|
188
|
+
self, feature_name: str, feature: np.ndarray, check_shape=True
|
|
189
|
+
):
|
|
190
|
+
"""Persistent features are written to the h5."""
|
|
191
|
+
if self.parent_h5_path is None:
|
|
192
|
+
raise ValueError(
|
|
193
|
+
f"Can't register persistent feature {feature_name}, because "
|
|
194
|
+
f"there is no .hdf5 file.",
|
|
195
|
+
)
|
|
196
|
+
if check_shape:
|
|
197
|
+
self._check_shape(feature_name, feature)
|
|
198
|
+
self._loaded_persistent_features.append(feature_name)
|
|
199
|
+
setattr(self, feature_name, feature)
|
|
200
|
+
try:
|
|
201
|
+
with h5py.File(
|
|
202
|
+
self.parent_h5_path, "r", libver="latest", locking=False
|
|
203
|
+
) as h5:
|
|
204
|
+
if feature_name not in h5:
|
|
205
|
+
logger.dartsortdebug(
|
|
206
|
+
"Registering persistent feature %s to %s.",
|
|
207
|
+
feature_name,
|
|
208
|
+
self.parent_h5_path,
|
|
209
|
+
)
|
|
210
|
+
h5.create_dataset(feature_name, data=feature)
|
|
211
|
+
except FileNotFoundError:
|
|
212
|
+
logger.warning(
|
|
213
|
+
f"Sorting's parent h5 file {self.parent_h5_path} is gone when registering "
|
|
214
|
+
f"persistent feature {feature_name}. Will continue, but this sorting won't "
|
|
215
|
+
"persist correctly.",
|
|
216
|
+
stacklevel=3,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
# save / load
|
|
220
|
+
|
|
221
|
+
@classmethod
|
|
222
|
+
def from_peeling_hdf5(
|
|
223
|
+
cls,
|
|
224
|
+
h5_path: str | Path,
|
|
225
|
+
*,
|
|
226
|
+
times_samples_dataset="times_samples",
|
|
227
|
+
channels_dataset="channels",
|
|
228
|
+
labels_dataset="labels",
|
|
229
|
+
load_feature_names: Sequence[str] | None = None,
|
|
230
|
+
load_simple_features=True,
|
|
231
|
+
load_all_features=False,
|
|
232
|
+
) -> Self:
|
|
233
|
+
"""Load sorting from .hdf5 format saved by peelers
|
|
234
|
+
|
|
235
|
+
Arguments
|
|
236
|
+
---------
|
|
237
|
+
load_feature_names : optional list of str
|
|
238
|
+
Load exactly these features, plus geom/channel index.
|
|
239
|
+
load_simple_features : bool
|
|
240
|
+
If load_feature_names unspecified, load all scalar or vector
|
|
241
|
+
features (per spike), but no matrix-valued features like waveforms
|
|
242
|
+
or multi-channel PCA features.
|
|
243
|
+
load_all_features : bool
|
|
244
|
+
"""
|
|
245
|
+
h5_path = resolve_path(h5_path, strict=True)
|
|
246
|
+
|
|
247
|
+
with h5py.File(h5_path, "r", libver="latest", locking=False) as h5:
|
|
248
|
+
times_samples = cast(h5py.Dataset, h5[times_samples_dataset])[:]
|
|
249
|
+
n = times_samples.shape[0]
|
|
250
|
+
channels = cast(h5py.Dataset, h5[channels_dataset])[:]
|
|
251
|
+
sampling_frequency = float(
|
|
252
|
+
cast(h5py.Dataset, h5["sampling_frequency"])[()].item()
|
|
253
|
+
)
|
|
254
|
+
if labels_dataset in h5:
|
|
255
|
+
labels = cast(h5py.Dataset, h5[labels_dataset])[:]
|
|
256
|
+
else:
|
|
257
|
+
labels = None
|
|
258
|
+
|
|
259
|
+
already_loaded = [
|
|
260
|
+
times_samples_dataset,
|
|
261
|
+
channels_dataset,
|
|
262
|
+
labels_dataset,
|
|
263
|
+
"sampling_frequency",
|
|
264
|
+
]
|
|
265
|
+
if load_feature_names is None and load_all_features:
|
|
266
|
+
load_feature_names = list(h5.keys())
|
|
267
|
+
elif load_feature_names is None and load_simple_features:
|
|
268
|
+
load_feature_names = []
|
|
269
|
+
for k in h5.keys():
|
|
270
|
+
if k in already_loaded:
|
|
271
|
+
continue
|
|
272
|
+
if cls._no_check_needed(k):
|
|
273
|
+
load_feature_names.append(k)
|
|
274
|
+
continue
|
|
275
|
+
dset = cast(h5py.Dataset, h5[k])
|
|
276
|
+
is_simple = 1 <= dset.ndim <= 2 and dset.shape[0] == n
|
|
277
|
+
if is_simple:
|
|
278
|
+
load_feature_names.append(k)
|
|
279
|
+
elif load_feature_names is None:
|
|
280
|
+
load_feature_names = [k for k in h5.keys() if cls._no_check_needed(k)]
|
|
281
|
+
assert load_feature_names is not None
|
|
282
|
+
load_feature_names = [
|
|
283
|
+
k for k in load_feature_names if k not in already_loaded
|
|
284
|
+
]
|
|
285
|
+
persistent_features = {
|
|
286
|
+
k: cast(h5py.Dataset, h5[k])[:] for k in load_feature_names
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
return cls(
|
|
290
|
+
times_samples=times_samples,
|
|
291
|
+
channels=channels,
|
|
292
|
+
labels=labels,
|
|
293
|
+
sampling_frequency=sampling_frequency,
|
|
294
|
+
persistent_features=persistent_features,
|
|
295
|
+
parent_h5_path=h5_path,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
def save(self, sorting_npz: str | Path):
|
|
299
|
+
"""Support persisting myself in non-h5-supportable cases
|
|
300
|
+
|
|
301
|
+
Cases:
|
|
302
|
+
- When there is no h5!
|
|
303
|
+
- When my labels disagree with the .h5 labels.
|
|
304
|
+
|
|
305
|
+
This is done by saving to .npz, with a pointer to the .h5 file if
|
|
306
|
+
it exists.
|
|
307
|
+
"""
|
|
308
|
+
sorting_npz = resolve_path(sorting_npz)
|
|
309
|
+
data = dict(
|
|
310
|
+
times_samples=self.times_samples,
|
|
311
|
+
channels=self.channels,
|
|
312
|
+
sampling_frequency=np.array(self.sampling_frequency),
|
|
313
|
+
)
|
|
314
|
+
if self.labels is not None:
|
|
315
|
+
data["labels"] = self.labels
|
|
316
|
+
|
|
317
|
+
have_hdf5 = self.parent_h5_path is not None
|
|
318
|
+
if have_hdf5:
|
|
319
|
+
# path needs to be relative to npz path's parent in case user moves stuff
|
|
320
|
+
h5p = resolve_path(self.parent_h5_path, strict=True)
|
|
321
|
+
h5p = h5p.relative_to(sorting_npz.parent)
|
|
322
|
+
data["parent_h5_path"] = np.array(str(h5p))
|
|
323
|
+
for k in self._ephemeral_feature_names:
|
|
324
|
+
data[k] = getattr(self, k)
|
|
325
|
+
data["ephemeral_feature_names"] = np.array(self._ephemeral_feature_names)
|
|
326
|
+
data["loaded_persistent_features"] = np.array(self._loaded_persistent_features)
|
|
327
|
+
np.savez(sorting_npz, **data, allow_pickle=False)
|
|
328
|
+
|
|
329
|
+
@classmethod
|
|
330
|
+
def load(cls, sorting_npz, additional_persistent_features=None) -> Self:
|
|
331
|
+
sorting_npz = resolve_path(sorting_npz, strict=True)
|
|
332
|
+
with np.load(sorting_npz) as data:
|
|
333
|
+
times_samples = data["times_samples"]
|
|
334
|
+
channels = data["channels"]
|
|
335
|
+
labels = data.get("labels", None)
|
|
336
|
+
sampling_frequency = data["sampling_frequency"]
|
|
337
|
+
if isinstance(sampling_frequency, np.ndarray):
|
|
338
|
+
sampling_frequency = sampling_frequency.item()
|
|
339
|
+
parent_h5_path = data.get("parent_h5_path", None)
|
|
340
|
+
if "ephemeral_feature_names" in data:
|
|
341
|
+
ephemeral_features = {
|
|
342
|
+
k: data[k] for k in data["ephemeral_feature_names"]
|
|
343
|
+
}
|
|
344
|
+
else:
|
|
345
|
+
ephemeral_features = {}
|
|
346
|
+
loaded_persistent_features = data.get("loaded_persistent_features", [])
|
|
347
|
+
|
|
348
|
+
if parent_h5_path is not None:
|
|
349
|
+
parent_h5_path = parent_h5_path.item()
|
|
350
|
+
assert isinstance(parent_h5_path, str)
|
|
351
|
+
parent_h5_path = sorting_npz.parent / Path(parent_h5_path)
|
|
352
|
+
parent_h5_path = resolve_path(parent_h5_path, strict=True)
|
|
353
|
+
if additional_persistent_features:
|
|
354
|
+
loaded_persistent_features = set(
|
|
355
|
+
loaded_persistent_features + additional_persistent_features
|
|
356
|
+
)
|
|
357
|
+
self = cls.from_peeling_hdf5(
|
|
358
|
+
parent_h5_path,
|
|
359
|
+
load_feature_names=list(loaded_persistent_features),
|
|
360
|
+
)
|
|
361
|
+
if labels is not None:
|
|
362
|
+
ephemeral_features["labels"] = labels
|
|
363
|
+
return self.ephemeral_replace(
|
|
364
|
+
times_samples=times_samples, channels=channels, **ephemeral_features
|
|
365
|
+
)
|
|
366
|
+
assert not loaded_persistent_features
|
|
367
|
+
|
|
368
|
+
return cls(
|
|
369
|
+
times_samples=times_samples,
|
|
370
|
+
channels=channels,
|
|
371
|
+
labels=labels,
|
|
372
|
+
sampling_frequency=sampling_frequency,
|
|
373
|
+
parent_h5_path=parent_h5_path,
|
|
374
|
+
ephemeral_features=ephemeral_features,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
def to_numpy_sorting(self) -> NumpySorting:
|
|
378
|
+
return NumpySorting.from_samples_and_labels(
|
|
379
|
+
samples_list=self.times_samples,
|
|
380
|
+
labels_list=self.labels,
|
|
381
|
+
sampling_frequency=self.sampling_frequency,
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
def mask(self, mask: np.ndarray) -> Self:
|
|
385
|
+
assert mask.ndim == 1
|
|
386
|
+
if np.dtype(mask.dtype).kind == "b":
|
|
387
|
+
assert mask.shape == (self.n_spikes,)
|
|
388
|
+
mask = np.flatnonzero(mask)
|
|
389
|
+
assert mask.max() < self.n_spikes
|
|
390
|
+
|
|
391
|
+
if self.labels is None:
|
|
392
|
+
labels = None
|
|
393
|
+
else:
|
|
394
|
+
labels = self.labels[mask]
|
|
395
|
+
|
|
396
|
+
eph = {}
|
|
397
|
+
for k in self._ephemeral_feature_names:
|
|
398
|
+
assert k != "mask_indices" # no recursion...
|
|
399
|
+
v = getattr(self, k)
|
|
400
|
+
if self._no_check_needed(k):
|
|
401
|
+
eph[k] = v
|
|
402
|
+
else:
|
|
403
|
+
eph[k] = v[mask]
|
|
404
|
+
eph["mask_indices"] = mask
|
|
405
|
+
|
|
406
|
+
per = {}
|
|
407
|
+
for k in self._loaded_persistent_features:
|
|
408
|
+
assert k != "mask_indices" # no recursion...
|
|
409
|
+
v = getattr(self, k)
|
|
410
|
+
if self._no_check_needed(k):
|
|
411
|
+
per[k] = v
|
|
412
|
+
else:
|
|
413
|
+
per[k] = v[mask]
|
|
414
|
+
|
|
415
|
+
return self.__class__(
|
|
416
|
+
times_samples=self.times_samples[mask],
|
|
417
|
+
channels=self.channels[mask],
|
|
418
|
+
labels=labels,
|
|
419
|
+
parent_h5_path=self.parent_h5_path,
|
|
420
|
+
sampling_frequency=self.sampling_frequency,
|
|
421
|
+
persistent_features=per,
|
|
422
|
+
ephemeral_features=eph,
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
def drop_missing(self) -> Self:
|
|
426
|
+
assert self.labels is not None
|
|
427
|
+
return self.mask(self.labels >= 0)
|
|
428
|
+
|
|
429
|
+
def drop_doubles(self):
|
|
430
|
+
assert self.labels is not None
|
|
431
|
+
viol_ixs = []
|
|
432
|
+
for uid in self.unit_ids:
|
|
433
|
+
inu = np.flatnonzero(self.labels == uid)
|
|
434
|
+
tu = self.times_samples[inu]
|
|
435
|
+
# this has the first indices of each pair. +1 has the second indices.
|
|
436
|
+
viol = np.flatnonzero(np.diff(tu) == 0)
|
|
437
|
+
viol_ixs.append(viol + 1)
|
|
438
|
+
viol_ixs = np.concatenate(viol_ixs)
|
|
439
|
+
if viol_ixs.size:
|
|
440
|
+
logger.dartsortdebug(f"Dropping {viol_ixs.size} duplicates.")
|
|
441
|
+
labels = self.labels.copy()
|
|
442
|
+
labels[viol_ixs] = -1
|
|
443
|
+
return self.ephemeral_replace(labels=labels)
|
|
444
|
+
else:
|
|
445
|
+
return self
|
|
446
|
+
|
|
447
|
+
def flatten(self) -> Self:
|
|
448
|
+
assert self.labels is not None
|
|
449
|
+
valid = np.flatnonzero(self.labels >= 0)
|
|
450
|
+
_, flat_labels = np.unique(self.labels[valid], return_inverse=True)
|
|
451
|
+
new_labels = np.full_like(self.labels, -1)
|
|
452
|
+
new_labels[valid] = flat_labels
|
|
453
|
+
return self.ephemeral_replace(labels=new_labels)
|
|
454
|
+
|
|
455
|
+
def __str__(self):
|
|
456
|
+
name = self.__class__.__name__
|
|
457
|
+
ns = self.n_spikes
|
|
458
|
+
nu = self.n_units
|
|
459
|
+
unit_str = f"{nu} unit" + "s" * (nu > 1)
|
|
460
|
+
feat_str = " "
|
|
461
|
+
if self._loaded_persistent_features:
|
|
462
|
+
s = ", ".join(self._loaded_persistent_features)
|
|
463
|
+
feat_str += f"Loaded HDF5 features: {s}. "
|
|
464
|
+
if self._ephemeral_feature_names:
|
|
465
|
+
s = ", ".join(self._ephemeral_feature_names)
|
|
466
|
+
feat_str += f"Features: {s}. "
|
|
467
|
+
h5_str = ""
|
|
468
|
+
if self.parent_h5_path:
|
|
469
|
+
h5_str = f"From HDF5 file {self.parent_h5_path}."
|
|
470
|
+
if self.labels is not None:
|
|
471
|
+
noise_prop = (self.labels < 0).mean().item()
|
|
472
|
+
noise_pct = 100 * noise_prop
|
|
473
|
+
noise_str = f" ({noise_pct:.2f}% noise)"
|
|
474
|
+
else:
|
|
475
|
+
noise_str = ""
|
|
476
|
+
return f"{name}: {ns} spikes{noise_str}, {unit_str}.{feat_str}{h5_str}"
|
|
477
|
+
|
|
478
|
+
def __repr__(self):
|
|
479
|
+
return str(self)
|
|
480
|
+
|
|
481
|
+
def __len__(self):
|
|
482
|
+
return self.n_spikes
|
|
483
|
+
|
|
484
|
+
@staticmethod
|
|
485
|
+
def _is_geom_related(k) -> bool:
|
|
486
|
+
return k == "geom" or k.endswith("channel_index")
|
|
487
|
+
|
|
488
|
+
@staticmethod
|
|
489
|
+
def _is_unit_related(k) -> bool:
|
|
490
|
+
return k.startswith("unit")
|
|
491
|
+
|
|
492
|
+
@classmethod
|
|
493
|
+
def _no_check_needed(cls, k) -> bool:
|
|
494
|
+
return cls._is_geom_related(k) or cls._is_unit_related(k)
|
|
495
|
+
|
|
496
|
+
def _check_shape(self, feature_name: str, feature: np.ndarray):
|
|
497
|
+
if feature.shape[0] != self.n_spikes:
|
|
498
|
+
raise ValueError(
|
|
499
|
+
f"Feature {feature_name}'s shape {feature.shape} didn't agree with spike count {self.n_spikes}."
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
def _load_dataset(self, dataset_name: str) -> np.ndarray:
|
|
503
|
+
assert self.parent_h5_path is not None
|
|
504
|
+
with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
|
|
505
|
+
dset = h5[dataset_name]
|
|
506
|
+
assert isinstance(dset, h5py.Dataset)
|
|
507
|
+
return dset[:]
|
|
508
|
+
|
|
509
|
+
def slice_feature_by_name(
|
|
510
|
+
self, dataset_name: str, mask: np.ndarray | slice = slice(None)
|
|
511
|
+
) -> np.ndarray:
|
|
512
|
+
if hasattr(self, dataset_name):
|
|
513
|
+
return getattr(self, dataset_name)[mask]
|
|
514
|
+
|
|
515
|
+
# otherwise, we don't have it loaded
|
|
516
|
+
assert dataset_name not in self._ephemeral_feature_names
|
|
517
|
+
assert dataset_name not in self._loaded_persistent_features
|
|
518
|
+
|
|
519
|
+
# but we can try to load it
|
|
520
|
+
if self.parent_h5_path is None:
|
|
521
|
+
raise ValueError(f"Can't load feature {dataset_name} with no HDF5.")
|
|
522
|
+
if isinstance(mask, slice) and mask == slice(None):
|
|
523
|
+
return self._load_dataset(dataset_name)
|
|
524
|
+
|
|
525
|
+
# h5 direct read is fine for a few indices
|
|
526
|
+
if isinstance(mask, np.ndarray) and mask.dtype.kind != "b":
|
|
527
|
+
if mask.size <= 768:
|
|
528
|
+
with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
|
|
529
|
+
return cast(h5py.Dataset, h5[dataset_name])[mask]
|
|
530
|
+
|
|
531
|
+
# mask needs to be boolean for _read_by_chunk
|
|
532
|
+
if not isinstance(mask, np.ndarray) or mask.dtype.kind != "b":
|
|
533
|
+
h5_mask = np.zeros(self.n_spikes, dtype=np.bool_)
|
|
534
|
+
h5_mask[mask] = True
|
|
535
|
+
else:
|
|
536
|
+
h5_mask = mask
|
|
537
|
+
|
|
538
|
+
with h5py.File(self.parent_h5_path, "r", locking=False) as h5:
|
|
539
|
+
dset = h5[dataset_name]
|
|
540
|
+
assert isinstance(dset, h5py.Dataset)
|
|
541
|
+
assert dset.shape[0] == self.n_spikes == h5_mask.shape[0]
|
|
542
|
+
return _read_by_chunk(mask, dset, show_progress=False)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def load_h5(f: str | Path) -> DARTsortSorting:
|
|
546
|
+
return DARTsortSorting.from_peeling_hdf5(h5_path=f)
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def try_get_model_dir(sorting: DARTsortSorting) -> Path | None:
|
|
550
|
+
if sorting.parent_h5_path is None:
|
|
551
|
+
return None
|
|
552
|
+
h5_path = resolve_path(sorting.parent_h5_path)
|
|
553
|
+
model_dir = h5_path.parent / f"{h5_path.stem}_models"
|
|
554
|
+
if model_dir.exists():
|
|
555
|
+
assert model_dir.is_dir()
|
|
556
|
+
return model_dir
|
|
557
|
+
else:
|
|
558
|
+
return None
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def get_featurization_pipeline(sorting, featurization_pipeline_pt=None):
|
|
562
|
+
"""Look for the pipeline in the usual place."""
|
|
563
|
+
from dartsort.transform import WaveformPipeline
|
|
564
|
+
|
|
565
|
+
if isinstance(sorting, Path):
|
|
566
|
+
base_dir = sorting.parent
|
|
567
|
+
stem = sorting.stem
|
|
568
|
+
# TODO how to type this better... don't understand.
|
|
569
|
+
geom = channel_index = None # type: ignore
|
|
570
|
+
else:
|
|
571
|
+
assert isinstance(sorting, DARTsortSorting)
|
|
572
|
+
if sorting.parent_h5_path is None:
|
|
573
|
+
raise ValueError("Can't load featurization pipeline.")
|
|
574
|
+
|
|
575
|
+
h5_path = resolve_path(sorting.parent_h5_path)
|
|
576
|
+
base_dir = h5_path.parent
|
|
577
|
+
stem = h5_path.stem
|
|
578
|
+
|
|
579
|
+
geom = getattr(sorting, "geom", None) # type: ignore
|
|
580
|
+
channel_index = getattr(sorting, "channel_index", None) # type: ignore
|
|
581
|
+
|
|
582
|
+
model_dir = base_dir / f"{stem}_models"
|
|
583
|
+
if geom is None or channel_index is None:
|
|
584
|
+
with h5py.File(base_dir / f"{stem}.h5", "r", locking=False) as h5:
|
|
585
|
+
geom: np.ndarray = h5["geom"][:] # type: ignore
|
|
586
|
+
channel_index: np.ndarray = h5["channel_index"][:] # type: ignore
|
|
587
|
+
assert geom is not None
|
|
588
|
+
assert channel_index is not None
|
|
589
|
+
|
|
590
|
+
if featurization_pipeline_pt is None:
|
|
591
|
+
featurization_pipeline_pt = model_dir / "featurization_pipeline.pt"
|
|
592
|
+
if not featurization_pipeline_pt.exists():
|
|
593
|
+
raise ValueError(f"No file at {featurization_pipeline_pt=}")
|
|
594
|
+
pipeline = WaveformPipeline.from_state_dict_pt(
|
|
595
|
+
geom, channel_index, featurization_pipeline_pt
|
|
596
|
+
)
|
|
597
|
+
return pipeline, featurization_pipeline_pt
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def get_tpca(sorting, tpca_name="collisioncleaned_tpca_features"):
|
|
601
|
+
"""Look for the TemporalPCAFeaturizer in the usual place."""
|
|
602
|
+
pipeline, _ = get_featurization_pipeline(sorting)
|
|
603
|
+
return pipeline.get_transformer(tpca_name)
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def load_stored_tsvd(sorting, tsvd_name="collisioncleaned_basis", to_sklearn=True):
|
|
607
|
+
from ..transform import BaseTemporalPCA
|
|
608
|
+
|
|
609
|
+
if sorting.parent_h5_path is None:
|
|
610
|
+
logger.info("Couldn't load stored basis.")
|
|
611
|
+
return None
|
|
612
|
+
pipeline, pt_path = get_featurization_pipeline(sorting)
|
|
613
|
+
tsvd = pipeline.get_transformer(tsvd_name)
|
|
614
|
+
assert tsvd is not None
|
|
615
|
+
assert tsvd.name == tsvd_name
|
|
616
|
+
assert isinstance(tsvd, BaseTemporalPCA)
|
|
617
|
+
if to_sklearn:
|
|
618
|
+
tsvd = tsvd.to_sklearn()
|
|
619
|
+
logger.info(
|
|
620
|
+
f"Loaded stored basis from %s (%s; components shape: %s).",
|
|
621
|
+
pt_path,
|
|
622
|
+
tsvd_name,
|
|
623
|
+
tsvd.components_.shape,
|
|
624
|
+
)
|
|
625
|
+
return tsvd
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def get_labels(h5_path) -> np.ndarray:
|
|
629
|
+
with h5py.File(h5_path, "r") as h5:
|
|
630
|
+
return h5["labels"][:] # type: ignore
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def get_residual_snips(h5_path) -> np.ndarray:
|
|
634
|
+
with h5py.File(h5_path, "r", locking=False) as h5:
|
|
635
|
+
return h5["residual"][:] # type: ignore
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def sorting_isis(sorting: DARTsortSorting):
|
|
639
|
+
assert sorting.labels is not None
|
|
640
|
+
isis_ms = np.zeros(len(sorting))
|
|
641
|
+
for uid in sorting.unit_ids:
|
|
642
|
+
inu = np.flatnonzero(sorting.labels == uid)
|
|
643
|
+
t_ms = sorting.times_seconds[inu] * 1000 # type: ignore
|
|
644
|
+
isi = np.diff(t_ms)
|
|
645
|
+
isi = np.concatenate([[np.inf], np.abs(isi), [np.inf]])
|
|
646
|
+
isi = np.minimum(isi[1:], isi[:-1])
|
|
647
|
+
isis_ms[inu] = isi
|
|
648
|
+
return isis_ms
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def keep_only_most_recent_spikes(
|
|
652
|
+
sorting,
|
|
653
|
+
n_min_spikes=250,
|
|
654
|
+
latest_time_sample=90_000_000,
|
|
655
|
+
):
|
|
656
|
+
"""
|
|
657
|
+
This function selects the n_min_spikes before latest_time (or most recent after latest_time)
|
|
658
|
+
"""
|
|
659
|
+
new_labels = np.full(sorting.labels.shape, -1)
|
|
660
|
+
units = np.unique(sorting.labels)
|
|
661
|
+
units = units[units > -1]
|
|
662
|
+
for k in units:
|
|
663
|
+
idx_k = np.flatnonzero(sorting.labels == k)
|
|
664
|
+
before_time = sorting.times_samples[idx_k] < latest_time_sample
|
|
665
|
+
if before_time.sum() <= n_min_spikes:
|
|
666
|
+
idx_k = idx_k[:n_min_spikes]
|
|
667
|
+
new_labels[idx_k] = k
|
|
668
|
+
else:
|
|
669
|
+
idx_k = idx_k[before_time][-n_min_spikes:]
|
|
670
|
+
new_labels[idx_k] = k
|
|
671
|
+
new_sorting = sorting.ephemeral_replace(labels=new_labels)
|
|
672
|
+
return new_sorting
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def check_recording(
|
|
676
|
+
rec,
|
|
677
|
+
threshold=5,
|
|
678
|
+
dedup_spatial_radius=75,
|
|
679
|
+
expected_value_range=1e4,
|
|
680
|
+
expected_spikes_per_sec=10_000,
|
|
681
|
+
num_chunks_per_segment=5,
|
|
682
|
+
dtype=torch.float,
|
|
683
|
+
):
|
|
684
|
+
"""Sanity check spike detection rate and data range of input recording."""
|
|
685
|
+
|
|
686
|
+
# grab random traces from throughout rec
|
|
687
|
+
random_chunks = get_random_data_chunks(
|
|
688
|
+
rec,
|
|
689
|
+
num_chunks_per_segment=num_chunks_per_segment,
|
|
690
|
+
chunk_size=min(rec.get_num_samples(), int(rec.sampling_frequency)),
|
|
691
|
+
concatenated=False,
|
|
692
|
+
)
|
|
693
|
+
dedup_channel_index = None
|
|
694
|
+
if dedup_spatial_radius:
|
|
695
|
+
dedup_channel_index = make_channel_index(
|
|
696
|
+
rec.get_channel_locations(), dedup_spatial_radius
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
# run detection and compute spike detection rate and data range
|
|
700
|
+
spike_rates = []
|
|
701
|
+
for chunk in random_chunks:
|
|
702
|
+
dres = detect_and_deduplicate(
|
|
703
|
+
torch.tensor(chunk, dtype=dtype),
|
|
704
|
+
threshold=threshold,
|
|
705
|
+
peak_sign="both",
|
|
706
|
+
dedup_channel_index=torch.tensor(dedup_channel_index),
|
|
707
|
+
)
|
|
708
|
+
times = dres[0]
|
|
709
|
+
del dres
|
|
710
|
+
chunk_len_s = rec.sampling_frequency / chunk.shape[0]
|
|
711
|
+
spike_rates.append(times.shape[0] / chunk_len_s)
|
|
712
|
+
|
|
713
|
+
avg_detections_per_second = np.mean(spike_rates)
|
|
714
|
+
max_abs = np.max(random_chunks)
|
|
715
|
+
|
|
716
|
+
failed = False
|
|
717
|
+
if avg_detections_per_second > expected_spikes_per_sec:
|
|
718
|
+
warnings.warn(
|
|
719
|
+
f"Detected {avg_detections_per_second:0.1f} spikes/s, which is "
|
|
720
|
+
"large. You may want to check that your data has been preprocessed, "
|
|
721
|
+
"including standardization. If it seems right, then you may need to "
|
|
722
|
+
"shrink the chunk_length_samples parameters in the configuration if "
|
|
723
|
+
"you experience memory issues.",
|
|
724
|
+
RuntimeWarning,
|
|
725
|
+
)
|
|
726
|
+
failed = True
|
|
727
|
+
|
|
728
|
+
if max_abs > expected_value_range:
|
|
729
|
+
warnings.warn(
|
|
730
|
+
f"Recording values exceed |{expected_value_range}|. You may want to "
|
|
731
|
+
"check that your data has been preprocessed, including standardization.",
|
|
732
|
+
RuntimeWarning,
|
|
733
|
+
)
|
|
734
|
+
failed = True
|
|
735
|
+
|
|
736
|
+
return failed, avg_detections_per_second, max_abs
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def subset_sorting_by_spike_count(sorting, min_spikes=0, max_spikes=np.inf):
|
|
740
|
+
if not min_spikes:
|
|
741
|
+
return sorting
|
|
742
|
+
|
|
743
|
+
units, counts = np.unique(sorting.labels, return_counts=True)
|
|
744
|
+
invalid = np.logical_or(counts < min_spikes, counts > max_spikes)
|
|
745
|
+
bad_units = units[invalid]
|
|
746
|
+
|
|
747
|
+
new_labels = np.where(np.isin(sorting.labels, bad_units), -1, sorting.labels)
|
|
748
|
+
|
|
749
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def subsample_to_max_count(
|
|
753
|
+
sorting, max_spikes=256, seed: int | np.random.Generator = 0, discard_triaged=False
|
|
754
|
+
):
|
|
755
|
+
units, counts = np.unique(sorting.labels, return_counts=True)
|
|
756
|
+
if counts.max() <= max_spikes:
|
|
757
|
+
return sorting
|
|
758
|
+
|
|
759
|
+
rg = np.random.default_rng(seed)
|
|
760
|
+
new_labels = sorting.labels.copy()
|
|
761
|
+
for u in units[counts > max_spikes]:
|
|
762
|
+
in_u = np.flatnonzero(sorting.labels == u)
|
|
763
|
+
new_labels[in_u] = -1
|
|
764
|
+
in_u = rg.choice(in_u, size=max_spikes, replace=False)
|
|
765
|
+
new_labels[in_u] = u
|
|
766
|
+
|
|
767
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def restrict_to_valid_times(sorting, recording, waveform_cfg, pad=0):
|
|
771
|
+
trough = waveform_cfg.trough_offset_samples(recording.sampling_frequency)
|
|
772
|
+
total = waveform_cfg.spike_length_samples(recording.sampling_frequency)
|
|
773
|
+
t_min = trough + pad
|
|
774
|
+
t_max = recording.get_total_samples() - (total - trough) - pad
|
|
775
|
+
new_labels = sorting.labels.copy()
|
|
776
|
+
new_labels[sorting.times_samples < t_min] = -1
|
|
777
|
+
new_labels[sorting.times_samples > t_max] = -1
|
|
778
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def subset_sorting_by_time_samples(
|
|
782
|
+
sorting, start_sample=0, end_sample=np.inf, reference_to_start_sample=True
|
|
783
|
+
):
|
|
784
|
+
new_times = sorting.times_samples.copy()
|
|
785
|
+
new_labels = sorting.labels.copy()
|
|
786
|
+
|
|
787
|
+
in_range = (new_times >= start_sample) & (new_times < end_sample)
|
|
788
|
+
new_labels[~in_range] = -1
|
|
789
|
+
|
|
790
|
+
if reference_to_start_sample:
|
|
791
|
+
new_times -= start_sample
|
|
792
|
+
|
|
793
|
+
return sorting.ephemeral_replace(labels=new_labels, times_samples=new_times)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def subset_sorting_by_time_seconds(sorting, t_start=0, t_end=np.inf):
|
|
797
|
+
new_labels = sorting.labels.copy()
|
|
798
|
+
t_s = sorting.times_seconds
|
|
799
|
+
in_range = t_s == t_s.clip(t_start, t_end)
|
|
800
|
+
new_labels[~in_range] = -1
|
|
801
|
+
|
|
802
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def time_chunk_sortings(
|
|
806
|
+
sorting, recording=None, chunk_length_samples=None, chunk_time_ranges_s=None
|
|
807
|
+
):
|
|
808
|
+
if chunk_time_ranges_s is None:
|
|
809
|
+
chunk_time_ranges_s = chunk_time_ranges(recording, chunk_length_samples)
|
|
810
|
+
chunk_sortings = [
|
|
811
|
+
subset_sorting_by_time_seconds(sorting, *tt) for tt in chunk_time_ranges_s
|
|
812
|
+
]
|
|
813
|
+
return chunk_time_ranges_s, chunk_sortings
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def reindex_sorting_labels(sorting):
|
|
817
|
+
new_labels = sorting.labels.copy()
|
|
818
|
+
kept = np.flatnonzero(new_labels >= 0)
|
|
819
|
+
_, new_labels[kept] = np.unique(new_labels[kept], return_inverse=True)
|
|
820
|
+
return sorting.ephemeral_replace(labels=new_labels)
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def combine_sortings(sortings, dodge=False):
|
|
824
|
+
labels = np.full_like(sortings[0].labels, -1)
|
|
825
|
+
times_samples = sortings[0].times_samples.copy()
|
|
826
|
+
assert all(s.labels.size == sortings[0].labels.size for s in sortings)
|
|
827
|
+
|
|
828
|
+
if dodge:
|
|
829
|
+
label_to_sorting_index = []
|
|
830
|
+
label_to_original_label = []
|
|
831
|
+
else:
|
|
832
|
+
label_to_sorting_index = None
|
|
833
|
+
label_to_original_label = None
|
|
834
|
+
|
|
835
|
+
next_label = 0
|
|
836
|
+
for j, sorting in enumerate(sortings):
|
|
837
|
+
kept = np.flatnonzero(sorting.labels >= 0)
|
|
838
|
+
assert np.all(labels[kept] < 0)
|
|
839
|
+
labels[kept] = sorting.labels[kept] + next_label
|
|
840
|
+
if dodge:
|
|
841
|
+
assert label_to_sorting_index is not None
|
|
842
|
+
assert label_to_original_label is not None
|
|
843
|
+
n_new_labels = 0
|
|
844
|
+
if kept.size:
|
|
845
|
+
n_new_labels = 1 + sorting.labels[kept].max()
|
|
846
|
+
next_label += n_new_labels
|
|
847
|
+
label_to_sorting_index.append(np.full(n_new_labels, j))
|
|
848
|
+
label_to_original_label.append(np.arange(n_new_labels))
|
|
849
|
+
times_samples[kept] = sorting.times_samples[kept]
|
|
850
|
+
|
|
851
|
+
sorting = sortings[0].ephemeral_replace(labels=labels, times_samples=times_samples)
|
|
852
|
+
|
|
853
|
+
if dodge:
|
|
854
|
+
assert label_to_sorting_index is not None
|
|
855
|
+
assert label_to_original_label is not None
|
|
856
|
+
label_to_sorting_index = np.concatenate(label_to_sorting_index)
|
|
857
|
+
label_to_original_label = np.concatenate(label_to_original_label)
|
|
858
|
+
return label_to_sorting_index, label_to_original_label, sorting
|
|
859
|
+
return sorting
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
# -- timing
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def chunk_time_ranges(recording, chunk_length_samples=None):
|
|
866
|
+
if chunk_length_samples is None or chunk_length_samples == np.inf:
|
|
867
|
+
n_chunks = 1
|
|
868
|
+
else:
|
|
869
|
+
n_chunks = recording.get_num_samples() / chunk_length_samples
|
|
870
|
+
# we'll count the remainder as a chunk if it's at least 2/3 of one
|
|
871
|
+
n_chunks = np.floor(n_chunks) + (n_chunks - np.floor(n_chunks) > 0.66)
|
|
872
|
+
n_chunks = int(max(1, n_chunks))
|
|
873
|
+
|
|
874
|
+
# evenly divide the recording into chunks
|
|
875
|
+
assert recording.get_num_segments() == 1
|
|
876
|
+
start_time_s, end_time_s = recording._recording_segments[0].sample_index_to_time(
|
|
877
|
+
np.array([0, recording.get_num_samples() - 1])
|
|
878
|
+
)
|
|
879
|
+
chunk_times_s = np.linspace(start_time_s, end_time_s, num=n_chunks + 1)
|
|
880
|
+
chunk_time_ranges_s = list(zip(chunk_times_s[:-1], chunk_times_s[1:]))
|
|
881
|
+
|
|
882
|
+
return chunk_time_ranges_s
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
# -- hdf5 util
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def batched_h5_read(dataset, indices=None, mask=None, show_progress=False):
|
|
889
|
+
if mask is None:
|
|
890
|
+
mask = np.zeros(len(dataset), dtype=bool)
|
|
891
|
+
mask[indices] = 1
|
|
892
|
+
return _read_by_chunk(mask, dataset, show_progress=show_progress)
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def _read_by_chunk(mask, dataset, show_progress=True):
|
|
896
|
+
"""
|
|
897
|
+
mask : boolean array of shape dataset.shape[:1]
|
|
898
|
+
dataset : chunked h5py.Dataset
|
|
899
|
+
"""
|
|
900
|
+
out = np.empty((mask.sum(), *dataset.shape[1:]), dtype=dataset.dtype)
|
|
901
|
+
n = 0
|
|
902
|
+
for sli, dsli in yield_chunks(dataset, show_progress):
|
|
903
|
+
m = np.flatnonzero(mask[sli])
|
|
904
|
+
nm = m.size
|
|
905
|
+
if not nm:
|
|
906
|
+
continue
|
|
907
|
+
x = dsli[m]
|
|
908
|
+
out[n : n + nm] = x
|
|
909
|
+
n += nm
|
|
910
|
+
return out
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def yield_chunks(
|
|
914
|
+
dataset, show_progress=True, desc_prefix=None, fallback_chunk_length=4096
|
|
915
|
+
) -> Generator[tuple[slice, np.ndarray], None, None]:
|
|
916
|
+
"""Iterate chunks of an h5py dataset
|
|
917
|
+
|
|
918
|
+
The dataset can either not be chunked or chunked only on the first axis.
|
|
919
|
+
"""
|
|
920
|
+
if dataset.chunks is None:
|
|
921
|
+
chunks = (
|
|
922
|
+
slice(s, min(s + fallback_chunk_length, len(dataset)))
|
|
923
|
+
for s in range(0, len(dataset), fallback_chunk_length)
|
|
924
|
+
)
|
|
925
|
+
else:
|
|
926
|
+
for c, s in zip(dataset.chunks[1:], dataset.shape[1:]):
|
|
927
|
+
if c == s:
|
|
928
|
+
continue
|
|
929
|
+
raise ValueError(
|
|
930
|
+
f"Dataset {dataset} can only be chunked on the first axis. "
|
|
931
|
+
f"Found {dataset.chunks=} with {dataset.shape=}."
|
|
932
|
+
)
|
|
933
|
+
|
|
934
|
+
chunks = dataset.iter_chunks()
|
|
935
|
+
# throw away slices along other axes
|
|
936
|
+
chunks = (chunk[0] for chunk in chunks)
|
|
937
|
+
|
|
938
|
+
if show_progress:
|
|
939
|
+
desc = dataset.name
|
|
940
|
+
if desc_prefix:
|
|
941
|
+
desc = f"{desc_prefix} {desc}"
|
|
942
|
+
if dataset.chunks is None:
|
|
943
|
+
n_chunks = int(np.ceil(dataset.shape[0] / fallback_chunk_length))
|
|
944
|
+
else:
|
|
945
|
+
n_chunks = int(np.ceil(dataset.shape[0] / dataset.chunks[0]))
|
|
946
|
+
chunks = tqdm(chunks, total=n_chunks, desc=desc)
|
|
947
|
+
|
|
948
|
+
for sli in chunks:
|
|
949
|
+
yield sli, dataset[sli]
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def yield_masked_chunks(mask, dataset, show_progress=True, desc_prefix=None):
|
|
953
|
+
offset = 0
|
|
954
|
+
for sli, data in yield_chunks(
|
|
955
|
+
dataset, show_progress=show_progress, desc_prefix=desc_prefix
|
|
956
|
+
):
|
|
957
|
+
source_ixs = np.flatnonzero(mask[sli])
|
|
958
|
+
dest_ixs = slice(offset, offset + source_ixs.size)
|
|
959
|
+
yield dest_ixs, data[source_ixs]
|
|
960
|
+
offset += source_ixs.size
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
# -- residual
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def extract_random_snips(
|
|
967
|
+
rg: int | np.random.Generator,
|
|
968
|
+
chunk: np.ndarray | torch.Tensor,
|
|
969
|
+
n: int,
|
|
970
|
+
sniplen: int,
|
|
971
|
+
):
|
|
972
|
+
"""Grab n (or as many as can fit) random non-overlapping snips from chunk."""
|
|
973
|
+
rg = np.random.default_rng(rg)
|
|
974
|
+
|
|
975
|
+
# we can extract at most this many snips
|
|
976
|
+
n = min(n, max(1, chunk.shape[0] // sniplen))
|
|
977
|
+
|
|
978
|
+
# how many samples will not be covered
|
|
979
|
+
empty_len = chunk.shape[0] - sniplen * n
|
|
980
|
+
assert empty_len >= 0
|
|
981
|
+
|
|
982
|
+
# sample points spaced by sniplen + deltaj, where sum of
|
|
983
|
+
# delta is <= empty_len. use an extra slack diff for the <.
|
|
984
|
+
if empty_len == 0:
|
|
985
|
+
delta = np.zeros((n + 1,), dtype=np.int64)
|
|
986
|
+
else:
|
|
987
|
+
delta = rg.integers(0, empty_len, size=n + 1)
|
|
988
|
+
delta.sort()
|
|
989
|
+
delta = np.diff(delta)[:n]
|
|
990
|
+
times = np.cumsum(delta + sniplen) - sniplen
|
|
991
|
+
tixs = times[:, None] + np.arange(sniplen)
|
|
992
|
+
return chunk[tixs], times
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
# -- dataset subsampling
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def subsample_waveforms(
|
|
999
|
+
hdf5_filename: str | Path | None = None,
|
|
1000
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
1001
|
+
random_state: int | np.random.Generator = 0,
|
|
1002
|
+
n_waveforms_fit=10_000,
|
|
1003
|
+
voltages_dataset_name="collisioncleaned_voltages",
|
|
1004
|
+
waveforms_dataset_name="collisioncleaned_waveforms",
|
|
1005
|
+
fit_max_reweighting=4.0,
|
|
1006
|
+
log_voltages=True,
|
|
1007
|
+
subsample_by_weighting=False,
|
|
1008
|
+
fixed_property_keys=("channels",),
|
|
1009
|
+
replace=True,
|
|
1010
|
+
h5=None,
|
|
1011
|
+
device: torch.device | str = "cpu",
|
|
1012
|
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
|
1013
|
+
random_state = np.random.default_rng(random_state)
|
|
1014
|
+
|
|
1015
|
+
need_open = h5 is None
|
|
1016
|
+
if need_open and hdf5_filename is not None:
|
|
1017
|
+
hdf5_filename = resolve_path(hdf5_filename, strict=True)
|
|
1018
|
+
h5 = h5py.File(hdf5_filename)
|
|
1019
|
+
elif need_open:
|
|
1020
|
+
raise ValueError("Need h5 or hdf5_filename.")
|
|
1021
|
+
|
|
1022
|
+
try:
|
|
1023
|
+
channels: np.ndarray = h5["channels"][:] # type: ignore
|
|
1024
|
+
n_wf = channels.shape[0]
|
|
1025
|
+
if not n_wf:
|
|
1026
|
+
emptyi = torch.tensor([], dtype=torch.long)
|
|
1027
|
+
wfshape = h5[waveforms_dataset_name].shape # type: ignore
|
|
1028
|
+
emptywf = torch.zeros(wfshape)
|
|
1029
|
+
return emptywf, dict(channels=emptyi)
|
|
1030
|
+
weights = fit_reweighting(
|
|
1031
|
+
h5=h5,
|
|
1032
|
+
log_voltages=log_voltages,
|
|
1033
|
+
fit_sampling=fit_sampling,
|
|
1034
|
+
fit_max_reweighting=fit_max_reweighting,
|
|
1035
|
+
voltages_dataset_name=voltages_dataset_name,
|
|
1036
|
+
)
|
|
1037
|
+
if n_wf > n_waveforms_fit and not subsample_by_weighting:
|
|
1038
|
+
choices = random_state.choice(
|
|
1039
|
+
n_wf, p=weights, size=n_waveforms_fit, replace=replace
|
|
1040
|
+
)
|
|
1041
|
+
if not replace:
|
|
1042
|
+
choices.sort()
|
|
1043
|
+
waveforms = batched_h5_read(h5[waveforms_dataset_name], choices)
|
|
1044
|
+
fixed_properties = {k: h5[k][choices] for k in fixed_property_keys} # type: ignore
|
|
1045
|
+
else:
|
|
1046
|
+
uchoices, ichoices = np.unique(choices, return_inverse=True)
|
|
1047
|
+
waveforms = batched_h5_read(h5[waveforms_dataset_name], uchoices)[
|
|
1048
|
+
ichoices
|
|
1049
|
+
]
|
|
1050
|
+
fixed_properties = {
|
|
1051
|
+
k: h5[k][uchoices][ichoices] # type: ignore
|
|
1052
|
+
for k in fixed_property_keys
|
|
1053
|
+
}
|
|
1054
|
+
else:
|
|
1055
|
+
waveforms: np.ndarray = h5[waveforms_dataset_name][:] # type: ignore
|
|
1056
|
+
fixed_properties = {k: h5[k][:] for k in fixed_property_keys} # type: ignore
|
|
1057
|
+
finally:
|
|
1058
|
+
if need_open:
|
|
1059
|
+
h5.close()
|
|
1060
|
+
del h5
|
|
1061
|
+
|
|
1062
|
+
device = torch.device(device)
|
|
1063
|
+
waveformsr = torch.as_tensor(waveforms, device=device)
|
|
1064
|
+
fixed_properties = {
|
|
1065
|
+
k: torch.as_tensor(v, device=device) for k, v in fixed_properties.items()
|
|
1066
|
+
}
|
|
1067
|
+
if subsample_by_weighting and weights is not None:
|
|
1068
|
+
fixed_properties["weights"] = torch.as_tensor(weights, device=device)
|
|
1069
|
+
elif subsample_by_weighting:
|
|
1070
|
+
fixed_properties["weights"] = torch.ones(waveforms.shape[0], device=device)
|
|
1071
|
+
|
|
1072
|
+
return waveformsr, fixed_properties
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def fit_reweighting(
|
|
1076
|
+
voltages: np.ndarray | None = None, # type: ignore
|
|
1077
|
+
h5=None,
|
|
1078
|
+
hdf5_path=None,
|
|
1079
|
+
log_voltages=True,
|
|
1080
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
1081
|
+
fit_max_reweighting=4.0,
|
|
1082
|
+
voltages_dataset_name="voltages",
|
|
1083
|
+
):
|
|
1084
|
+
if fit_sampling == "random":
|
|
1085
|
+
return None
|
|
1086
|
+
assert fit_sampling == "amp_reweighted"
|
|
1087
|
+
|
|
1088
|
+
if voltages is None:
|
|
1089
|
+
if h5 is not None:
|
|
1090
|
+
voltages = h5[voltages_dataset_name][:]
|
|
1091
|
+
elif hdf5_path is not None:
|
|
1092
|
+
with h5py.File(hdf5_path) as h5:
|
|
1093
|
+
voltages: np.ndarray = h5[voltages_dataset_name][:] # type: ignore
|
|
1094
|
+
else:
|
|
1095
|
+
assert False
|
|
1096
|
+
assert isinstance(voltages, np.ndarray)
|
|
1097
|
+
|
|
1098
|
+
from ..clustering.density import get_smoothed_density
|
|
1099
|
+
|
|
1100
|
+
if torch.is_tensor(voltages):
|
|
1101
|
+
voltages = voltages.numpy(force=True)
|
|
1102
|
+
if log_voltages:
|
|
1103
|
+
sign = np.sign(voltages)
|
|
1104
|
+
voltages = sign * np.log(np.abs(voltages))
|
|
1105
|
+
voltages = np.nan_to_num(voltages)
|
|
1106
|
+
sigma = 1.06 * voltages.std() * np.power(len(voltages), -0.2)
|
|
1107
|
+
assert np.isfinite(sigma)
|
|
1108
|
+
dens = get_smoothed_density(voltages[:, None], sigma=sigma)
|
|
1109
|
+
assert isinstance(dens, np.ndarray)
|
|
1110
|
+
sample_p = dens.mean() / dens
|
|
1111
|
+
sample_p = sample_p.clip(1.0 / fit_max_reweighting, fit_max_reweighting)
|
|
1112
|
+
sample_p = sample_p.astype(np.float64) # ensure double before normalizing
|
|
1113
|
+
sample_p /= sample_p.sum()
|
|
1114
|
+
return sample_p
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def divide_randomly(n_things, n_bins, rg):
|
|
1118
|
+
things_per_bin = np.zeros(n_bins, dtype=np.int64)
|
|
1119
|
+
n_even_split = n_things // n_bins
|
|
1120
|
+
things_per_bin += n_even_split
|
|
1121
|
+
n_things_remaining = n_things - n_bins * n_even_split
|
|
1122
|
+
assert n_things_remaining >= 0
|
|
1123
|
+
if n_things_remaining:
|
|
1124
|
+
rg = np.random.default_rng(rg)
|
|
1125
|
+
choices = rg.choice(n_bins, size=n_things_remaining)
|
|
1126
|
+
np.add.at(things_per_bin, choices, 1)
|
|
1127
|
+
assert things_per_bin.sum() == n_things
|
|
1128
|
+
return things_per_bin
|