DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
"""Deeper object-oriented interaction with sorter data
|
|
2
|
+
|
|
3
|
+
This is meant to make implementing plotting code easier: this
|
|
4
|
+
code becomes the model in a MVC framework, and vis/unit.py can
|
|
5
|
+
implement a view and controller.
|
|
6
|
+
|
|
7
|
+
This should also make it easier to compute drift-aware metrics
|
|
8
|
+
(e.g., d' using registered templates and shifted waveforms).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import spikeinterface.core as sc
|
|
15
|
+
import torch
|
|
16
|
+
from dredge.motion_util import MotionEstimate
|
|
17
|
+
from sklearn.decomposition import PCA
|
|
18
|
+
|
|
19
|
+
from ..clustering import merge
|
|
20
|
+
from ..util.internal_config import (
|
|
21
|
+
TemplateConfig,
|
|
22
|
+
TemplateMergeConfig,
|
|
23
|
+
ComputationConfig,
|
|
24
|
+
ClusteringFeaturesConfig,
|
|
25
|
+
default_clustering_features_cfg,
|
|
26
|
+
)
|
|
27
|
+
from ..templates import TemplateData
|
|
28
|
+
from ..util.data_util import (
|
|
29
|
+
DARTsortSorting,
|
|
30
|
+
get_tpca,
|
|
31
|
+
try_get_model_dir,
|
|
32
|
+
sorting_isis,
|
|
33
|
+
)
|
|
34
|
+
from ..util.drift_util import (
|
|
35
|
+
get_spike_pitch_shifts,
|
|
36
|
+
get_waveforms_on_static_channels,
|
|
37
|
+
get_stable_channels,
|
|
38
|
+
)
|
|
39
|
+
from ..util.interpolation_util import StableFeaturesInterpolator, pad_geom
|
|
40
|
+
from ..util.py_util import databag
|
|
41
|
+
from ..util.spikeio import read_waveforms_channel_index
|
|
42
|
+
from ..util.waveform_util import make_channel_index
|
|
43
|
+
from ..util import job_util, logging_util
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
logger = logging_util.get_logger(__name__)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class DARTsortAnalysis:
|
|
51
|
+
"""Stores all relevant properties for a drift-aware waveform analysis"""
|
|
52
|
+
|
|
53
|
+
sorting: DARTsortSorting
|
|
54
|
+
recording: sc.BaseRecording
|
|
55
|
+
template_data: TemplateData | None
|
|
56
|
+
coarse_template_data: TemplateData | None
|
|
57
|
+
motion_est: MotionEstimate | None
|
|
58
|
+
merge_distances: np.ndarray | None
|
|
59
|
+
geom: np.ndarray
|
|
60
|
+
registered_geom: np.ndarray
|
|
61
|
+
extract_channel_index: np.ndarray | None
|
|
62
|
+
vis_channel_index: np.ndarray
|
|
63
|
+
xyza: np.ndarray | None
|
|
64
|
+
x: np.ndarray | None
|
|
65
|
+
z: np.ndarray | None
|
|
66
|
+
registered_z: np.ndarray | None
|
|
67
|
+
shifting: bool
|
|
68
|
+
times_seconds: np.ndarray | None
|
|
69
|
+
amplitudes: np.ndarray | None
|
|
70
|
+
unit_ids: np.ndarray
|
|
71
|
+
spike_counts: np.ndarray
|
|
72
|
+
amplitude_vectors: np.ndarray | None
|
|
73
|
+
erp: StableFeaturesInterpolator | None
|
|
74
|
+
sklearn_tpca: PCA | None
|
|
75
|
+
tpca_temporal_slice: slice
|
|
76
|
+
name: str | None = None
|
|
77
|
+
vis_radius: float = 50.0
|
|
78
|
+
trough_offset_samples: int = 42
|
|
79
|
+
spike_length_samples: int = 121
|
|
80
|
+
tpca_features_dset: str = "collisioncleaned_tpca_features"
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def from_sorting(
|
|
84
|
+
cls,
|
|
85
|
+
recording: sc.BaseRecording,
|
|
86
|
+
sorting: DARTsortSorting,
|
|
87
|
+
motion_est=None,
|
|
88
|
+
name: str | None = None,
|
|
89
|
+
template_data: TemplateData | None = None,
|
|
90
|
+
template_cfg: TemplateConfig | None = TemplateConfig(denoising_method="none"),
|
|
91
|
+
template_merge_cfg: TemplateMergeConfig = TemplateMergeConfig(
|
|
92
|
+
min_spatial_cosine=0.8
|
|
93
|
+
),
|
|
94
|
+
clustering_features_cfg: ClusteringFeaturesConfig = default_clustering_features_cfg,
|
|
95
|
+
computation_cfg: ComputationConfig | None = None,
|
|
96
|
+
vis_radius: float = 50.0,
|
|
97
|
+
vis_neighborhood_p: float = np.inf,
|
|
98
|
+
):
|
|
99
|
+
"""Try to re-load as much info as possible from the sorting itself
|
|
100
|
+
|
|
101
|
+
Templates are re-computed if labels are not the same as in h5
|
|
102
|
+
or if the template npz does not exist.
|
|
103
|
+
"""
|
|
104
|
+
computation_cfg = job_util.ensure_computation_config(computation_cfg)
|
|
105
|
+
has_hdf5 = sorting.parent_h5_path is not None
|
|
106
|
+
|
|
107
|
+
if has_hdf5 and vis_radius and (tpca := get_tpca(sorting)) is not None:
|
|
108
|
+
sklearn_tpca = tpca.to_sklearn() # type: ignore
|
|
109
|
+
tpca_temporal_slice = sklearn_tpca.temporal_slice
|
|
110
|
+
else:
|
|
111
|
+
sklearn_tpca = None
|
|
112
|
+
tpca_temporal_slice = slice(None)
|
|
113
|
+
|
|
114
|
+
model_dir = try_get_model_dir(sorting)
|
|
115
|
+
if template_data is not None:
|
|
116
|
+
pass
|
|
117
|
+
elif has_hdf5 and model_dir is not None:
|
|
118
|
+
template_npz = model_dir / "template_data.npz"
|
|
119
|
+
can_reload = sorting.has_persistent_labels()
|
|
120
|
+
if can_reload and template_npz.exists():
|
|
121
|
+
logger.info(f"Reloading templates from {template_npz}...")
|
|
122
|
+
template_data = TemplateData.from_npz(template_npz)
|
|
123
|
+
|
|
124
|
+
if template_data is None and template_cfg is not None:
|
|
125
|
+
template_data = TemplateData.from_config(
|
|
126
|
+
recording,
|
|
127
|
+
sorting,
|
|
128
|
+
template_cfg,
|
|
129
|
+
overwrite=False,
|
|
130
|
+
motion_est=motion_est,
|
|
131
|
+
computation_cfg=computation_cfg,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if template_data is not None:
|
|
135
|
+
coarse_template_data = template_data.coarsen()
|
|
136
|
+
merge_distances = merge.get_merge_distances(
|
|
137
|
+
template_data=coarse_template_data,
|
|
138
|
+
template_merge_cfg=template_merge_cfg,
|
|
139
|
+
computation_cfg=computation_cfg,
|
|
140
|
+
sampling_frequency=recording.sampling_frequency,
|
|
141
|
+
)[1]
|
|
142
|
+
trough_offset_samples = template_data.trough_offset_samples
|
|
143
|
+
spike_length_samples = template_data.spike_length_samples
|
|
144
|
+
else:
|
|
145
|
+
trough_offset_samples = spike_length_samples = 0
|
|
146
|
+
coarse_template_data = merge_distances = None
|
|
147
|
+
|
|
148
|
+
channel_index = getattr(sorting, "channel_index", None)
|
|
149
|
+
amplitudes = getattr(
|
|
150
|
+
sorting, clustering_features_cfg.amplitudes_dataset_name, None
|
|
151
|
+
)
|
|
152
|
+
amplitude_vecs = getattr(
|
|
153
|
+
sorting, clustering_features_cfg.amplitude_vectors_dataset_name, None
|
|
154
|
+
)
|
|
155
|
+
xyza = getattr(
|
|
156
|
+
sorting, clustering_features_cfg.localizations_dataset_name, None
|
|
157
|
+
)
|
|
158
|
+
tpca_features_dset = clustering_features_cfg.pca_dataset_name
|
|
159
|
+
times_seconds = getattr(sorting, "times_seconds", None)
|
|
160
|
+
assert times_seconds is not None
|
|
161
|
+
if motion_est is None and xyza is not None:
|
|
162
|
+
reg_z = xyza[:, 2]
|
|
163
|
+
elif xyza is not None:
|
|
164
|
+
assert motion_est is not None
|
|
165
|
+
reg_z = motion_est.correct_s(times_seconds, xyza[:, 2])
|
|
166
|
+
else:
|
|
167
|
+
reg_z = None
|
|
168
|
+
if xyza is not None:
|
|
169
|
+
x = xyza[:, 0]
|
|
170
|
+
z = xyza[:, 2]
|
|
171
|
+
else:
|
|
172
|
+
x = z = None
|
|
173
|
+
|
|
174
|
+
geom = recording.get_channel_locations()
|
|
175
|
+
if motion_est is None:
|
|
176
|
+
rgeom = geom
|
|
177
|
+
if template_data is not None:
|
|
178
|
+
trg = template_data.registered_geom
|
|
179
|
+
assert (trg is None) or np.array_equal(rgeom, trg)
|
|
180
|
+
else:
|
|
181
|
+
assert template_data is not None
|
|
182
|
+
rgeom = template_data.registered_geom
|
|
183
|
+
assert rgeom is not None
|
|
184
|
+
|
|
185
|
+
device = computation_cfg.actual_device()
|
|
186
|
+
if vis_radius and channel_index is not None:
|
|
187
|
+
# interping to geom with shifts on fly rather than rgeom.
|
|
188
|
+
erp = StableFeaturesInterpolator(
|
|
189
|
+
source_geom=pad_geom(geom, device=device),
|
|
190
|
+
target_geom=pad_geom(geom, device=device),
|
|
191
|
+
channel_index=torch.asarray(channel_index, device=device),
|
|
192
|
+
params=clustering_features_cfg.interp_params,
|
|
193
|
+
)
|
|
194
|
+
else:
|
|
195
|
+
erp = None
|
|
196
|
+
|
|
197
|
+
unit_ids, spike_counts = np.unique(sorting.labels, return_counts=True) # type: ignore
|
|
198
|
+
spike_counts = spike_counts[unit_ids >= 0]
|
|
199
|
+
unit_ids = unit_ids[unit_ids >= 0]
|
|
200
|
+
|
|
201
|
+
return cls(
|
|
202
|
+
sorting=sorting,
|
|
203
|
+
recording=recording,
|
|
204
|
+
template_data=template_data,
|
|
205
|
+
coarse_template_data=coarse_template_data,
|
|
206
|
+
motion_est=motion_est,
|
|
207
|
+
merge_distances=merge_distances,
|
|
208
|
+
geom=geom,
|
|
209
|
+
registered_geom=rgeom,
|
|
210
|
+
extract_channel_index=channel_index,
|
|
211
|
+
vis_channel_index=make_channel_index(
|
|
212
|
+
geom=rgeom, radius=vis_radius, p=vis_neighborhood_p, to_torch=False
|
|
213
|
+
),
|
|
214
|
+
xyza=xyza,
|
|
215
|
+
x=x,
|
|
216
|
+
z=z,
|
|
217
|
+
registered_z=reg_z,
|
|
218
|
+
shifting=motion_est is not None,
|
|
219
|
+
times_seconds=times_seconds,
|
|
220
|
+
amplitudes=amplitudes,
|
|
221
|
+
amplitude_vectors=amplitude_vecs,
|
|
222
|
+
erp=erp,
|
|
223
|
+
sklearn_tpca=sklearn_tpca,
|
|
224
|
+
tpca_temporal_slice=tpca_temporal_slice,
|
|
225
|
+
name=name,
|
|
226
|
+
vis_radius=vis_radius,
|
|
227
|
+
trough_offset_samples=trough_offset_samples,
|
|
228
|
+
spike_length_samples=spike_length_samples,
|
|
229
|
+
tpca_features_dset=tpca_features_dset,
|
|
230
|
+
unit_ids=unit_ids,
|
|
231
|
+
spike_counts=spike_counts,
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
def in_unit(self, unit_id):
|
|
235
|
+
assert self.sorting.labels is not None
|
|
236
|
+
return np.flatnonzero(np.isin(self.sorting.labels, unit_id))
|
|
237
|
+
|
|
238
|
+
def in_template(self, template_index):
|
|
239
|
+
template_indices = getattr(self.sorting, "template_inds", None)
|
|
240
|
+
assert template_indices is not None
|
|
241
|
+
return np.flatnonzero(np.isin(template_indices, template_index))
|
|
242
|
+
|
|
243
|
+
def unit_template_indices(self, unit_id):
|
|
244
|
+
assert self.template_data is not None
|
|
245
|
+
return np.flatnonzero(self.template_data.unit_ids == unit_id)
|
|
246
|
+
|
|
247
|
+
def unit_amplitudes(self, unit_id=None):
|
|
248
|
+
assert self.template_data is not None
|
|
249
|
+
if unit_id is not None:
|
|
250
|
+
return np.ptp(self.template_data.unit_templates(unit_id))
|
|
251
|
+
amplitudes = np.zeros(self.sorting.unit_ids.shape)
|
|
252
|
+
for j, unit_id in enumerate(self.sorting.unit_ids):
|
|
253
|
+
temps = self.template_data.unit_templates(unit_id)
|
|
254
|
+
amplitudes[j] = np.ptp(np.nan_to_num(temps))
|
|
255
|
+
return amplitudes
|
|
256
|
+
|
|
257
|
+
def firing_rates(self):
|
|
258
|
+
assert self.sorting.labels is not None
|
|
259
|
+
assert np.array_equal(self.unit_ids, self.sorting.unit_ids)
|
|
260
|
+
frs = self.spike_counts / self.recording.get_duration()
|
|
261
|
+
return frs
|
|
262
|
+
|
|
263
|
+
def named_feature(self, fname: str, which: slice | np.ndarray):
|
|
264
|
+
if hasattr(self, fname):
|
|
265
|
+
return getattr(self, fname)[which]
|
|
266
|
+
else:
|
|
267
|
+
return self.sorting.slice_feature_by_name(fname, mask=which)
|
|
268
|
+
|
|
269
|
+
# cluster-dependent feature loading methods
|
|
270
|
+
|
|
271
|
+
def unit_raw_waveforms(
|
|
272
|
+
self,
|
|
273
|
+
unit_id: int | None = None,
|
|
274
|
+
which: np.ndarray | None = None,
|
|
275
|
+
template_index: int | None = None,
|
|
276
|
+
max_count=250,
|
|
277
|
+
main_channel=None,
|
|
278
|
+
random_seed=0,
|
|
279
|
+
dtype=np.float32,
|
|
280
|
+
) -> "WaveformsBag | None":
|
|
281
|
+
"""Load raw waveforms for visualization"""
|
|
282
|
+
if which is not None:
|
|
283
|
+
pass
|
|
284
|
+
elif template_index is not None:
|
|
285
|
+
assert template_index in self.unit_template_indices(unit_id)
|
|
286
|
+
which = self.in_template(template_index)
|
|
287
|
+
elif unit_id is not None:
|
|
288
|
+
which = self.in_unit(unit_id)
|
|
289
|
+
else:
|
|
290
|
+
assert False
|
|
291
|
+
|
|
292
|
+
if max_count is not None and which.size > max_count:
|
|
293
|
+
rg = np.random.default_rng(random_seed)
|
|
294
|
+
which = rg.choice(which, size=max_count, replace=False)
|
|
295
|
+
which.sort()
|
|
296
|
+
|
|
297
|
+
times = self.sorting.times_samples[which]
|
|
298
|
+
mnt = self.trough_offset_samples
|
|
299
|
+
mxt = self.recording.get_num_samples() - self.spike_length_samples
|
|
300
|
+
which = which[times == times.clip(mnt, mxt)]
|
|
301
|
+
|
|
302
|
+
if not which.size:
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
# read waveforms from disk
|
|
306
|
+
read_chans = self.sorting.channels[which]
|
|
307
|
+
if self.extract_channel_index is None:
|
|
308
|
+
read_channel_index = self.vis_channel_index
|
|
309
|
+
else:
|
|
310
|
+
read_channel_index = self.extract_channel_index
|
|
311
|
+
waveforms = read_waveforms_channel_index(
|
|
312
|
+
recording=self.recording,
|
|
313
|
+
times_samples=self.sorting.times_samples[which],
|
|
314
|
+
channel_index=read_channel_index,
|
|
315
|
+
main_channels=read_chans,
|
|
316
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
317
|
+
spike_length_samples=self.spike_length_samples,
|
|
318
|
+
fill_value=np.nan,
|
|
319
|
+
)
|
|
320
|
+
waveforms = waveforms.astype(dtype)
|
|
321
|
+
|
|
322
|
+
waveforms, main_channel = self.unit_select_channels(
|
|
323
|
+
unit_id=unit_id,
|
|
324
|
+
which=which,
|
|
325
|
+
waveforms=waveforms,
|
|
326
|
+
read_chans=read_chans,
|
|
327
|
+
main_channel=main_channel,
|
|
328
|
+
)
|
|
329
|
+
return WaveformsBag(
|
|
330
|
+
which=which,
|
|
331
|
+
waveforms=waveforms,
|
|
332
|
+
main_channel=main_channel,
|
|
333
|
+
geom=self.registered_geom,
|
|
334
|
+
channel_index=self.vis_channel_index,
|
|
335
|
+
temporal_slice=None,
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
def tpca_features(self, which: np.ndarray):
|
|
339
|
+
assert self.erp is not None
|
|
340
|
+
features = self.sorting.slice_feature_by_name(
|
|
341
|
+
self.tpca_features_dset, mask=which
|
|
342
|
+
)
|
|
343
|
+
device = self.erp.b.source_geom.device
|
|
344
|
+
channels = torch.asarray(self.sorting.channels[which], device=device)
|
|
345
|
+
if self.motion_est is None:
|
|
346
|
+
shifts = torch.zeros(channels.shape, device=device)
|
|
347
|
+
else:
|
|
348
|
+
assert self.z is not None
|
|
349
|
+
assert self.registered_z is not None
|
|
350
|
+
# target will be geom - shift
|
|
351
|
+
# want to move from original position to reg pos = z - disp
|
|
352
|
+
# so, should take shift=disp=z-reg_z
|
|
353
|
+
shifts = self.z[which] - self.registered_z[which]
|
|
354
|
+
shifts = torch.asarray(shifts, device=device).float()
|
|
355
|
+
features = self.erp.interp(
|
|
356
|
+
features=torch.asarray(features, device=device),
|
|
357
|
+
source_main_channels=channels,
|
|
358
|
+
target_channels=self.erp.b.channel_index[channels],
|
|
359
|
+
source_shifts=shifts,
|
|
360
|
+
)
|
|
361
|
+
return features.numpy(force=True)
|
|
362
|
+
|
|
363
|
+
def unit_tpca_waveforms(
|
|
364
|
+
self,
|
|
365
|
+
unit_id,
|
|
366
|
+
which: np.ndarray | None = None,
|
|
367
|
+
template_index=None,
|
|
368
|
+
max_count=250,
|
|
369
|
+
random_seed=0,
|
|
370
|
+
) -> "WaveformsBag | None":
|
|
371
|
+
assert self.sklearn_tpca is not None
|
|
372
|
+
if which is not None:
|
|
373
|
+
pass
|
|
374
|
+
elif template_index is not None:
|
|
375
|
+
assert template_index in self.unit_template_indices(unit_id)
|
|
376
|
+
which = self.in_template(template_index)
|
|
377
|
+
elif unit_id is not None:
|
|
378
|
+
which = self.in_unit(unit_id)
|
|
379
|
+
else:
|
|
380
|
+
assert False
|
|
381
|
+
|
|
382
|
+
if max_count is not None and which.size > max_count:
|
|
383
|
+
rg = np.random.default_rng(random_seed)
|
|
384
|
+
which = rg.choice(which, size=max_count, replace=False)
|
|
385
|
+
which.sort()
|
|
386
|
+
|
|
387
|
+
if not which.size:
|
|
388
|
+
return None
|
|
389
|
+
|
|
390
|
+
tpca_embeds = self.tpca_features(which=which)
|
|
391
|
+
n, rank, c = tpca_embeds.shape
|
|
392
|
+
tpca_embeds = tpca_embeds.transpose(0, 2, 1).reshape(n * c, rank)
|
|
393
|
+
waveforms = np.full(
|
|
394
|
+
(n * c, self.sklearn_tpca.components_.shape[1]),
|
|
395
|
+
np.nan,
|
|
396
|
+
dtype=tpca_embeds.dtype,
|
|
397
|
+
)
|
|
398
|
+
valid = np.flatnonzero(np.isfinite(tpca_embeds[:, 0]))
|
|
399
|
+
waveforms[valid] = self.sklearn_tpca.inverse_transform(tpca_embeds[valid])
|
|
400
|
+
t = waveforms.shape[1]
|
|
401
|
+
waveforms = waveforms.reshape(n, c, t).transpose(0, 2, 1)
|
|
402
|
+
|
|
403
|
+
waveforms, main_channel = self.unit_select_channels(
|
|
404
|
+
unit_id=unit_id, which=which, waveforms=waveforms
|
|
405
|
+
)
|
|
406
|
+
return WaveformsBag(
|
|
407
|
+
which=which,
|
|
408
|
+
waveforms=waveforms,
|
|
409
|
+
main_channel=main_channel,
|
|
410
|
+
geom=self.registered_geom,
|
|
411
|
+
channel_index=self.vis_channel_index,
|
|
412
|
+
temporal_slice=self.tpca_temporal_slice,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
def unit_pca_features(
|
|
416
|
+
self,
|
|
417
|
+
unit_id,
|
|
418
|
+
rank=2,
|
|
419
|
+
random_seed=0,
|
|
420
|
+
max_count=500,
|
|
421
|
+
max_wfs_fit=10_000,
|
|
422
|
+
random_state=0,
|
|
423
|
+
):
|
|
424
|
+
tpca_waves = self.unit_tpca_waveforms(
|
|
425
|
+
unit_id,
|
|
426
|
+
random_seed=random_seed,
|
|
427
|
+
max_count=max_count,
|
|
428
|
+
)
|
|
429
|
+
if tpca_waves is None:
|
|
430
|
+
return None, None
|
|
431
|
+
|
|
432
|
+
waveforms = tpca_waves.waveforms
|
|
433
|
+
|
|
434
|
+
# remove chans with no signal at all
|
|
435
|
+
not_entirely_nan_channels = np.flatnonzero(
|
|
436
|
+
np.isfinite(waveforms[:, 0]).any(axis=0)
|
|
437
|
+
)
|
|
438
|
+
if (
|
|
439
|
+
not_entirely_nan_channels.size
|
|
440
|
+
and not_entirely_nan_channels.size < waveforms.shape[2]
|
|
441
|
+
):
|
|
442
|
+
waveforms = waveforms[:, :, not_entirely_nan_channels]
|
|
443
|
+
|
|
444
|
+
waveforms = waveforms.reshape(len(waveforms), -1)
|
|
445
|
+
no_nan = np.flatnonzero(~np.isnan(waveforms).any(axis=1))
|
|
446
|
+
|
|
447
|
+
features = np.full((len(waveforms), rank), np.nan, dtype=waveforms.dtype)
|
|
448
|
+
if no_nan.size < rank:
|
|
449
|
+
return tpca_waves.which, features
|
|
450
|
+
|
|
451
|
+
pca = PCA(rank, random_state=random_seed, whiten=True)
|
|
452
|
+
if no_nan.size > max_wfs_fit:
|
|
453
|
+
rg = np.random.default_rng(random_state)
|
|
454
|
+
choices = rg.choice(no_nan, size=max_wfs_fit, replace=False)
|
|
455
|
+
choices.sort()
|
|
456
|
+
pca.fit(waveforms[choices])
|
|
457
|
+
# features[no_nan] = pca.transform(waveforms[no_nan])
|
|
458
|
+
else:
|
|
459
|
+
# features[no_nan] = pca.fit_transform(waveforms[no_nan])
|
|
460
|
+
pca.fit(waveforms[no_nan])
|
|
461
|
+
features = pca.transform(
|
|
462
|
+
np.where(np.isfinite(waveforms), waveforms, pca.mean_[None])
|
|
463
|
+
)
|
|
464
|
+
return tpca_waves.which, features
|
|
465
|
+
|
|
466
|
+
def unit_max_channel(self, unit_id) -> int:
|
|
467
|
+
assert self.coarse_template_data is not None
|
|
468
|
+
temp = self.coarse_template_data.unit_templates(unit_id)
|
|
469
|
+
assert temp.ndim == 3 and temp.shape[0] == np.atleast_1d(unit_id).size
|
|
470
|
+
|
|
471
|
+
which = self.in_unit(unit_id)
|
|
472
|
+
if self.motion_est is not None:
|
|
473
|
+
assert self.z is not None
|
|
474
|
+
assert self.registered_z is not None
|
|
475
|
+
times_seconds = getattr(self.sorting, "times_seconds", None)
|
|
476
|
+
assert times_seconds is not None
|
|
477
|
+
n_pitches_shift = get_spike_pitch_shifts(
|
|
478
|
+
self.z[which],
|
|
479
|
+
geom=self.geom,
|
|
480
|
+
registered_depths_um=self.registered_z[which],
|
|
481
|
+
times_s=times_seconds[which],
|
|
482
|
+
motion_est=self.motion_est,
|
|
483
|
+
)
|
|
484
|
+
covered_chans = get_stable_channels(
|
|
485
|
+
geom=self.geom,
|
|
486
|
+
registered_geom=self.registered_geom,
|
|
487
|
+
channels=self.sorting.channels[which],
|
|
488
|
+
channel_index=self.vis_channel_index,
|
|
489
|
+
n_pitches_shift=n_pitches_shift,
|
|
490
|
+
)[0]
|
|
491
|
+
else:
|
|
492
|
+
covered_chans = self.vis_channel_index[self.sorting.channels[which]]
|
|
493
|
+
covered_chans = np.unique(covered_chans)
|
|
494
|
+
covered_chans = covered_chans[covered_chans < len(self.vis_channel_index)]
|
|
495
|
+
amp_vec = np.ptp(temp.mean(0), 0)
|
|
496
|
+
max_chan = covered_chans[amp_vec[covered_chans].argmax()]
|
|
497
|
+
return max_chan.item()
|
|
498
|
+
|
|
499
|
+
def unit_select_channels(
|
|
500
|
+
self,
|
|
501
|
+
unit_id,
|
|
502
|
+
which,
|
|
503
|
+
waveforms,
|
|
504
|
+
read_chans=None,
|
|
505
|
+
main_channel=None,
|
|
506
|
+
):
|
|
507
|
+
if read_chans is None:
|
|
508
|
+
read_chans = self.sorting.channels[which]
|
|
509
|
+
if main_channel is None:
|
|
510
|
+
main_channel = self.unit_max_channel(unit_id)
|
|
511
|
+
|
|
512
|
+
show_chans = self.vis_channel_index[main_channel]
|
|
513
|
+
show_valid = show_chans < len(self.registered_geom)
|
|
514
|
+
show_chans = show_chans[show_valid]
|
|
515
|
+
|
|
516
|
+
if self.shifting:
|
|
517
|
+
assert self.z is not None
|
|
518
|
+
assert self.registered_z is not None
|
|
519
|
+
times_seconds = getattr(self.sorting, "times_seconds", None)
|
|
520
|
+
assert times_seconds is not None
|
|
521
|
+
n_pitches_shift = get_spike_pitch_shifts(
|
|
522
|
+
self.z[which],
|
|
523
|
+
geom=self.geom,
|
|
524
|
+
registered_depths_um=self.registered_z[which],
|
|
525
|
+
times_s=times_seconds[which],
|
|
526
|
+
motion_est=self.motion_est,
|
|
527
|
+
)
|
|
528
|
+
else:
|
|
529
|
+
n_pitches_shift = None
|
|
530
|
+
|
|
531
|
+
if self.extract_channel_index is None:
|
|
532
|
+
read_channel_index = self.vis_channel_index
|
|
533
|
+
else:
|
|
534
|
+
read_channel_index = self.extract_channel_index
|
|
535
|
+
|
|
536
|
+
waveforms_valid = get_waveforms_on_static_channels(
|
|
537
|
+
waveforms=waveforms,
|
|
538
|
+
geom=self.geom,
|
|
539
|
+
n_pitches_shift=n_pitches_shift,
|
|
540
|
+
main_channels=read_chans,
|
|
541
|
+
channel_index=read_channel_index,
|
|
542
|
+
target_channels=show_chans,
|
|
543
|
+
registered_geom=self.registered_geom,
|
|
544
|
+
)
|
|
545
|
+
waveforms = np.full_like(
|
|
546
|
+
waveforms_valid,
|
|
547
|
+
shape=(*waveforms_valid.shape[:2], show_valid.size),
|
|
548
|
+
fill_value=np.nan,
|
|
549
|
+
)
|
|
550
|
+
waveforms[:, :, show_valid] = waveforms_valid
|
|
551
|
+
|
|
552
|
+
return waveforms, main_channel
|
|
553
|
+
|
|
554
|
+
def nearby_coarse_templates(self, unit_id, n_neighbors=5):
|
|
555
|
+
td = self.coarse_template_data
|
|
556
|
+
assert td is not None
|
|
557
|
+
assert self.merge_distances is not None
|
|
558
|
+
|
|
559
|
+
unit_ix = np.searchsorted(td.unit_ids, unit_id)
|
|
560
|
+
unit_dists = self.merge_distances[unit_ix]
|
|
561
|
+
distance_order = np.argsort(unit_dists)
|
|
562
|
+
distance_order = np.concatenate(
|
|
563
|
+
([unit_ix], distance_order[distance_order != unit_ix])
|
|
564
|
+
)
|
|
565
|
+
# assert distance_order[0] == unit_ix
|
|
566
|
+
neighb_ixs = distance_order[:n_neighbors]
|
|
567
|
+
neighb_ids = td.unit_ids[neighb_ixs]
|
|
568
|
+
neighb_dists = self.merge_distances[neighb_ixs[:, None], neighb_ixs[None, :]]
|
|
569
|
+
neighb_coarse_templates = td.templates[neighb_ixs]
|
|
570
|
+
return neighb_ixs, neighb_ids, neighb_dists, neighb_coarse_templates
|
|
571
|
+
|
|
572
|
+
def spike_isis(self):
|
|
573
|
+
return sorting_isis(self.sorting)
|
|
574
|
+
|
|
575
|
+
def viol_rate(self, dt_ms=0.8):
|
|
576
|
+
viol_rates = np.zeros(self.unit_ids.shape)
|
|
577
|
+
for j, uid in enumerate(self.unit_ids):
|
|
578
|
+
inu = self.in_unit(uid)
|
|
579
|
+
if not inu.size > 1:
|
|
580
|
+
continue
|
|
581
|
+
t_ms = self.sorting.times_seconds[inu] * 1000 # type: ignore
|
|
582
|
+
isi = np.diff(t_ms)
|
|
583
|
+
viol_rates[j] = (np.abs(isi) < dt_ms).mean()
|
|
584
|
+
return viol_rates
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
@databag
|
|
588
|
+
class WaveformsBag:
|
|
589
|
+
temporal_slice: slice | None
|
|
590
|
+
which: np.ndarray
|
|
591
|
+
waveforms: np.ndarray
|
|
592
|
+
main_channel: int
|
|
593
|
+
geom: np.ndarray
|
|
594
|
+
channel_index: np.ndarray
|