DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1498 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import math
|
|
3
|
+
import warnings
|
|
4
|
+
from dataclasses import dataclass, field, replace
|
|
5
|
+
from logging import getLogger
|
|
6
|
+
from sys import getrefcount
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
import h5py
|
|
10
|
+
import numpy as np
|
|
11
|
+
import torch
|
|
12
|
+
import torch.nn.functional as F
|
|
13
|
+
from scipy.special import ndtri
|
|
14
|
+
from spikeinterface.core import BaseRecording
|
|
15
|
+
from torch.distributions import StudentT
|
|
16
|
+
|
|
17
|
+
from ..templates.superres_util import superres_sorting
|
|
18
|
+
from ..transform import TemporalPCAFeaturizer, Waveform, WaveformPipeline
|
|
19
|
+
from ..transform.transform_base import BaseWaveformModule
|
|
20
|
+
from ..util import drift_util
|
|
21
|
+
from ..util.data_util import (
|
|
22
|
+
DARTsortSorting,
|
|
23
|
+
load_stored_tsvd,
|
|
24
|
+
restrict_to_valid_times,
|
|
25
|
+
subsample_to_max_count,
|
|
26
|
+
yield_chunks,
|
|
27
|
+
)
|
|
28
|
+
from ..util.internal_config import FitSamplingConfig, TemplateConfig, WaveformConfig
|
|
29
|
+
from ..util.logging_util import DARTsortLogger
|
|
30
|
+
from ..util.spiketorch import ptp
|
|
31
|
+
from ..util.waveform_util import full_channel_index
|
|
32
|
+
from .grab import GrabAndFeaturize
|
|
33
|
+
|
|
34
|
+
logger: DARTsortLogger = getLogger(__name__) # type: ignore
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# TODO: don't compute things in the zero subspace.
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class RunningTemplates(GrabAndFeaturize):
|
|
41
|
+
"""Compute templates online by Welford summation.
|
|
42
|
+
|
|
43
|
+
No medians :(. But has some other denoising methods.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
peel_kind = "Templates"
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
recording,
|
|
51
|
+
channel_index,
|
|
52
|
+
times_samples,
|
|
53
|
+
channels,
|
|
54
|
+
labels,
|
|
55
|
+
motion_est=None,
|
|
56
|
+
n_pitches_shift=None,
|
|
57
|
+
use_zero=False,
|
|
58
|
+
use_outlier=False,
|
|
59
|
+
use_raw=True,
|
|
60
|
+
use_raw_outlier=False,
|
|
61
|
+
use_svd=False,
|
|
62
|
+
use_svd_outlier=False,
|
|
63
|
+
with_raw_std_dev=False,
|
|
64
|
+
discard_outlier_classes=True,
|
|
65
|
+
loot_cov: Literal["diag", "global"] = "global",
|
|
66
|
+
pi_init: Literal["flat", "exp_weight_split"] = "exp_weight_split",
|
|
67
|
+
exp_split_svd_prop: float = 0.6,
|
|
68
|
+
outlier_split_prop: float = 0.1,
|
|
69
|
+
global_sigma: float | None = None,
|
|
70
|
+
denoising_method: Literal["none", "exp_weighted", "loot", "t", "coll"] = "none",
|
|
71
|
+
tsvd=None,
|
|
72
|
+
tsvd_rank=5,
|
|
73
|
+
tsvd_fit_radius=75.0,
|
|
74
|
+
exp_weight_snr_threshold=50.0,
|
|
75
|
+
coll_sorting: DARTsortSorting | None = None,
|
|
76
|
+
mask_indices: np.ndarray | None = None,
|
|
77
|
+
gamma_df: float | tuple[float, ...] | None = None,
|
|
78
|
+
initial_df: float = 3.0,
|
|
79
|
+
t_iters: int = 1,
|
|
80
|
+
group_ids=None,
|
|
81
|
+
properties=None,
|
|
82
|
+
trough_offset_samples=42,
|
|
83
|
+
spike_length_samples=121,
|
|
84
|
+
chunk_length_samples=30_000,
|
|
85
|
+
n_seconds_fit=600,
|
|
86
|
+
n_waveforms_fit=25000,
|
|
87
|
+
max_waveforms_fit=25000,
|
|
88
|
+
fit_sampling: Literal["random", "amp_reweighted"] = "random",
|
|
89
|
+
fit_subsampling_random_state: int | np.random.Generator = 0,
|
|
90
|
+
):
|
|
91
|
+
n_channels = recording.get_num_channels()
|
|
92
|
+
channel_index = torch.tile(torch.arange(n_channels), (n_channels, 1))
|
|
93
|
+
assert labels.min() >= 0
|
|
94
|
+
assert denoising_method in ("none", "exp_weighted", "loot", "t", "coll")
|
|
95
|
+
denoising = denoising_method != "none"
|
|
96
|
+
|
|
97
|
+
self.use_zero = use_zero and denoising and (denoising_method != "exp_weighted")
|
|
98
|
+
self.use_outlier = use_outlier and denoising
|
|
99
|
+
self.use_raw = use_raw
|
|
100
|
+
self.use_raw_outlier = use_raw_outlier and denoising
|
|
101
|
+
self.use_svd = use_svd and denoising
|
|
102
|
+
self.use_svd_outlier = use_svd_outlier and denoising
|
|
103
|
+
self.n_subspaces = (
|
|
104
|
+
self.use_zero
|
|
105
|
+
+ self.use_outlier
|
|
106
|
+
+ self.use_raw
|
|
107
|
+
+ self.use_raw_outlier
|
|
108
|
+
+ self.use_svd
|
|
109
|
+
+ self.use_svd_outlier
|
|
110
|
+
)
|
|
111
|
+
self.discard_outlier_classes = discard_outlier_classes
|
|
112
|
+
self.pi_init = pi_init
|
|
113
|
+
self.exp_split_svd_prop = exp_split_svd_prop
|
|
114
|
+
self.outlier_split_prop = outlier_split_prop
|
|
115
|
+
self.coll_sorting = coll_sorting
|
|
116
|
+
self.mask_indices = mask_indices
|
|
117
|
+
if self.n_subspaces == 1 and denoising_method == "exp_weighted":
|
|
118
|
+
# this would do nothing.
|
|
119
|
+
denoising_method = "none"
|
|
120
|
+
if denoising_method == "exp_weighted":
|
|
121
|
+
assert self.n_subspaces == 2
|
|
122
|
+
self.denoising_method = denoising_method
|
|
123
|
+
|
|
124
|
+
self.exp_weight_snr_threshold = exp_weight_snr_threshold
|
|
125
|
+
self.gamma_df = gamma_df
|
|
126
|
+
self.initial_df = gamma_df if gamma_df is not None else initial_df
|
|
127
|
+
|
|
128
|
+
waveform_feature = Waveform(channel_index=channel_index)
|
|
129
|
+
feats: list[BaseWaveformModule] = [waveform_feature]
|
|
130
|
+
if self.use_svd or self.use_svd_outlier:
|
|
131
|
+
if tsvd is not None:
|
|
132
|
+
tpca = TemporalPCAFeaturizer.from_sklearn(
|
|
133
|
+
channel_index, tsvd, getattr(tsvd, "temporal_slice", None)
|
|
134
|
+
)
|
|
135
|
+
else:
|
|
136
|
+
tpca = TemporalPCAFeaturizer(
|
|
137
|
+
channel_index=channel_index,
|
|
138
|
+
geom=torch.asarray(recording.get_channel_locations()),
|
|
139
|
+
rank=tsvd_rank,
|
|
140
|
+
centered=False,
|
|
141
|
+
fit_radius=tsvd_fit_radius,
|
|
142
|
+
)
|
|
143
|
+
feats.append(tpca)
|
|
144
|
+
self.denoising_rank = tpca.rank
|
|
145
|
+
use_svd = True
|
|
146
|
+
else:
|
|
147
|
+
self.denoising_rank = None
|
|
148
|
+
tpca = None
|
|
149
|
+
|
|
150
|
+
featurization_pipeline = WaveformPipeline(feats)
|
|
151
|
+
|
|
152
|
+
super().__init__(
|
|
153
|
+
recording=recording,
|
|
154
|
+
featurization_pipeline=featurization_pipeline,
|
|
155
|
+
times_samples=times_samples,
|
|
156
|
+
channels=channels,
|
|
157
|
+
labels=labels,
|
|
158
|
+
channel_index=channel_index,
|
|
159
|
+
trough_offset_samples=trough_offset_samples,
|
|
160
|
+
spike_length_samples=spike_length_samples,
|
|
161
|
+
chunk_length_samples=chunk_length_samples,
|
|
162
|
+
n_seconds_fit=n_seconds_fit,
|
|
163
|
+
fit_subsampling_random_state=fit_subsampling_random_state,
|
|
164
|
+
n_waveforms_fit=n_waveforms_fit,
|
|
165
|
+
max_waveforms_fit=max_waveforms_fit,
|
|
166
|
+
fit_sampling=fit_sampling,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
self.motion_aware = motion_est is not None
|
|
170
|
+
self.motion_est = motion_est
|
|
171
|
+
self.with_raw_std_dev = with_raw_std_dev
|
|
172
|
+
self.n_units = labels.max() + 1
|
|
173
|
+
logger.dartsortdebug(f"{self.__class__.__name__}: n_units={self.n_units}.")
|
|
174
|
+
self.group_ids = group_ids
|
|
175
|
+
self.n_pitches_shift = n_pitches_shift
|
|
176
|
+
self.short_pipeline = WaveformPipeline([waveform_feature])
|
|
177
|
+
self.tpca = tpca
|
|
178
|
+
if self.denoising_method in ("t", "loot") and loot_cov == "global":
|
|
179
|
+
assert global_sigma is not None
|
|
180
|
+
self.global_sigma = global_sigma
|
|
181
|
+
|
|
182
|
+
self.subspace_projectors = []
|
|
183
|
+
self.subspace_temporal_projectors = []
|
|
184
|
+
self.raw_subspace_ix = int(self.use_zero) + int(self.use_outlier)
|
|
185
|
+
self.svd_subspace_ix = int(self.use_zero) + int(self.use_outlier) + 1
|
|
186
|
+
inlier_mask = []
|
|
187
|
+
if self.use_zero:
|
|
188
|
+
self.subspace_projectors.append(torch.zeros_like)
|
|
189
|
+
self.subspace_temporal_projectors.append(torch.zeros_like)
|
|
190
|
+
inlier_mask.append(1)
|
|
191
|
+
if self.use_outlier:
|
|
192
|
+
self.subspace_projectors.append(torch.zeros_like)
|
|
193
|
+
self.subspace_temporal_projectors.append(torch.zeros_like)
|
|
194
|
+
inlier_mask.append(0)
|
|
195
|
+
if self.use_raw:
|
|
196
|
+
self.subspace_projectors.append(identity)
|
|
197
|
+
self.subspace_temporal_projectors.append(identity)
|
|
198
|
+
inlier_mask.append(1)
|
|
199
|
+
if self.use_svd:
|
|
200
|
+
assert self.tpca is not None
|
|
201
|
+
self.subspace_projectors.append(self.tpca.force_project)
|
|
202
|
+
self.subspace_temporal_projectors.append(self.tpca._project_in_probe)
|
|
203
|
+
inlier_mask.append(1)
|
|
204
|
+
if self.use_raw_outlier:
|
|
205
|
+
self.subspace_projectors.append(identity)
|
|
206
|
+
self.subspace_temporal_projectors.append(identity)
|
|
207
|
+
inlier_mask.append(0)
|
|
208
|
+
if self.use_svd_outlier:
|
|
209
|
+
assert self.tpca is not None
|
|
210
|
+
self.subspace_projectors.append(self.tpca.force_project)
|
|
211
|
+
self.subspace_temporal_projectors.append(self.tpca._project_in_probe)
|
|
212
|
+
inlier_mask.append(0)
|
|
213
|
+
self.inlier_mask = np.array(inlier_mask, dtype=bool)
|
|
214
|
+
|
|
215
|
+
# these are not used internally, just stored to record what happened in
|
|
216
|
+
# .from_config() and passed through to the TemplateData's properties later
|
|
217
|
+
self.properties = properties or {}
|
|
218
|
+
|
|
219
|
+
self.tasks = RunningTemplatesTasks(
|
|
220
|
+
denoising_method=denoising_method,
|
|
221
|
+
fixed_df=self.gamma_df is not None,
|
|
222
|
+
with_raw_std_dev=with_raw_std_dev,
|
|
223
|
+
loot_cov=loot_cov,
|
|
224
|
+
t_iters=t_iters,
|
|
225
|
+
pi_init=pi_init,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
@classmethod
|
|
229
|
+
def from_config( # pyright: ignore
|
|
230
|
+
cls,
|
|
231
|
+
*,
|
|
232
|
+
sorting: DARTsortSorting,
|
|
233
|
+
recording: BaseRecording,
|
|
234
|
+
waveform_cfg: WaveformConfig,
|
|
235
|
+
template_cfg: TemplateConfig,
|
|
236
|
+
tsvd=None,
|
|
237
|
+
motion_est=None,
|
|
238
|
+
random_state=0,
|
|
239
|
+
):
|
|
240
|
+
assert sorting.labels is not None
|
|
241
|
+
random_state = np.random.default_rng(random_state)
|
|
242
|
+
coll_sorting = sorting if template_cfg.denoising_method == "coll" else None
|
|
243
|
+
|
|
244
|
+
sorting = restrict_to_valid_times(sorting, recording, waveform_cfg)
|
|
245
|
+
|
|
246
|
+
# superres sorting if requested
|
|
247
|
+
if template_cfg.superres_templates:
|
|
248
|
+
res = superres_sorting(
|
|
249
|
+
sorting=sorting,
|
|
250
|
+
geom=recording.get_channel_locations(),
|
|
251
|
+
motion_est=motion_est,
|
|
252
|
+
strategy=template_cfg.superres_strategy,
|
|
253
|
+
superres_bin_size_um=template_cfg.superres_bin_size_um,
|
|
254
|
+
min_spikes_per_bin=template_cfg.superres_bin_min_spikes,
|
|
255
|
+
)
|
|
256
|
+
group_ids = res["group_ids"] # pyright: ignore
|
|
257
|
+
sorting = res["sorting"] # pyright: ignore
|
|
258
|
+
else:
|
|
259
|
+
# check if label set is flat
|
|
260
|
+
assert sorting.labels is not None
|
|
261
|
+
unit_ids = np.unique(sorting.labels)
|
|
262
|
+
group_ids = unit_ids[unit_ids >= 0]
|
|
263
|
+
if not np.array_equal(group_ids, np.arange(len(group_ids))):
|
|
264
|
+
# flatten label set if necessary
|
|
265
|
+
l_valid = np.flatnonzero(sorting.labels >= 0)
|
|
266
|
+
ixs = np.searchsorted(group_ids, sorting.labels[l_valid])
|
|
267
|
+
assert np.array_equal(group_ids[ixs], sorting.labels[l_valid])
|
|
268
|
+
labels_flat = np.full_like(sorting.labels, -1)
|
|
269
|
+
labels_flat[l_valid] = ixs
|
|
270
|
+
sorting = sorting.ephemeral_replace(labels=labels_flat)
|
|
271
|
+
assert sorting.labels is not None
|
|
272
|
+
logger.dartsortverbose(
|
|
273
|
+
f"Flatten from {group_ids.max()} to {len(group_ids)}=={sorting.labels.max() + 1}."
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
# restrict to max spikes/template
|
|
277
|
+
sorting = subsample_to_max_count(
|
|
278
|
+
sorting, max_spikes=template_cfg.spikes_per_unit, seed=random_state
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
# try to load denoiser if denoising is happening
|
|
282
|
+
denoising = template_cfg.denoising_method != "none"
|
|
283
|
+
if tsvd is None and template_cfg.use_svd and denoising:
|
|
284
|
+
if not template_cfg.recompute_tsvd:
|
|
285
|
+
tsvd = load_stored_tsvd(sorting)
|
|
286
|
+
|
|
287
|
+
loot_or_t = template_cfg.denoising_method in ("loot", "t")
|
|
288
|
+
if template_cfg.loot_cov == "global" and loot_or_t:
|
|
289
|
+
global_sigma = global_std_estimate(sorting)
|
|
290
|
+
else:
|
|
291
|
+
global_sigma = None
|
|
292
|
+
|
|
293
|
+
# remove discarded spikes so they don't get loaded
|
|
294
|
+
sorting = sorting.drop_missing()
|
|
295
|
+
|
|
296
|
+
if template_cfg.denoising_method == "coll":
|
|
297
|
+
assert hasattr(sorting, "mask_indices")
|
|
298
|
+
mask_indices = sorting.mask_indices # pyright: ignore
|
|
299
|
+
else:
|
|
300
|
+
mask_indices = None
|
|
301
|
+
|
|
302
|
+
fs = recording.sampling_frequency
|
|
303
|
+
sampling_cfg = template_cfg.denoising_fit_sampling_cfg
|
|
304
|
+
return cls(
|
|
305
|
+
recording=recording,
|
|
306
|
+
channel_index=full_channel_index(recording.get_num_channels()),
|
|
307
|
+
times_samples=sorting.times_samples,
|
|
308
|
+
channels=sorting.channels,
|
|
309
|
+
labels=sorting.labels,
|
|
310
|
+
denoising_method=template_cfg.denoising_method,
|
|
311
|
+
tsvd=tsvd,
|
|
312
|
+
tsvd_rank=template_cfg.denoising_rank,
|
|
313
|
+
tsvd_fit_radius=template_cfg.denoising_fit_radius,
|
|
314
|
+
exp_weight_snr_threshold=template_cfg.exp_weight_snr_threshold,
|
|
315
|
+
group_ids=group_ids,
|
|
316
|
+
motion_est=motion_est,
|
|
317
|
+
use_zero=template_cfg.use_zero,
|
|
318
|
+
use_outlier=template_cfg.use_outlier,
|
|
319
|
+
use_svd=template_cfg.use_svd,
|
|
320
|
+
use_raw_outlier=template_cfg.use_raw_outlier,
|
|
321
|
+
use_svd_outlier=template_cfg.use_svd_outlier,
|
|
322
|
+
with_raw_std_dev=template_cfg.with_raw_std_dev,
|
|
323
|
+
loot_cov=template_cfg.loot_cov,
|
|
324
|
+
global_sigma=global_sigma,
|
|
325
|
+
trough_offset_samples=waveform_cfg.trough_offset_samples(fs),
|
|
326
|
+
spike_length_samples=waveform_cfg.spike_length_samples(fs),
|
|
327
|
+
n_waveforms_fit=sampling_cfg.n_waveforms_fit,
|
|
328
|
+
max_waveforms_fit=sampling_cfg.max_waveforms_fit,
|
|
329
|
+
fit_sampling=sampling_cfg.fit_sampling,
|
|
330
|
+
fit_subsampling_random_state=random_state,
|
|
331
|
+
gamma_df=template_cfg.fixed_t_df,
|
|
332
|
+
initial_df=template_cfg.initial_t_df,
|
|
333
|
+
t_iters=template_cfg.t_iters,
|
|
334
|
+
coll_sorting=coll_sorting,
|
|
335
|
+
mask_indices=mask_indices,
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
def compute_template_data(
|
|
339
|
+
self, show_progress=True, computation_cfg=None, task_name=None
|
|
340
|
+
):
|
|
341
|
+
# fit my TSVD object if necessary, and then turn it off
|
|
342
|
+
# we don't need the TSVD projections in the processing, just need
|
|
343
|
+
# to project at the end. (used to need it inside for the median.)
|
|
344
|
+
super().fit_featurization_pipeline(computation_cfg=computation_cfg)
|
|
345
|
+
self.featurization_pipeline = self.short_pipeline
|
|
346
|
+
|
|
347
|
+
self.setup()
|
|
348
|
+
while self.tasks.plan_step():
|
|
349
|
+
self.setup_step()
|
|
350
|
+
self.peel(
|
|
351
|
+
output_hdf5_filename=None,
|
|
352
|
+
ignore_resuming=True,
|
|
353
|
+
show_progress=show_progress,
|
|
354
|
+
computation_cfg=computation_cfg,
|
|
355
|
+
task_name=task_name or self.tasks.step_description(),
|
|
356
|
+
)
|
|
357
|
+
self.finalize_step()
|
|
358
|
+
self.tasks.finalize_step()
|
|
359
|
+
|
|
360
|
+
return self.to_template_data()
|
|
361
|
+
|
|
362
|
+
def finalize_step(self):
|
|
363
|
+
if self.tasks.needs_basic_pass:
|
|
364
|
+
self.finalize_basic()
|
|
365
|
+
elif self.tasks.needs_coll_pass:
|
|
366
|
+
self.finalize_basic() # same.
|
|
367
|
+
elif self.tasks.needs_mixture_pass:
|
|
368
|
+
self.finalize_mixture()
|
|
369
|
+
else:
|
|
370
|
+
assert False
|
|
371
|
+
|
|
372
|
+
def setup(self):
|
|
373
|
+
# this sets motion-related buffers (pitch shifts, geom)
|
|
374
|
+
self.setup_global()
|
|
375
|
+
# sets up buffers for raw means + meansq
|
|
376
|
+
self.setup_basic()
|
|
377
|
+
self.setup_coll()
|
|
378
|
+
# sets up subspace stuff and gamma stats
|
|
379
|
+
self.setup_mixture()
|
|
380
|
+
|
|
381
|
+
def setup_step(self):
|
|
382
|
+
self.n_spikes = 0
|
|
383
|
+
if self.tasks.active_step == "mix":
|
|
384
|
+
self.b.total_resp.zero_()
|
|
385
|
+
self.b.total_weight.zero_()
|
|
386
|
+
self.b.subspace_resp.zero_()
|
|
387
|
+
|
|
388
|
+
def to_template_data(self):
|
|
389
|
+
from ..templates.templates import TemplateData
|
|
390
|
+
|
|
391
|
+
counts_by_channel = self.b.counts.numpy(force=True)
|
|
392
|
+
if self.group_ids is None:
|
|
393
|
+
unit_ids = np.arange(self.n_units)
|
|
394
|
+
else:
|
|
395
|
+
unit_ids = self.group_ids
|
|
396
|
+
|
|
397
|
+
templates = self.templates()
|
|
398
|
+
assert templates.shape == (
|
|
399
|
+
self.n_units,
|
|
400
|
+
self.spike_length_samples,
|
|
401
|
+
self.n_channels_full,
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
if not self.with_raw_std_dev:
|
|
405
|
+
raw_stds = None
|
|
406
|
+
elif self.denoising_method in ("none", "exp_weighted", "loot", "coll"):
|
|
407
|
+
raw_stds = self.b.meansq.sub(self.raw_means.square())
|
|
408
|
+
raw_stds = raw_stds.clamp_(min=0.0).sqrt_()
|
|
409
|
+
elif self.denoising_method == "t":
|
|
410
|
+
pi = self.b.log_props.exp()
|
|
411
|
+
raw_stds = (self.b.scale * pi).sum(dim=0)
|
|
412
|
+
else:
|
|
413
|
+
assert False
|
|
414
|
+
|
|
415
|
+
if raw_stds is not None:
|
|
416
|
+
assert raw_stds.isfinite().all()
|
|
417
|
+
assert raw_stds.shape == templates.shape
|
|
418
|
+
raw_stds = raw_stds.numpy(force=True)
|
|
419
|
+
|
|
420
|
+
if self.tpca is None:
|
|
421
|
+
tsvd = None
|
|
422
|
+
else:
|
|
423
|
+
tsvd = self.tpca.to_sklearn()
|
|
424
|
+
|
|
425
|
+
return TemplateData(
|
|
426
|
+
templates=templates,
|
|
427
|
+
unit_ids=unit_ids,
|
|
428
|
+
spike_counts_by_channel=counts_by_channel,
|
|
429
|
+
spike_counts=counts_by_channel.max(1),
|
|
430
|
+
raw_std_dev=raw_stds,
|
|
431
|
+
registered_geom=self.reg_geom,
|
|
432
|
+
trough_offset_samples=self.trough_offset_samples,
|
|
433
|
+
properties=self.properties or None,
|
|
434
|
+
tsvd=tsvd,
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
def templates(self):
|
|
438
|
+
# assert self.tasks.done
|
|
439
|
+
|
|
440
|
+
if self.denoising_method in (None, "none"):
|
|
441
|
+
t = self.raw_means
|
|
442
|
+
assert t.isfinite().all()
|
|
443
|
+
return t.numpy(force=True)
|
|
444
|
+
|
|
445
|
+
if self.denoising_method in ("coll", "exp_weighted"):
|
|
446
|
+
from ..templates.get_templates import denoising_weights
|
|
447
|
+
|
|
448
|
+
raw_means = self.raw_means.nan_to_num_().numpy(force=True)
|
|
449
|
+
snrs = ptp(self.raw_means.nan_to_num(nan=-torch.inf)).mul_(
|
|
450
|
+
self.b.counts.to(self.raw_means).sqrt()
|
|
451
|
+
)
|
|
452
|
+
weights = denoising_weights(
|
|
453
|
+
snrs.numpy(force=True),
|
|
454
|
+
spike_length_samples=self.spike_length_samples,
|
|
455
|
+
trough_offset=self.trough_offset_samples,
|
|
456
|
+
snr_threshold=self.exp_weight_snr_threshold,
|
|
457
|
+
)
|
|
458
|
+
w = weights.astype(raw_means.dtype)
|
|
459
|
+
svd_means = self.b.means[self.svd_subspace_ix]
|
|
460
|
+
svd_means = svd_means.nan_to_num_().numpy(force=True)
|
|
461
|
+
logger.dartsortdebug(
|
|
462
|
+
f"exp_weighted_svd: Raw weight mean/max={w.mean().item()}/{w.max().item()}"
|
|
463
|
+
)
|
|
464
|
+
m = w * raw_means + (1 - w) * svd_means
|
|
465
|
+
assert np.isfinite(m).all()
|
|
466
|
+
return m
|
|
467
|
+
|
|
468
|
+
if self.denoising_method in ("loot", "t"):
|
|
469
|
+
if self.discard_outlier_classes:
|
|
470
|
+
inliers = self.inlier_mask
|
|
471
|
+
inliers = torch.asarray(inliers)
|
|
472
|
+
else:
|
|
473
|
+
inliers = slice(None)
|
|
474
|
+
pi = F.softmax(self.b.log_props[inliers], dim=0)
|
|
475
|
+
raw_pi = pi[self.raw_subspace_ix]
|
|
476
|
+
logger.dartsortdebug(
|
|
477
|
+
"Template demixing: Raw weight mean/max="
|
|
478
|
+
f"{raw_pi.mean().item():.4f}/{raw_pi.max().item():.4f}"
|
|
479
|
+
)
|
|
480
|
+
mu = (pi * self.b.means[inliers]).sum(dim=0)
|
|
481
|
+
assert mu.isfinite().all()
|
|
482
|
+
return mu.numpy(force=True)
|
|
483
|
+
|
|
484
|
+
assert False
|
|
485
|
+
|
|
486
|
+
def process_chunk(
|
|
487
|
+
self,
|
|
488
|
+
chunk_start_samples,
|
|
489
|
+
*,
|
|
490
|
+
chunk_end_samples=None,
|
|
491
|
+
return_residual=False,
|
|
492
|
+
skip_features=False,
|
|
493
|
+
n_resid_snips=None,
|
|
494
|
+
to_cpu=False,
|
|
495
|
+
):
|
|
496
|
+
res = super().process_chunk(
|
|
497
|
+
chunk_start_samples=chunk_start_samples,
|
|
498
|
+
n_resid_snips=n_resid_snips,
|
|
499
|
+
chunk_end_samples=chunk_end_samples,
|
|
500
|
+
return_residual=return_residual,
|
|
501
|
+
skip_features=skip_features,
|
|
502
|
+
to_cpu=to_cpu,
|
|
503
|
+
)
|
|
504
|
+
if to_cpu or not res["n_spikes"]:
|
|
505
|
+
return res
|
|
506
|
+
|
|
507
|
+
if self.tasks.active_step == "coll":
|
|
508
|
+
# TODO: remove if we don't use this method.
|
|
509
|
+
assert chunk_end_samples is not None
|
|
510
|
+
coll_kw = dict(
|
|
511
|
+
chunk_start_samples=chunk_start_samples,
|
|
512
|
+
chunk_length_samples=chunk_end_samples - chunk_start_samples,
|
|
513
|
+
left_margin=self.chunk_margin_samples if chunk_start_samples > 0 else 0,
|
|
514
|
+
right_margin=(
|
|
515
|
+
self.chunk_margin_samples
|
|
516
|
+
if chunk_end_samples < self.recording.get_num_samples()
|
|
517
|
+
else 0
|
|
518
|
+
),
|
|
519
|
+
times_samples=res["times_samples"],
|
|
520
|
+
)
|
|
521
|
+
else:
|
|
522
|
+
coll_kw = {}
|
|
523
|
+
|
|
524
|
+
ires = self._process_chunk_main(
|
|
525
|
+
res["waveforms"], # pyright: ignore
|
|
526
|
+
res["indices"], # pyright: ignore
|
|
527
|
+
res["labels"], # pyright: ignore
|
|
528
|
+
**coll_kw, # pyright: ignore
|
|
529
|
+
)
|
|
530
|
+
res.update(ires) # pyright: ignore
|
|
531
|
+
|
|
532
|
+
return res
|
|
533
|
+
|
|
534
|
+
def gather_chunk_result(
|
|
535
|
+
self,
|
|
536
|
+
cur_n_spikes,
|
|
537
|
+
chunk_start_samples,
|
|
538
|
+
chunk_result,
|
|
539
|
+
h5_spike_datasets,
|
|
540
|
+
output_h5,
|
|
541
|
+
residual_file,
|
|
542
|
+
residual_to_h5,
|
|
543
|
+
ignore_resuming,
|
|
544
|
+
skip_features,
|
|
545
|
+
):
|
|
546
|
+
if output_h5 is not None:
|
|
547
|
+
# this path is active when fitting models (i.e., temporal svd).
|
|
548
|
+
# otherwise, peeling is run with no output file, and everything
|
|
549
|
+
# is done in the arrays created during setup().
|
|
550
|
+
# in this branch, to_numpy was True in process_chunk().
|
|
551
|
+
return super().gather_chunk_result(
|
|
552
|
+
cur_n_spikes,
|
|
553
|
+
chunk_start_samples,
|
|
554
|
+
chunk_result,
|
|
555
|
+
h5_spike_datasets,
|
|
556
|
+
output_h5,
|
|
557
|
+
residual_file,
|
|
558
|
+
residual_to_h5,
|
|
559
|
+
ignore_resuming,
|
|
560
|
+
skip_features,
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
assert residual_file is None
|
|
564
|
+
assert not residual_to_h5
|
|
565
|
+
assert ignore_resuming
|
|
566
|
+
|
|
567
|
+
n_spikes = chunk_result["n_spikes"]
|
|
568
|
+
if not n_spikes:
|
|
569
|
+
return n_spikes
|
|
570
|
+
|
|
571
|
+
self.n_spikes += n_spikes
|
|
572
|
+
self._gather_chunk_main(chunk_result)
|
|
573
|
+
|
|
574
|
+
return n_spikes
|
|
575
|
+
|
|
576
|
+
def _process_chunk_main(
|
|
577
|
+
self,
|
|
578
|
+
waveforms: torch.Tensor,
|
|
579
|
+
spike_ix: torch.LongTensor,
|
|
580
|
+
labels: torch.LongTensor,
|
|
581
|
+
**coll_kw,
|
|
582
|
+
):
|
|
583
|
+
"""Handles batch statistics for raw or exp weighted templates
|
|
584
|
+
|
|
585
|
+
Also is the initialization stage for the t flavored versions.
|
|
586
|
+
"""
|
|
587
|
+
# scatter waveforms to reg channels, if necessary
|
|
588
|
+
if self.motion_aware:
|
|
589
|
+
chanix = self.b.target_channels[self.b.pitch_shift_ixs[spike_ix]]
|
|
590
|
+
waveforms = registerize(waveforms, self.n_channels_full, chanix)
|
|
591
|
+
else:
|
|
592
|
+
chanix = None
|
|
593
|
+
|
|
594
|
+
if self.tasks.active_step == "basic":
|
|
595
|
+
return self._process_chunk_basic(waveforms, labels)
|
|
596
|
+
elif self.tasks.active_step == "mix":
|
|
597
|
+
return self._process_chunk_mixture(waveforms, labels)
|
|
598
|
+
elif self.tasks.active_step == "coll":
|
|
599
|
+
return self._process_chunk_coll(
|
|
600
|
+
waveforms=waveforms,
|
|
601
|
+
labels=labels,
|
|
602
|
+
**coll_kw,
|
|
603
|
+
)
|
|
604
|
+
else:
|
|
605
|
+
assert False
|
|
606
|
+
|
|
607
|
+
assert False
|
|
608
|
+
|
|
609
|
+
def _process_chunk_basic(self, waveforms: torch.Tensor, labels: torch.LongTensor):
|
|
610
|
+
res = {}
|
|
611
|
+
|
|
612
|
+
# determine each unit's per-channel counts, for weighting purposes
|
|
613
|
+
# TODO: sum responsibility weights here
|
|
614
|
+
counts = waveforms.new_zeros((self.n_units, self.n_channels_full))
|
|
615
|
+
weights = waveforms[:, 0].isfinite().to(waveforms)
|
|
616
|
+
ix = labels[:, None].broadcast_to(weights.shape)
|
|
617
|
+
counts.scatter_add_(dim=0, index=ix, src=weights)
|
|
618
|
+
res["counts"] = counts
|
|
619
|
+
|
|
620
|
+
# normalized weights in each unit
|
|
621
|
+
denom = torch.where(counts > 0, counts, 1.0)
|
|
622
|
+
weights.div_(denom[labels])
|
|
623
|
+
|
|
624
|
+
waveforms.nan_to_num_()
|
|
625
|
+
|
|
626
|
+
# weighted avg
|
|
627
|
+
raw_means = waveforms.new_zeros(self.raw_means.shape)
|
|
628
|
+
x = waveforms.mul(weights[:, None])
|
|
629
|
+
ix = labels[:, None, None].broadcast_to(x.shape)
|
|
630
|
+
raw_means.scatter_add_(dim=0, index=ix, src=x)
|
|
631
|
+
res["raw_means"] = raw_means
|
|
632
|
+
|
|
633
|
+
if self.tasks.updating_meansq:
|
|
634
|
+
meansq = waveforms.new_zeros(self.raw_means.shape)
|
|
635
|
+
x.mul_(waveforms)
|
|
636
|
+
meansq.scatter_add_(dim=0, index=ix, src=x)
|
|
637
|
+
res["meansq"] = meansq
|
|
638
|
+
|
|
639
|
+
return res
|
|
640
|
+
|
|
641
|
+
def _process_chunk_coll(
|
|
642
|
+
self,
|
|
643
|
+
chunk_start_samples: int,
|
|
644
|
+
chunk_length_samples: int,
|
|
645
|
+
times_samples: int,
|
|
646
|
+
waveforms: torch.Tensor,
|
|
647
|
+
labels: torch.LongTensor,
|
|
648
|
+
left_margin: int = 0,
|
|
649
|
+
right_margin: int = 0,
|
|
650
|
+
):
|
|
651
|
+
counts = waveforms.new_zeros((self.n_units, self.n_channels_full))
|
|
652
|
+
weights = waveforms[:, 0].isfinite().to(waveforms)
|
|
653
|
+
ix = labels[:, None].broadcast_to(weights.shape)
|
|
654
|
+
counts.scatter_add_(dim=0, index=ix, src=weights)
|
|
655
|
+
valid = self.times_samples == self.times_samples.clip( # type: ignore
|
|
656
|
+
chunk_start_samples,
|
|
657
|
+
chunk_start_samples + chunk_length_samples - 1,
|
|
658
|
+
)
|
|
659
|
+
minds = self.mask_indices[valid.numpy(force=True)] # type: ignore
|
|
660
|
+
weights = coll_weights(
|
|
661
|
+
times_samples,
|
|
662
|
+
waveforms,
|
|
663
|
+
self.coll_sorting,
|
|
664
|
+
chunk_start_samples,
|
|
665
|
+
chunk_length_samples,
|
|
666
|
+
self.trough_offset_samples,
|
|
667
|
+
left_margin,
|
|
668
|
+
right_margin,
|
|
669
|
+
mask_indices=minds,
|
|
670
|
+
)
|
|
671
|
+
wsum, xbar, xsqbar = count_and_mean(
|
|
672
|
+
resp=None,
|
|
673
|
+
w=weights[None],
|
|
674
|
+
x=waveforms,
|
|
675
|
+
labels=labels,
|
|
676
|
+
k=self.n_units,
|
|
677
|
+
with_sq=self.tasks.updating_subspace_stds,
|
|
678
|
+
)
|
|
679
|
+
assert wsum.shape[0] == xbar.shape[0] == 1
|
|
680
|
+
res = dict(
|
|
681
|
+
weights=wsum[0],
|
|
682
|
+
means=xbar[0],
|
|
683
|
+
counts=counts,
|
|
684
|
+
)
|
|
685
|
+
if xsqbar is not None:
|
|
686
|
+
res["meansq"] = xsqbar[0]
|
|
687
|
+
return res
|
|
688
|
+
|
|
689
|
+
def _process_chunk_mixture(self, waveforms: torch.Tensor, labels: torch.LongTensor):
|
|
690
|
+
resp, what = smsm_resps_and_latents(
|
|
691
|
+
x=waveforms,
|
|
692
|
+
labels=labels,
|
|
693
|
+
lps=self.log_props,
|
|
694
|
+
nus=self.nu,
|
|
695
|
+
mus=self.means,
|
|
696
|
+
sigmas=self.scale,
|
|
697
|
+
meansqs=self.meansq,
|
|
698
|
+
loo=self.tasks.current_step_is_loot,
|
|
699
|
+
loo_N=self.counts,
|
|
700
|
+
global_sigma=self.global_sigma,
|
|
701
|
+
projectors=self.subspace_temporal_projectors,
|
|
702
|
+
)
|
|
703
|
+
wsum, xbar, xsqbar = count_and_mean(
|
|
704
|
+
resp=resp,
|
|
705
|
+
w=what,
|
|
706
|
+
x=waveforms,
|
|
707
|
+
labels=labels,
|
|
708
|
+
k=self.n_units,
|
|
709
|
+
with_sq=self.tasks.updating_subspace_stds,
|
|
710
|
+
)
|
|
711
|
+
rsum, rtotal, wbar, sqwbar, logwbar = gamma_count_and_mean(
|
|
712
|
+
resp=resp,
|
|
713
|
+
w=what,
|
|
714
|
+
labels=labels,
|
|
715
|
+
k=self.n_units,
|
|
716
|
+
count_only=not self.tasks.updating_df,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
res = {}
|
|
720
|
+
res["weights"] = wsum
|
|
721
|
+
res["means"] = xbar
|
|
722
|
+
if xsqbar is not None:
|
|
723
|
+
res["meansq"] = xsqbar
|
|
724
|
+
res["resps"] = rsum
|
|
725
|
+
if wbar is not None:
|
|
726
|
+
res["rtotal"] = rtotal
|
|
727
|
+
res["wbar"] = wbar
|
|
728
|
+
res["sqwbar"] = sqwbar
|
|
729
|
+
res["logwbar"] = logwbar
|
|
730
|
+
|
|
731
|
+
return res
|
|
732
|
+
|
|
733
|
+
def _gather_chunk_main(self, chunk_result):
|
|
734
|
+
"""Update running stats with Welford formula."""
|
|
735
|
+
if self.tasks.active_step == "basic":
|
|
736
|
+
self._gather_chunk_basic(chunk_result)
|
|
737
|
+
elif self.tasks.active_step == "mix":
|
|
738
|
+
self._gather_chunk_mixture(chunk_result)
|
|
739
|
+
elif self.tasks.active_step == "coll":
|
|
740
|
+
self._gather_chunk_coll(chunk_result)
|
|
741
|
+
else:
|
|
742
|
+
assert False
|
|
743
|
+
|
|
744
|
+
def _gather_chunk_basic(self, ch_res):
|
|
745
|
+
self.counts += ch_res["counts"]
|
|
746
|
+
w = (
|
|
747
|
+
ch_res["counts"]
|
|
748
|
+
.div_(self.counts)
|
|
749
|
+
.nan_to_num_()
|
|
750
|
+
.unsqueeze(1)
|
|
751
|
+
.to(self.raw_means)
|
|
752
|
+
)
|
|
753
|
+
self.b.means[self.raw_subspace_ix] += (
|
|
754
|
+
ch_res["raw_means"].sub_(self.raw_means).mul_(w)
|
|
755
|
+
)
|
|
756
|
+
if "meansq" in ch_res:
|
|
757
|
+
self.meansq += ch_res["meansq"].sub_(self.meansq).mul_(w)
|
|
758
|
+
|
|
759
|
+
def _gather_chunk_coll(self, ch_res):
|
|
760
|
+
self.counts += ch_res["counts"]
|
|
761
|
+
self.total_weight += ch_res["weights"]
|
|
762
|
+
w = ch_res["weights"].div_(self.total_weight).nan_to_num_()
|
|
763
|
+
self.b.means[self.raw_subspace_ix] += (
|
|
764
|
+
ch_res["means"].sub_(self.raw_means).mul_(w)
|
|
765
|
+
)
|
|
766
|
+
if "meansq" in ch_res:
|
|
767
|
+
self.meansq += ch_res["meansq"].sub_(self.meansq).mul_(w)
|
|
768
|
+
|
|
769
|
+
def _gather_chunk_mixture(self, ch_res):
|
|
770
|
+
# track responsibility sum for proportions / scale update
|
|
771
|
+
self.total_resp += ch_res["resps"]
|
|
772
|
+
|
|
773
|
+
# gamma x resp weight, used in mean and meansq update
|
|
774
|
+
self.total_weight += ch_res["weights"]
|
|
775
|
+
gw = ch_res["weights"].div_(self.total_weight).nan_to_num_()
|
|
776
|
+
|
|
777
|
+
# mean update
|
|
778
|
+
self.means_new += ch_res["means"].sub_(self.means_new).mul_(gw)
|
|
779
|
+
|
|
780
|
+
# scale update -- to be rescaled/subspaced in finalize()
|
|
781
|
+
if self.tasks.updating_subspace_stds:
|
|
782
|
+
self.scale_new += ch_res["meansq"].sub_(self.scale_new).mul_(gw)
|
|
783
|
+
|
|
784
|
+
if self.tasks.updating_df:
|
|
785
|
+
# resp weight only needed for gamma stats
|
|
786
|
+
self.subspace_resp += ch_res["rtotal"]
|
|
787
|
+
rw = ch_res["rtotal"].div_(self.subspace_resp).nan_to_num_()
|
|
788
|
+
self.wbar += rw * (ch_res["wbar"] - self.wbar)
|
|
789
|
+
self.logwbar += rw * (ch_res["logwbar"] - self.logwbar)
|
|
790
|
+
self.sqwbar += rw * (ch_res["sqwbar"] - self.sqwbar)
|
|
791
|
+
|
|
792
|
+
def setup_global(self):
|
|
793
|
+
geom = self.recording.get_channel_locations()
|
|
794
|
+
if self.motion_aware:
|
|
795
|
+
self.reg_geom = drift_util.registered_geometry(geom, self.motion_est)
|
|
796
|
+
self.n_channels_full = len(self.reg_geom)
|
|
797
|
+
|
|
798
|
+
n_pitches_shift = self.n_pitches_shift
|
|
799
|
+
if n_pitches_shift is None:
|
|
800
|
+
depths = np.atleast_1d(geom[self.channels, 1])
|
|
801
|
+
times = self.recording.sample_index_to_time(self.times_samples)
|
|
802
|
+
n_pitches_shift = drift_util.get_spike_pitch_shifts(
|
|
803
|
+
depths, geom, times_s=times, motion_est=self.motion_est
|
|
804
|
+
)
|
|
805
|
+
unique_shifts, pitch_shift_ixs = np.unique(
|
|
806
|
+
n_pitches_shift, return_inverse=True
|
|
807
|
+
)
|
|
808
|
+
res = drift_util.get_stable_channels(
|
|
809
|
+
geom=geom,
|
|
810
|
+
channels=np.zeros_like(unique_shifts),
|
|
811
|
+
channel_index=self.b.channel_index.numpy(force=True),
|
|
812
|
+
registered_geom=self.reg_geom,
|
|
813
|
+
n_pitches_shift=unique_shifts,
|
|
814
|
+
)
|
|
815
|
+
target_channels = torch.asarray(res[0], device=self.b.channel_index.device)
|
|
816
|
+
pitch_shift_ixs = torch.asarray(
|
|
817
|
+
pitch_shift_ixs, device=self.b.channel_index.device
|
|
818
|
+
)
|
|
819
|
+
self.register_buffer("pitch_shift_ixs", pitch_shift_ixs)
|
|
820
|
+
self.register_buffer("target_channels", target_channels)
|
|
821
|
+
else:
|
|
822
|
+
self.reg_geom = geom
|
|
823
|
+
self.n_channels_full = self.recording.get_num_channels()
|
|
824
|
+
|
|
825
|
+
def setup_basic(self):
|
|
826
|
+
k = self.n_units
|
|
827
|
+
t = self.spike_length_samples
|
|
828
|
+
c = self.n_channels_full
|
|
829
|
+
s = self.n_subspaces
|
|
830
|
+
|
|
831
|
+
self.register_buffer("counts", torch.zeros((k, c), dtype=torch.double))
|
|
832
|
+
self.register_buffer("means", torch.zeros((s, k, t, c)))
|
|
833
|
+
if self.tasks.needs_meansq:
|
|
834
|
+
self.register_buffer("meansq", torch.zeros((k, t, c)))
|
|
835
|
+
else:
|
|
836
|
+
self.meansq = None
|
|
837
|
+
|
|
838
|
+
if self.tasks.needs_double_buffer:
|
|
839
|
+
self.register_buffer("means_new", torch.zeros_like(self.b.means))
|
|
840
|
+
|
|
841
|
+
def setup_coll(self):
|
|
842
|
+
k = self.n_units
|
|
843
|
+
t = self.spike_length_samples
|
|
844
|
+
c = self.n_channels_full
|
|
845
|
+
if not self.tasks.is_coll:
|
|
846
|
+
return
|
|
847
|
+
self.register_buffer("total_weight", torch.zeros((k, t, c), dtype=torch.double))
|
|
848
|
+
|
|
849
|
+
@property
|
|
850
|
+
def raw_means(self):
|
|
851
|
+
return self.b.means[self.raw_subspace_ix]
|
|
852
|
+
|
|
853
|
+
def setup_mixture(self):
|
|
854
|
+
if not self.tasks.n_mixture_passes:
|
|
855
|
+
return
|
|
856
|
+
|
|
857
|
+
k = self.n_units
|
|
858
|
+
t = self.spike_length_samples
|
|
859
|
+
c = self.n_channels_full
|
|
860
|
+
s = self.n_subspaces
|
|
861
|
+
|
|
862
|
+
# -- gamma df
|
|
863
|
+
# initial value for subspace dfs will be used in first mixture pass
|
|
864
|
+
if isinstance(self.initial_df, (float, int)):
|
|
865
|
+
nu = torch.full((s,), self.initial_df)
|
|
866
|
+
else:
|
|
867
|
+
nu = torch.asarray(self.initial_df)
|
|
868
|
+
self.register_buffer("nu", nu)
|
|
869
|
+
# ^ will be updated based on ...
|
|
870
|
+
self.register_buffer("subspace_resp", torch.zeros((s,), dtype=torch.double))
|
|
871
|
+
self.register_buffer("wbar", torch.zeros((s,), dtype=torch.double))
|
|
872
|
+
self.register_buffer("sqwbar", torch.zeros((s,), dtype=torch.double))
|
|
873
|
+
self.register_buffer("logwbar", torch.zeros((s,), dtype=torch.double))
|
|
874
|
+
|
|
875
|
+
# -- pixelwise weights and responsibilities
|
|
876
|
+
# log proportion is zero in first pass
|
|
877
|
+
self.register_buffer("log_props", torch.zeros_like(self.b.means))
|
|
878
|
+
# ^ is updated based on ...
|
|
879
|
+
self.register_buffer(
|
|
880
|
+
"total_resp", torch.zeros_like(self.b.means, dtype=torch.double)
|
|
881
|
+
)
|
|
882
|
+
# total weight doesn't contribute to total resp, but it needs to be
|
|
883
|
+
# known for intermediate averaging and also to rescale sigma ests in finalize
|
|
884
|
+
# they're (sum weight moment)/Nk, not (sum weight moment)/(sum weight)
|
|
885
|
+
self.register_buffer(
|
|
886
|
+
"total_weight", torch.zeros_like(self.b.means, dtype=torch.double)
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
# -- subspace scale params
|
|
890
|
+
if self.tasks.needs_subspace_stds:
|
|
891
|
+
self.register_buffer("scale", torch.zeros_like(self.b.means))
|
|
892
|
+
else:
|
|
893
|
+
self.scale = None
|
|
894
|
+
if self.tasks.needs_double_buffer:
|
|
895
|
+
self.register_buffer("scale_new", torch.zeros_like(self.b.means))
|
|
896
|
+
|
|
897
|
+
def finalize_basic(self):
|
|
898
|
+
# save projected means
|
|
899
|
+
for j in range(self.n_subspaces):
|
|
900
|
+
if j == self.raw_subspace_ix:
|
|
901
|
+
continue
|
|
902
|
+
self.b.means[j].copy_(self.subspace_projectors[j](self.raw_means))
|
|
903
|
+
assert self.b.means.isfinite().all()
|
|
904
|
+
|
|
905
|
+
# we wrote into means here, not means_new, and i am acknowledging that
|
|
906
|
+
if self.tasks.needs_double_buffer:
|
|
907
|
+
pass
|
|
908
|
+
|
|
909
|
+
self.initialize_pi()
|
|
910
|
+
|
|
911
|
+
def finalize_mixture(self):
|
|
912
|
+
assert self.tasks.needs_double_buffer
|
|
913
|
+
|
|
914
|
+
# -- means
|
|
915
|
+
# this update is split into two pieces.
|
|
916
|
+
# first we project the new sample means into their subspaces
|
|
917
|
+
# and put them in the main buffer. then in the stddev update
|
|
918
|
+
# we use both the sample and projected versions. after the
|
|
919
|
+
# stddev update we can zero out the new buffer and be done.
|
|
920
|
+
for j in range(self.n_subspaces):
|
|
921
|
+
if j == self.raw_subspace_ix:
|
|
922
|
+
continue
|
|
923
|
+
self.b.means[j].copy_(self.subspace_projectors[j](self.b.means_new[j]))
|
|
924
|
+
assert self.b.means.isfinite().all()
|
|
925
|
+
|
|
926
|
+
# -- stds
|
|
927
|
+
if self.tasks.updating_subspace_stds:
|
|
928
|
+
# to update sigma, formula as follows. let R be sum of resp,
|
|
929
|
+
# W be sum of weight (ie total weight and total resp).
|
|
930
|
+
# we have currently 1/W sum w y^2. subspace means_new are
|
|
931
|
+
# ybar = 1/W sum w y, but we want projected version mu.
|
|
932
|
+
# so use variance identities to get...
|
|
933
|
+
# 1/R sum w (y-mu)^2 = W/R 1/W sum w [(y-ybar)+(ybar-mu)]^2
|
|
934
|
+
# = W/R [ 1/W sum w (y-ybar)^2 ] + W/R (ybar-mu)^2
|
|
935
|
+
# and lastly
|
|
936
|
+
# 1/W sum w (y-ybar)^2 = [1/W sum w y^2] - ybar^2.
|
|
937
|
+
# W/R can be computed stably as resp avg of weights, 1 when nu=inf.
|
|
938
|
+
assert self.b.scale_new.isfinite().all()
|
|
939
|
+
assert self.b.means_new.isfinite().all()
|
|
940
|
+
assert self.b.means.isfinite().all()
|
|
941
|
+
self.b.scale_new.sub_(self.b.means_new.square())
|
|
942
|
+
bias = self.b.means_new.sub_(self.b.means).square_()
|
|
943
|
+
self.b.scale_new.add_(bias)
|
|
944
|
+
rescale = self.b.total_weight.div(self.total_resp).nan_to_num_()
|
|
945
|
+
self.b.scale_new.mul_(rescale)
|
|
946
|
+
self.b.scale_new.clamp_(min=1e-5).sqrt_()
|
|
947
|
+
self.b.scale.copy_(self.b.scale_new)
|
|
948
|
+
self.b.scale_new.zero_()
|
|
949
|
+
assert self.b.scale.isfinite().all()
|
|
950
|
+
|
|
951
|
+
# //-- means
|
|
952
|
+
self.b.means_new.zero_()
|
|
953
|
+
|
|
954
|
+
# -- proportions
|
|
955
|
+
self.b.total_resp.log_()
|
|
956
|
+
self.b.log_props.copy_(F.log_softmax(self.total_resp, dim=0))
|
|
957
|
+
assert not self.b.log_props.isnan().any()
|
|
958
|
+
self.b.total_weight.zero_()
|
|
959
|
+
self.b.total_resp.zero_()
|
|
960
|
+
|
|
961
|
+
# -- dof
|
|
962
|
+
if self.tasks.updating_df:
|
|
963
|
+
dfs = [
|
|
964
|
+
estimate_gamma_df(wb, logwb, sqwb)
|
|
965
|
+
for wb, logwb, sqwb in zip(self.wbar, self.logwbar, self.sqwbar)
|
|
966
|
+
]
|
|
967
|
+
dfs = torch.tensor(dfs).to(self.b.nu)
|
|
968
|
+
self.b.nu.copy_(dfs)
|
|
969
|
+
self.b.wbar.zero_()
|
|
970
|
+
self.b.logwbar.zero_()
|
|
971
|
+
self.b.sqwbar.zero_()
|
|
972
|
+
# assert self.b.nu.isfinite().all()
|
|
973
|
+
|
|
974
|
+
def initialize_pi(self):
|
|
975
|
+
if not self.tasks.pi_needs_init:
|
|
976
|
+
return
|
|
977
|
+
|
|
978
|
+
from ..templates.get_templates import denoising_weights
|
|
979
|
+
|
|
980
|
+
snrs = ptp(self.raw_means.nan_to_num(nan=-torch.inf)).mul_(
|
|
981
|
+
self.b.counts.to(self.raw_means).sqrt()
|
|
982
|
+
)
|
|
983
|
+
w = denoising_weights(
|
|
984
|
+
snrs.numpy(force=True),
|
|
985
|
+
spike_length_samples=self.spike_length_samples,
|
|
986
|
+
trough_offset=self.trough_offset_samples,
|
|
987
|
+
snr_threshold=self.exp_weight_snr_threshold,
|
|
988
|
+
)
|
|
989
|
+
w = torch.asarray(
|
|
990
|
+
w, dtype=self.b.log_props.dtype, device=self.b.log_props.device
|
|
991
|
+
)
|
|
992
|
+
self.b.log_props.fill_(self.outlier_split_prop)
|
|
993
|
+
self.b.log_props[self.raw_subspace_ix] = w
|
|
994
|
+
wcomp = torch.sub(1.0, w, out=w)
|
|
995
|
+
del w
|
|
996
|
+
if self.use_zero and self.use_svd:
|
|
997
|
+
wsvd = self.exp_split_svd_prop * wcomp
|
|
998
|
+
wzero = (1.0 - self.exp_split_svd_prop) * wcomp
|
|
999
|
+
self.b.log_props[0] = wzero
|
|
1000
|
+
self.b.log_props[self.svd_subspace_ix] = wsvd
|
|
1001
|
+
elif self.use_zero:
|
|
1002
|
+
self.b.log_props[0] = wcomp
|
|
1003
|
+
elif self.use_svd:
|
|
1004
|
+
self.b.log_props[self.svd_subspace_ix] = wcomp
|
|
1005
|
+
else:
|
|
1006
|
+
assert False
|
|
1007
|
+
self.b.log_props.copy_(F.log_softmax(self.b.log_props.log_(), dim=0))
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
@dataclass(kw_only=True)
|
|
1011
|
+
class RunningTemplatesTasks:
|
|
1012
|
+
"""Helper state machine for the RunningTemplates peeler.
|
|
1013
|
+
|
|
1014
|
+
Cooperates with peeler's .compute_template_data() and .finalize_step().
|
|
1015
|
+
Sets updating_* flags read by peeler's chunk process and gather methods
|
|
1016
|
+
so it knows what it's doing.
|
|
1017
|
+
"""
|
|
1018
|
+
|
|
1019
|
+
denoising_method: Literal["none", "exp_weighted", "loot", "coll", "t"]
|
|
1020
|
+
fixed_df: bool
|
|
1021
|
+
with_raw_std_dev: bool
|
|
1022
|
+
t_iters: int
|
|
1023
|
+
loot_cov: Literal["diag", "global"]
|
|
1024
|
+
pi_init: Literal["flat", "exp_weight_split"]
|
|
1025
|
+
|
|
1026
|
+
# things that are requested, so that peeler knows what to setup()
|
|
1027
|
+
needs_meansq: bool = field(init=False)
|
|
1028
|
+
needs_subspace_stds: bool = field(init=False)
|
|
1029
|
+
needs_double_buffer: bool = field(init=False)
|
|
1030
|
+
pi_needs_init: bool = field(init=False)
|
|
1031
|
+
|
|
1032
|
+
# progress tracker
|
|
1033
|
+
n_mixture_passes: int = field(init=False)
|
|
1034
|
+
n_steps: int = field(init=False)
|
|
1035
|
+
active_step: Literal["", "basic", "coll", "mix"] = ""
|
|
1036
|
+
|
|
1037
|
+
# currently being processed
|
|
1038
|
+
updating_meansq: bool = False
|
|
1039
|
+
updating_subspace_stds: bool = False
|
|
1040
|
+
current_step_is_loot: bool = False
|
|
1041
|
+
basic_pass_done: int = False
|
|
1042
|
+
coll_pass_done: int = False
|
|
1043
|
+
mixture_passes_done: int = 0
|
|
1044
|
+
|
|
1045
|
+
def __post_init__(self):
|
|
1046
|
+
assert self.denoising_method in ("none", "exp_weighted", "loot", "coll", "t")
|
|
1047
|
+
self.is_none = self.denoising_method == "none"
|
|
1048
|
+
self.is_loot = self.denoising_method == "loot"
|
|
1049
|
+
self.is_coll = self.denoising_method == "coll"
|
|
1050
|
+
self.is_t = self.denoising_method == "t"
|
|
1051
|
+
self.is_weighted = self.denoising_method == "exp_weighted"
|
|
1052
|
+
self.is_t_or_loot = self.is_t or self.is_loot
|
|
1053
|
+
|
|
1054
|
+
self.n_mixture_passes = (self.t_iters * self.is_t) + self.is_t_or_loot
|
|
1055
|
+
self.n_steps = 1 + self.n_mixture_passes
|
|
1056
|
+
|
|
1057
|
+
self.loot_needs_meansq = self.is_loot and self.loot_cov == "diag"
|
|
1058
|
+
none_with_std = self.is_none and self.with_raw_std_dev
|
|
1059
|
+
self.raw_needs_meansq = none_with_std or self.loot_needs_meansq
|
|
1060
|
+
self.needs_meansq = self.with_raw_std_dev or self.is_t or self.loot_needs_meansq
|
|
1061
|
+
# only t needs to actually instantiate subspace std buffers
|
|
1062
|
+
# for loot, can read it off from meansq and subspace means
|
|
1063
|
+
self.needs_subspace_stds = self.is_t
|
|
1064
|
+
self.needs_double_buffer = self.is_t_or_loot
|
|
1065
|
+
self.pi_needs_init = self.pi_init != "flat" and self.is_t_or_loot
|
|
1066
|
+
|
|
1067
|
+
def step_description(self):
|
|
1068
|
+
s = f"{self.denoising_method}:{self.active_step}"
|
|
1069
|
+
if self.active_step == "basic" and self.needs_meansq:
|
|
1070
|
+
s = f"{s}+meansq"
|
|
1071
|
+
if self.n_steps > 1:
|
|
1072
|
+
istep = 1 + self.basic_pass_done + self.mixture_passes_done
|
|
1073
|
+
s = f"{s} [{istep}/{self.n_steps}]"
|
|
1074
|
+
return s
|
|
1075
|
+
|
|
1076
|
+
def plan_step(self):
|
|
1077
|
+
if self.done:
|
|
1078
|
+
self.active_step = ""
|
|
1079
|
+
logger.dartsortdebug("Template tasks done.")
|
|
1080
|
+
return False
|
|
1081
|
+
elif self.needs_basic_pass:
|
|
1082
|
+
self.active_step = "basic"
|
|
1083
|
+
self.updating_meansq = self.raw_needs_meansq
|
|
1084
|
+
elif self.needs_coll_pass:
|
|
1085
|
+
self.active_step = "coll"
|
|
1086
|
+
self.updating_meansq = self.raw_needs_meansq
|
|
1087
|
+
elif self.needs_mixture_pass:
|
|
1088
|
+
self.active_step = "mix"
|
|
1089
|
+
self.updating_subspace_stds = (
|
|
1090
|
+
self.needs_subspace_stds and not self.is_last_pass
|
|
1091
|
+
)
|
|
1092
|
+
self.updating_df = not (self.is_last_pass or self.fixed_df)
|
|
1093
|
+
self.current_step_is_loot = self.mixture_passes_done == 0
|
|
1094
|
+
else:
|
|
1095
|
+
assert False
|
|
1096
|
+
logger.dartsortdebug(f"Template task: {self.step_description()}.")
|
|
1097
|
+
return True
|
|
1098
|
+
|
|
1099
|
+
def finalize_step(self):
|
|
1100
|
+
if self.done:
|
|
1101
|
+
assert False
|
|
1102
|
+
elif self.needs_basic_pass:
|
|
1103
|
+
self.basic_pass_done = True
|
|
1104
|
+
self.updating_meansq = False
|
|
1105
|
+
self.pi_needs_init = False
|
|
1106
|
+
elif self.needs_coll_pass:
|
|
1107
|
+
self.coll_pass_done = True
|
|
1108
|
+
self.updating_meansq = False
|
|
1109
|
+
self.pi_needs_init = False
|
|
1110
|
+
elif self.needs_mixture_pass:
|
|
1111
|
+
self.mixture_passes_done += 1
|
|
1112
|
+
self.updating_subspace_stds = False
|
|
1113
|
+
self.updating_df = False
|
|
1114
|
+
self.current_step_is_loot = False
|
|
1115
|
+
else:
|
|
1116
|
+
assert False
|
|
1117
|
+
|
|
1118
|
+
@property
|
|
1119
|
+
def done(self):
|
|
1120
|
+
return not (
|
|
1121
|
+
self.needs_basic_pass or self.needs_mixture_pass or self.needs_coll_pass
|
|
1122
|
+
)
|
|
1123
|
+
|
|
1124
|
+
@property
|
|
1125
|
+
def needs_basic_pass(self):
|
|
1126
|
+
return (not self.basic_pass_done) and not self.is_coll
|
|
1127
|
+
|
|
1128
|
+
@property
|
|
1129
|
+
def needs_coll_pass(self):
|
|
1130
|
+
return self.is_coll and not self.coll_pass_done
|
|
1131
|
+
|
|
1132
|
+
@property
|
|
1133
|
+
def needs_mixture_pass(self):
|
|
1134
|
+
return self.mixture_passes_done < self.n_mixture_passes
|
|
1135
|
+
|
|
1136
|
+
@property
|
|
1137
|
+
def is_last_pass(self):
|
|
1138
|
+
if not self.n_mixture_passes:
|
|
1139
|
+
return True
|
|
1140
|
+
else:
|
|
1141
|
+
return self.mixture_passes_done == self.n_mixture_passes - 1
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def registerize(
|
|
1145
|
+
x: torch.Tensor,
|
|
1146
|
+
nc: int,
|
|
1147
|
+
chanix: torch.Tensor | None = None,
|
|
1148
|
+
pad_value=torch.nan,
|
|
1149
|
+
) -> torch.Tensor:
|
|
1150
|
+
if chanix is None:
|
|
1151
|
+
return x
|
|
1152
|
+
reg_x = x.new_full((*x.shape[:2], nc), pad_value)
|
|
1153
|
+
ix = chanix[:, None, :].broadcast_to(x.shape)
|
|
1154
|
+
reg_x.scatter_(src=x, dim=2, index=ix)
|
|
1155
|
+
return reg_x
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def get_gamma_latent(x, labels, mu, nu=1.0, sigma=1.0):
|
|
1159
|
+
nu = torch.asarray(nu)
|
|
1160
|
+
if torch.isinf(nu):
|
|
1161
|
+
return x.isfinite().to(x)
|
|
1162
|
+
if isinstance(sigma, (float, int)):
|
|
1163
|
+
sigma = torch.asarray(float(sigma)).to(x)
|
|
1164
|
+
else:
|
|
1165
|
+
sigma = sigma[labels]
|
|
1166
|
+
z = mu[labels].sub_(x).div_(sigma)
|
|
1167
|
+
denom = z.square_().add_(nu)
|
|
1168
|
+
w = torch.divide(nu + 1.0, denom, out=denom)
|
|
1169
|
+
return w
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
def estimate_gamma_df(wbar, logwbar, sqwbar, cutoff=100, min_df=0.01):
|
|
1173
|
+
"""Solves digamma(nu/2) - log(nu) = 1 + logwbar - wbar."""
|
|
1174
|
+
from scipy.optimize import root_scalar
|
|
1175
|
+
from scipy.special import psi # , polygamma
|
|
1176
|
+
|
|
1177
|
+
dtype = wbar.dtype
|
|
1178
|
+
dev = wbar.device
|
|
1179
|
+
|
|
1180
|
+
wbar = wbar.numpy(force=True).astype(np.float64)
|
|
1181
|
+
logwbar = logwbar.numpy(force=True).astype(np.float64)
|
|
1182
|
+
sqwbar = sqwbar.numpy(force=True).astype(np.float64)
|
|
1183
|
+
|
|
1184
|
+
# start with a guess based on the gamma MoM
|
|
1185
|
+
vw = sqwbar - wbar**2
|
|
1186
|
+
if not np.isfinite(vw) and vw >= 0:
|
|
1187
|
+
raise ValueError(f"Strange gamma statistics {wbar=} {logwbar=} {sqwbar=}.")
|
|
1188
|
+
if vw <= 0:
|
|
1189
|
+
logger.info("Estimated infinite gamma df.")
|
|
1190
|
+
return torch.asarray(torch.inf, device=dev, dtype=dtype)
|
|
1191
|
+
x0 = -0.16 / (1 + logwbar - wbar) # tay
|
|
1192
|
+
x1 = 1.0 / vw # mom
|
|
1193
|
+
|
|
1194
|
+
rhs = 1 + logwbar - wbar
|
|
1195
|
+
min_rhs = psi(min_df / 2) - np.log(min_df / 2)
|
|
1196
|
+
if min_rhs > rhs:
|
|
1197
|
+
logger.dartsortdebug(f"df from {rhs=} too small, setting to {min_df}.")
|
|
1198
|
+
return torch.asarray(min_df, device=dev, dtype=dtype)
|
|
1199
|
+
max_rhs = psi(cutoff / 2) - np.log(cutoff / 2)
|
|
1200
|
+
if max_rhs < rhs:
|
|
1201
|
+
logger.dartsortdebug(f"df from {rhs=} too big, setting to inf.")
|
|
1202
|
+
return torch.asarray(torch.inf, device=dev, dtype=dtype)
|
|
1203
|
+
|
|
1204
|
+
# brent turned out to be better than deriv aware methods
|
|
1205
|
+
def f(x):
|
|
1206
|
+
p = psi(x)
|
|
1207
|
+
# p1 = polygamma(1, x)
|
|
1208
|
+
# p2 = polygamma(2, x)
|
|
1209
|
+
# invx = np.reciprocal(x)
|
|
1210
|
+
f = p - np.log(x) - rhs
|
|
1211
|
+
# df = p1 - invx
|
|
1212
|
+
# ddf = p2 + invx * invx
|
|
1213
|
+
return f # , df, ddf
|
|
1214
|
+
|
|
1215
|
+
init = x0 if x0 > 0 else x1
|
|
1216
|
+
assert init > 0
|
|
1217
|
+
left = x0
|
|
1218
|
+
while f(left) > 0:
|
|
1219
|
+
left /= 2
|
|
1220
|
+
assert left > 0
|
|
1221
|
+
right = x0
|
|
1222
|
+
while f(right) < 0:
|
|
1223
|
+
right *= 2
|
|
1224
|
+
assert right > left > 0
|
|
1225
|
+
if x1 > right or x1 < left:
|
|
1226
|
+
x1 = (right + left) / 2
|
|
1227
|
+
assert left <= x0 <= right
|
|
1228
|
+
assert left <= x1 <= right
|
|
1229
|
+
|
|
1230
|
+
sol = root_scalar(f, bracket=(left, right), x0=x0, x1=x1) # fprime2=True)
|
|
1231
|
+
nu = sol.root / 2.0
|
|
1232
|
+
assert sol.converged
|
|
1233
|
+
|
|
1234
|
+
logger.dartsortdebug(
|
|
1235
|
+
"Gamma nu estimate: final=%s x0=%s for wbar=%s logwbar=%s sqwbar=%s",
|
|
1236
|
+
nu,
|
|
1237
|
+
x0,
|
|
1238
|
+
wbar,
|
|
1239
|
+
logwbar,
|
|
1240
|
+
sqwbar,
|
|
1241
|
+
)
|
|
1242
|
+
if nu > cutoff:
|
|
1243
|
+
nu = torch.inf
|
|
1244
|
+
return torch.asarray(nu, device=dev, dtype=dtype)
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
LOG_PI = math.log(math.pi)
|
|
1248
|
+
LOG_2PI = math.log(math.tau)
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
def smsm_resps_and_latents(
|
|
1252
|
+
x,
|
|
1253
|
+
labels,
|
|
1254
|
+
lps,
|
|
1255
|
+
nus,
|
|
1256
|
+
mus,
|
|
1257
|
+
sigmas=None,
|
|
1258
|
+
meansqs=None,
|
|
1259
|
+
loo=False,
|
|
1260
|
+
loo_N=None,
|
|
1261
|
+
global_sigma=None,
|
|
1262
|
+
projectors=None,
|
|
1263
|
+
):
|
|
1264
|
+
s = len(lps)
|
|
1265
|
+
|
|
1266
|
+
ii, cc = x[:, 0, :].isfinite().nonzero(as_tuple=True)
|
|
1267
|
+
xf = x[ii, :, cc]
|
|
1268
|
+
uu = labels[ii]
|
|
1269
|
+
|
|
1270
|
+
# log liks for computing resps
|
|
1271
|
+
ll = x.new_full((s, *x.shape), -torch.inf)
|
|
1272
|
+
# gamma latent weight
|
|
1273
|
+
w = x.new_zeros((s, *x.shape))
|
|
1274
|
+
|
|
1275
|
+
if loo:
|
|
1276
|
+
assert loo_N is not None
|
|
1277
|
+
assert loo_N.ndim == 2
|
|
1278
|
+
Nf = loo_N[uu, cc][:, None]
|
|
1279
|
+
N_N1 = Nf / (Nf - 1)
|
|
1280
|
+
else:
|
|
1281
|
+
N_N1 = Nf = None
|
|
1282
|
+
|
|
1283
|
+
if sigmas is None:
|
|
1284
|
+
sigmas = [None] * s
|
|
1285
|
+
if meansqs is None:
|
|
1286
|
+
meansqs = [None] * s
|
|
1287
|
+
if projectors is None:
|
|
1288
|
+
projectors = [identity] * s
|
|
1289
|
+
|
|
1290
|
+
params = zip(lps, nus, mus, sigmas, [meansqs] * s, projectors)
|
|
1291
|
+
for j, (lp, nu, mu, sigma, meansq, P) in enumerate(params):
|
|
1292
|
+
lp = lp[uu, :, cc]
|
|
1293
|
+
loc = mu[uu, :, cc]
|
|
1294
|
+
|
|
1295
|
+
if loo and global_sigma is not None:
|
|
1296
|
+
Pxf_N = P(xf).div_(Nf)
|
|
1297
|
+
loc.sub_(Pxf_N).mul_(N_N1)
|
|
1298
|
+
del Pxf_N
|
|
1299
|
+
assert global_sigma is not None
|
|
1300
|
+
scale = global_sigma
|
|
1301
|
+
elif loo and meansq is not None:
|
|
1302
|
+
Pxf = P(xf)
|
|
1303
|
+
Pxfsq_N = Pxf.square().div_(Nf)
|
|
1304
|
+
scale = meansq[uu, :, cc].sub_(Pxfsq_N).mul_(N_N1)
|
|
1305
|
+
del Pxfsq_N
|
|
1306
|
+
Pxf_N = Pxf.div_(Nf)
|
|
1307
|
+
loc.sub_(Pxf_N).mul_(N_N1)
|
|
1308
|
+
del Pxf, Pxf_N
|
|
1309
|
+
scale.sub_(loc.square()).clamp_(min=1e-5).sqrt_()
|
|
1310
|
+
else:
|
|
1311
|
+
assert not loo
|
|
1312
|
+
assert sigma is not None
|
|
1313
|
+
scale = sigma[uu, :, cc]
|
|
1314
|
+
|
|
1315
|
+
zsq = loc.sub(xf).div_(scale).square_()
|
|
1316
|
+
|
|
1317
|
+
if s == 1:
|
|
1318
|
+
l = 0.0 # doesn't matter at all
|
|
1319
|
+
elif nu == 1.0:
|
|
1320
|
+
# t_1 is Cauchy
|
|
1321
|
+
log_scale = (
|
|
1322
|
+
scale.log_()
|
|
1323
|
+
if torch.is_tensor(scale)
|
|
1324
|
+
else torch.tensor(math.log(scale))
|
|
1325
|
+
)
|
|
1326
|
+
del scale
|
|
1327
|
+
l = zsq.add(1.0).log_().neg_().sub_(log_scale.add_(LOG_PI))
|
|
1328
|
+
l.add_(lp)
|
|
1329
|
+
elif nu < torch.inf:
|
|
1330
|
+
l = StudentT(nu, loc=loc, scale=scale, validate_args=False).log_prob(xf)
|
|
1331
|
+
l.add_(lp)
|
|
1332
|
+
else:
|
|
1333
|
+
log_scale = (
|
|
1334
|
+
scale.log_()
|
|
1335
|
+
if torch.is_tensor(scale)
|
|
1336
|
+
else torch.tensor(math.log(scale))
|
|
1337
|
+
)
|
|
1338
|
+
del scale
|
|
1339
|
+
l = zsq.add(log_scale.add_(LOG_2PI)).mul_(-0.5)
|
|
1340
|
+
l.add_(lp)
|
|
1341
|
+
|
|
1342
|
+
ll[j, ii, :, cc] = l
|
|
1343
|
+
|
|
1344
|
+
if nu < torch.inf:
|
|
1345
|
+
denom = zsq.add_(nu)
|
|
1346
|
+
myw = torch.divide(nu + 1.0, denom, out=denom)
|
|
1347
|
+
w[j, ii, :, cc] = myw
|
|
1348
|
+
else:
|
|
1349
|
+
w[j, ii, :, cc] = 1.0
|
|
1350
|
+
|
|
1351
|
+
resp = F.softmax(ll, dim=0)
|
|
1352
|
+
return resp, w
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def count_and_mean(
|
|
1356
|
+
resp,
|
|
1357
|
+
w,
|
|
1358
|
+
x,
|
|
1359
|
+
labels,
|
|
1360
|
+
k,
|
|
1361
|
+
with_sq=False,
|
|
1362
|
+
):
|
|
1363
|
+
s, n, t, c = w.shape
|
|
1364
|
+
assert x.shape == (n, t, c)
|
|
1365
|
+
|
|
1366
|
+
wr = w.double() if resp is None else w.double() * resp
|
|
1367
|
+
del w, resp
|
|
1368
|
+
|
|
1369
|
+
wsum = wr.new_zeros((s, k, t, c), dtype=torch.double)
|
|
1370
|
+
ix = labels[None, :, None, None].broadcast_to(wr.shape)
|
|
1371
|
+
wsum.scatter_add_(dim=1, index=ix, src=wr)
|
|
1372
|
+
|
|
1373
|
+
xbar = x.new_zeros((s, k, t, c))
|
|
1374
|
+
wx = torch.empty_like(x)
|
|
1375
|
+
ix = labels[:, None, None].broadcast_to(wx.shape)
|
|
1376
|
+
for j in range(s):
|
|
1377
|
+
wx.copy_(x)
|
|
1378
|
+
wx.mul_(wr[j] / wsum[j, labels].clamp_(min=1e-10))
|
|
1379
|
+
xbar[j].scatter_add_(dim=0, index=ix, src=wx)
|
|
1380
|
+
|
|
1381
|
+
if with_sq:
|
|
1382
|
+
xsqbar = x.new_zeros((s, k, t, c))
|
|
1383
|
+
xsq = x.square()
|
|
1384
|
+
for j in range(s):
|
|
1385
|
+
wx.copy_(xsq)
|
|
1386
|
+
wx.mul_(wr[j] / wsum[j, labels].clamp_(min=1e-10))
|
|
1387
|
+
xsqbar[j].scatter_add_(dim=0, index=ix, src=wx)
|
|
1388
|
+
else:
|
|
1389
|
+
xsqbar = None
|
|
1390
|
+
|
|
1391
|
+
return wsum, xbar, xsqbar
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def gamma_count_and_mean(resp, w, labels, k, count_only=False):
|
|
1395
|
+
s, n, t, c = w.shape
|
|
1396
|
+
|
|
1397
|
+
rsum = resp.new_zeros((s, k, t, c), dtype=torch.double)
|
|
1398
|
+
ix = labels[None, :, None, None].broadcast_to(w.shape)
|
|
1399
|
+
rsum.scatter_add_(dim=1, index=ix, src=resp.double())
|
|
1400
|
+
|
|
1401
|
+
if count_only:
|
|
1402
|
+
return rsum, None, None, None, None
|
|
1403
|
+
|
|
1404
|
+
rtotal = rsum.sum(dim=(1, 2, 3))
|
|
1405
|
+
weight = resp / rtotal[:, None, None, None]
|
|
1406
|
+
ww = w * weight
|
|
1407
|
+
wbar = ww.sum(dim=(1, 2, 3))
|
|
1408
|
+
sqwbar = (ww * w).sum(dim=(1, 2, 3))
|
|
1409
|
+
logwbar = weight.xlogy(w).sum(dim=(1, 2, 3))
|
|
1410
|
+
|
|
1411
|
+
return rsum, rtotal, wbar, sqwbar, logwbar
|
|
1412
|
+
|
|
1413
|
+
|
|
1414
|
+
def wsum(shp, w, ix, src, dim=0):
|
|
1415
|
+
out = src.new_zeros(shp)
|
|
1416
|
+
out.scatter_add_(dim=dim, ix=ix, src=src.mul(w))
|
|
1417
|
+
|
|
1418
|
+
|
|
1419
|
+
def identity(x):
|
|
1420
|
+
return x.clone()
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def global_std_estimate(sorting, res_ds="residual", q=0.75, eps=1e-6):
|
|
1424
|
+
if sorting.parent_h5_path is None:
|
|
1425
|
+
warnings.warn("Sorting has no residual data, falling back to global std 1.0")
|
|
1426
|
+
return 1.0
|
|
1427
|
+
with h5py.File(sorting.parent_h5_path, "r", locking=False) as h5:
|
|
1428
|
+
ds: h5py.Dataset = h5[res_ds] # pyright: ignore reportAssignmentType
|
|
1429
|
+
mads = np.zeros(len(ds))
|
|
1430
|
+
for sli, res in yield_chunks(
|
|
1431
|
+
h5[res_ds], desc_prefix="MAD", fallback_chunk_length=128
|
|
1432
|
+
):
|
|
1433
|
+
res = np.abs(res, out=res)
|
|
1434
|
+
mads[sli] = np.median(res, axis=(1, 2))
|
|
1435
|
+
mad = np.median(mads)
|
|
1436
|
+
std = (mad / ndtri(q)).item()
|
|
1437
|
+
logger.info(f"Global std dev estimate: {std:0.3f} ({q:0.3f}).")
|
|
1438
|
+
std = max(std, eps) # so things don't crash when running noise-free tests
|
|
1439
|
+
return std
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
def coll_weights(
|
|
1443
|
+
times_samples,
|
|
1444
|
+
waveforms,
|
|
1445
|
+
coll_sorting,
|
|
1446
|
+
chunk_start_samples,
|
|
1447
|
+
chunk_length_samples,
|
|
1448
|
+
trough_offset_samples,
|
|
1449
|
+
left_margin,
|
|
1450
|
+
right_margin,
|
|
1451
|
+
mask_indices,
|
|
1452
|
+
):
|
|
1453
|
+
# TODO drift
|
|
1454
|
+
# TODO tpca
|
|
1455
|
+
# TODO store dataset
|
|
1456
|
+
|
|
1457
|
+
# todo search
|
|
1458
|
+
in_chunk = np.flatnonzero(
|
|
1459
|
+
coll_sorting.times_samples.clip(
|
|
1460
|
+
chunk_start_samples, chunk_start_samples + chunk_length_samples
|
|
1461
|
+
)
|
|
1462
|
+
== coll_sorting.times_samples
|
|
1463
|
+
)
|
|
1464
|
+
|
|
1465
|
+
with h5py.File(coll_sorting.parent_h5_path, "r", locking=False) as h5:
|
|
1466
|
+
ds: np.ndarray = h5["subtracted_waveforms"][in_chunk] # pyright: ignore
|
|
1467
|
+
dsm: np.ndarray = h5["subtracted_waveforms"][mask_indices] # pyright: ignore
|
|
1468
|
+
ci: np.ndarray = h5["subtract_channel_index"][:] # pyright: ignore
|
|
1469
|
+
signal = np.zeros(
|
|
1470
|
+
(chunk_length_samples + left_margin + right_margin, len(ci) + 1),
|
|
1471
|
+
dtype=np.float32,
|
|
1472
|
+
)
|
|
1473
|
+
|
|
1474
|
+
tix = coll_sorting.times_samples[in_chunk] - (chunk_start_samples - left_margin)
|
|
1475
|
+
trel = np.arange(ds.shape[1]) - trough_offset_samples
|
|
1476
|
+
tix = tix[:, None] + trel
|
|
1477
|
+
cix = ci[coll_sorting.channels[in_chunk]]
|
|
1478
|
+
np.add.at(signal, (tix[:, :, None], cix[:, None, :]), ds)
|
|
1479
|
+
|
|
1480
|
+
# now get the collision estimates
|
|
1481
|
+
times_rel = times_samples.numpy(force=True) - (chunk_start_samples - left_margin)
|
|
1482
|
+
signal_snips = signal[times_rel[:, None] + trel]
|
|
1483
|
+
|
|
1484
|
+
cixm = ci[coll_sorting.channels[mask_indices]]
|
|
1485
|
+
np.subtract.at(
|
|
1486
|
+
signal_snips,
|
|
1487
|
+
(
|
|
1488
|
+
np.arange(len(dsm))[:, None, None],
|
|
1489
|
+
np.arange(dsm.shape[1])[None, :, None],
|
|
1490
|
+
cixm[:, None, :],
|
|
1491
|
+
),
|
|
1492
|
+
dsm,
|
|
1493
|
+
)
|
|
1494
|
+
signal_snips = torch.asarray(
|
|
1495
|
+
signal_snips, dtype=waveforms.dtype, device=waveforms.device
|
|
1496
|
+
)
|
|
1497
|
+
signal_snips.square_().add_(1.0).reciprocal_()
|
|
1498
|
+
return signal_snips[..., :-1]
|