DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Literal, cast
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import h5py
|
|
7
|
+
import numpy as np
|
|
8
|
+
import probeinterface
|
|
9
|
+
from scipy.spatial.distance import cdist
|
|
10
|
+
from spikeinterface.core import NumpySorting
|
|
11
|
+
import torch
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
from importlib.resources import files
|
|
15
|
+
except ImportError:
|
|
16
|
+
try:
|
|
17
|
+
from importlib_resources import files # pyright: ignore
|
|
18
|
+
except ImportError:
|
|
19
|
+
raise ValueError("Need python>=3.10 or pip install importlib_resources.")
|
|
20
|
+
|
|
21
|
+
from ..transform import WaveformPipeline
|
|
22
|
+
from ..util.internal_config import FeaturizationConfig, WaveformConfig
|
|
23
|
+
from ..util.data_util import subsample_waveforms, yield_chunks
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
default_temporal_kernel_npy = files("dartsort.pretrained")
|
|
27
|
+
default_temporal_kernel_npy = default_temporal_kernel_npy.joinpath(
|
|
28
|
+
"default_temporal_kernel.npy"
|
|
29
|
+
)
|
|
30
|
+
default_temporal_kernel_npy = Path(str(default_temporal_kernel_npy))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# -- spike train sims
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def refractory_poisson_spike_train(
|
|
37
|
+
rate_hz,
|
|
38
|
+
duration_samples,
|
|
39
|
+
seed: int | np.random.Generator = 0,
|
|
40
|
+
refractory_samples=40,
|
|
41
|
+
trough_offset_samples=42,
|
|
42
|
+
spike_length_samples=121,
|
|
43
|
+
sampling_frequency=30000.0,
|
|
44
|
+
overestimation=2.0,
|
|
45
|
+
):
|
|
46
|
+
"""Sample a refractory Poisson spike train
|
|
47
|
+
|
|
48
|
+
Arguments
|
|
49
|
+
---------
|
|
50
|
+
rate : float
|
|
51
|
+
Spikes / second, well, except it'll be slower due to refractoriness.
|
|
52
|
+
duration : float
|
|
53
|
+
"""
|
|
54
|
+
rg = np.random.default_rng(seed)
|
|
55
|
+
|
|
56
|
+
seconds_per_sample = 1.0 / sampling_frequency
|
|
57
|
+
refractory_s = refractory_samples * seconds_per_sample
|
|
58
|
+
duration_s = duration_samples * seconds_per_sample
|
|
59
|
+
|
|
60
|
+
# overestimate the number of spikes needed
|
|
61
|
+
overest_count = int(duration_s * rate_hz * overestimation)
|
|
62
|
+
overest_count = max(10, overest_count)
|
|
63
|
+
|
|
64
|
+
# generate interspike intervals
|
|
65
|
+
intervals = rg.exponential(scale=1.0 / rate_hz, size=overest_count)
|
|
66
|
+
intervals += refractory_s
|
|
67
|
+
intervals_samples = np.floor(intervals * sampling_frequency).astype(int)
|
|
68
|
+
|
|
69
|
+
# determine spike times and restrict to ones which we can actually
|
|
70
|
+
# add into / read from a recording with this duration and trough offset
|
|
71
|
+
spike_samples = np.cumsum(intervals_samples)
|
|
72
|
+
max_spike_time = duration_samples - (spike_length_samples - trough_offset_samples)
|
|
73
|
+
# check that we overestimated enough
|
|
74
|
+
assert spike_samples.max() > max_spike_time
|
|
75
|
+
valid = spike_samples == spike_samples.clip(trough_offset_samples, max_spike_time)
|
|
76
|
+
spike_samples = spike_samples[valid]
|
|
77
|
+
assert spike_samples.size
|
|
78
|
+
|
|
79
|
+
return spike_samples
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def piecewise_refractory_poisson_spike_train(rates, bins, binsize_samples, **kwargs):
|
|
83
|
+
"""
|
|
84
|
+
Returns a spike train with variable firing rate using refractory_poisson_spike_train().
|
|
85
|
+
|
|
86
|
+
:param rates: list of firing rates in Hz
|
|
87
|
+
:param bins: bin starting samples (same shape as rates)
|
|
88
|
+
:param binsize_samples: number of samples per bin
|
|
89
|
+
:param **kwargs: kwargs to feed to refractory_poisson_spike_train()
|
|
90
|
+
"""
|
|
91
|
+
st = []
|
|
92
|
+
for rate, bin in zip(rates, bins):
|
|
93
|
+
if rate < 0.1:
|
|
94
|
+
continue
|
|
95
|
+
binst = refractory_poisson_spike_train(rate, binsize_samples, **kwargs)
|
|
96
|
+
st.append(bin + binst)
|
|
97
|
+
st = np.concatenate(st)
|
|
98
|
+
return st
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def simulate_sorting(
|
|
102
|
+
num_units,
|
|
103
|
+
n_samples,
|
|
104
|
+
firing_rates=None,
|
|
105
|
+
rg: int | np.random.Generator = 0,
|
|
106
|
+
nbefore: int = 42,
|
|
107
|
+
spike_length_samples: int = 128,
|
|
108
|
+
sampling_frequency=30_000.0,
|
|
109
|
+
globally_refractory=False,
|
|
110
|
+
refractory_samples=40,
|
|
111
|
+
):
|
|
112
|
+
rg = np.random.default_rng(rg)
|
|
113
|
+
|
|
114
|
+
# Default firing rates drawn uniformly from 1-10Hz
|
|
115
|
+
if firing_rates is not None:
|
|
116
|
+
assert firing_rates.shape[0] == num_units
|
|
117
|
+
else:
|
|
118
|
+
firing_rates = rg.uniform(1.0, 10.0, num_units)
|
|
119
|
+
|
|
120
|
+
if not globally_refractory:
|
|
121
|
+
spike_trains = [
|
|
122
|
+
refractory_poisson_spike_train(
|
|
123
|
+
firing_rates[i],
|
|
124
|
+
n_samples,
|
|
125
|
+
trough_offset_samples=nbefore,
|
|
126
|
+
spike_length_samples=spike_length_samples,
|
|
127
|
+
seed=rg,
|
|
128
|
+
refractory_samples=refractory_samples,
|
|
129
|
+
sampling_frequency=sampling_frequency,
|
|
130
|
+
)
|
|
131
|
+
for i in range(num_units)
|
|
132
|
+
]
|
|
133
|
+
if num_units:
|
|
134
|
+
spike_times = np.concatenate(spike_trains)
|
|
135
|
+
spike_labels = np.repeat(
|
|
136
|
+
np.arange(num_units),
|
|
137
|
+
np.array([spike_trains[i].shape[0] for i in range(num_units)]),
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
spike_times = np.array([], dtype=np.int64)
|
|
141
|
+
spike_labels = np.array([], dtype=np.int64)
|
|
142
|
+
else:
|
|
143
|
+
global_rate = np.sum(firing_rates)
|
|
144
|
+
spike_times = refractory_poisson_spike_train(
|
|
145
|
+
global_rate,
|
|
146
|
+
n_samples,
|
|
147
|
+
trough_offset_samples=nbefore,
|
|
148
|
+
spike_length_samples=spike_length_samples,
|
|
149
|
+
seed=rg,
|
|
150
|
+
refractory_samples=refractory_samples,
|
|
151
|
+
sampling_frequency=sampling_frequency,
|
|
152
|
+
)
|
|
153
|
+
unit_proportions = firing_rates / global_rate
|
|
154
|
+
spike_labels = rg.choice(num_units, p=unit_proportions, size=spike_times.size)
|
|
155
|
+
|
|
156
|
+
# order = np.argsort(spike_train)
|
|
157
|
+
# spike_train = spike_train[order]
|
|
158
|
+
# spike_labels = spike_labels[order]
|
|
159
|
+
|
|
160
|
+
sorting = NumpySorting.from_samples_and_labels(
|
|
161
|
+
[spike_times], [spike_labels], sampling_frequency=sampling_frequency
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
return sorting
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# -- spatial utils
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def generate_geom(
|
|
171
|
+
num_columns=4,
|
|
172
|
+
num_contact_per_column=96,
|
|
173
|
+
xpitch=-16,
|
|
174
|
+
ypitch=40,
|
|
175
|
+
x_start=59,
|
|
176
|
+
y_start=20,
|
|
177
|
+
y_shift_per_column: Literal["stagger", "flat"]
|
|
178
|
+
| tuple[float]
|
|
179
|
+
| list[float] = "stagger",
|
|
180
|
+
stagger: float = 20.0,
|
|
181
|
+
sort=True,
|
|
182
|
+
sort_x_down=True,
|
|
183
|
+
):
|
|
184
|
+
"""Defaults match NP1 geometry as returned by ibl-neuropixel."""
|
|
185
|
+
if y_shift_per_column == "stagger" and num_columns == 1:
|
|
186
|
+
y_shift_per_column = [0.0]
|
|
187
|
+
elif y_shift_per_column == "stagger":
|
|
188
|
+
num_columns_even_half = 1 + (num_columns // 2)
|
|
189
|
+
y_shift_per_column = [stagger, 0.0] * num_columns_even_half
|
|
190
|
+
y_shift_per_column = y_shift_per_column[:num_columns]
|
|
191
|
+
elif y_shift_per_column == "flat" or y_shift_per_column is None:
|
|
192
|
+
y_shift_per_column = [0.0] * num_columns
|
|
193
|
+
else:
|
|
194
|
+
assert isinstance(y_shift_per_column, (tuple, list, np.ndarray))
|
|
195
|
+
p = probeinterface.generate_multi_columns_probe(
|
|
196
|
+
num_columns=num_columns,
|
|
197
|
+
num_contact_per_column=num_contact_per_column,
|
|
198
|
+
xpitch=xpitch,
|
|
199
|
+
ypitch=ypitch,
|
|
200
|
+
y_shift_per_column=y_shift_per_column,
|
|
201
|
+
)
|
|
202
|
+
geom = p.contact_positions
|
|
203
|
+
assert geom is not None
|
|
204
|
+
geom[:, 0] += x_start
|
|
205
|
+
geom[:, 1] += y_start
|
|
206
|
+
if sort_x_down:
|
|
207
|
+
order = np.lexsort((geom * [-1, 1]).T)
|
|
208
|
+
elif sort:
|
|
209
|
+
order = np.lexsort(geom.T)
|
|
210
|
+
else:
|
|
211
|
+
order = slice(None)
|
|
212
|
+
geom = geom[order]
|
|
213
|
+
return geom
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def rbf_kernel_sqrt(geom, bandwidth=15.0, dtype="float32"):
|
|
217
|
+
x = geom / (np.sqrt(2.0) * bandwidth)
|
|
218
|
+
k = cdist(x, x, metric="sqeuclidean")
|
|
219
|
+
np.negative(k, out=k)
|
|
220
|
+
np.exp(k, out=k)
|
|
221
|
+
vals, vecs = np.linalg.eigh(k)
|
|
222
|
+
spatial_std = np.sqrt(vals, dtype=dtype)
|
|
223
|
+
spatial_vt = np.ascontiguousarray(vecs.T, dtype=dtype)
|
|
224
|
+
return spatial_std, spatial_vt
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# -- sorting h5 helpers
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
default_sim_featurization_cfg = FeaturizationConfig(
|
|
231
|
+
do_enforce_decrease=False, additional_com_localization=True
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def add_features(h5_path, recording, featurization_cfg):
|
|
236
|
+
with h5py.File(h5_path, "r+", locking=False) as h5:
|
|
237
|
+
geom = cast(h5py.Dataset, h5["geom"])[:]
|
|
238
|
+
channel_index = cast(h5py.Dataset, h5["channel_index"])[:]
|
|
239
|
+
waveforms, fixed_properties = subsample_waveforms(h5=h5)
|
|
240
|
+
if not len(waveforms):
|
|
241
|
+
return
|
|
242
|
+
featurization_cfg = replace(featurization_cfg, do_localization=len(geom) > 1)
|
|
243
|
+
gt_pipeline = WaveformPipeline.from_config(
|
|
244
|
+
featurization_cfg,
|
|
245
|
+
WaveformConfig(),
|
|
246
|
+
geom=geom,
|
|
247
|
+
channel_index=channel_index,
|
|
248
|
+
sampling_frequency=recording.sampling_frequency,
|
|
249
|
+
)
|
|
250
|
+
gt_pipeline.fit(recording, waveforms, **fixed_properties)
|
|
251
|
+
models_dir = h5_path.parent / f"{h5_path.stem}_models"
|
|
252
|
+
models_dir.mkdir(exist_ok=True)
|
|
253
|
+
torch.save(gt_pipeline.state_dict(), models_dir / "featurization_pipeline.pt")
|
|
254
|
+
|
|
255
|
+
wf_dset = cast(h5py.Dataset, h5["collisioncleaned_waveforms"])
|
|
256
|
+
n = wf_dset.shape[0]
|
|
257
|
+
f_dsets = {
|
|
258
|
+
sd.name: h5.create_dataset(
|
|
259
|
+
sd.name, shape=(n, *sd.shape_per_spike), dtype=sd.dtype
|
|
260
|
+
)
|
|
261
|
+
for sd in gt_pipeline.spike_datasets()
|
|
262
|
+
}
|
|
263
|
+
for sli, chunk in yield_chunks(
|
|
264
|
+
h5["collisioncleaned_waveforms"], desc_prefix="Featurize"
|
|
265
|
+
):
|
|
266
|
+
_, feats = gt_pipeline(
|
|
267
|
+
chunk, channels=cast(h5py.Dataset, h5["channels"])[sli]
|
|
268
|
+
)
|
|
269
|
+
for k, v in feats.items():
|
|
270
|
+
f_dsets[k][sli] = v.numpy(force=True)
|
|
File without changes
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn.functional as F
|
|
3
|
+
from torch import vmap
|
|
4
|
+
from torch.func import grad_and_value, hessian
|
|
5
|
+
|
|
6
|
+
from dartsort.util.torch_optimization_util import batched_levenberg_marquardt
|
|
7
|
+
from dartsort.util.waveform_util import channel_subset_by_radius, full_channel_index
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def localize_amplitude_vectors(
|
|
11
|
+
amplitude_vectors,
|
|
12
|
+
geom,
|
|
13
|
+
main_channels,
|
|
14
|
+
channel_index=None,
|
|
15
|
+
radius=None,
|
|
16
|
+
n_channels_subset=None,
|
|
17
|
+
logbarrier=True,
|
|
18
|
+
model="pointsource",
|
|
19
|
+
dtype=torch.double,
|
|
20
|
+
y0=1.0,
|
|
21
|
+
levenberg_marquardt_kwargs=None,
|
|
22
|
+
th_dipole_proj_dist=250.0,
|
|
23
|
+
):
|
|
24
|
+
"""Localize a bunch of amplitude vectors with torch
|
|
25
|
+
|
|
26
|
+
Here, amplitude_vectors is a tensor with shape (n_spikes, c), where
|
|
27
|
+
n_spikes is the number of spikes and c is the number of channels each
|
|
28
|
+
spike was extracted on, according to the channel_index structure
|
|
29
|
+
(which has shape (n_channels_tot, c), matching geom's
|
|
30
|
+
(n_channels_tot, 2) shape). main_channels is the detection
|
|
31
|
+
channel for each spike, and spike i lives on the channels in
|
|
32
|
+
channel_index[main_channels[i]]. Some spikes don't occupy c channels
|
|
33
|
+
when the channel neighborhoods in channel_index are for instance
|
|
34
|
+
determined by a distance threshold, in which case the values
|
|
35
|
+
in channel_index[main_channels[i]] will be n_channels_tot for
|
|
36
|
+
such cases and the values in amplitude_vectors may be nan or
|
|
37
|
+
another filler value.
|
|
38
|
+
|
|
39
|
+
Localization is run on a subset of channels determined by `radius`
|
|
40
|
+
(or `n_channels_subset`), so that it is possible to compute
|
|
41
|
+
amplitude vectors on a larger channel neighborhood than you would
|
|
42
|
+
want to trust your localization model over.
|
|
43
|
+
|
|
44
|
+
Arguments
|
|
45
|
+
---------
|
|
46
|
+
amplitude_vectors : tensor, shape (n_spikes, c)
|
|
47
|
+
geom : tensor, shape (n_channels_tot, 2)
|
|
48
|
+
main_channels : LongTensor, shape (n_spikes,)
|
|
49
|
+
channel_index : LongTensor, shape (n_channels_tot, c)
|
|
50
|
+
radius : float
|
|
51
|
+
Only consider channels closer than this number (in microns)
|
|
52
|
+
to geom[main_channels[i]] for amplitude_vectors[i]
|
|
53
|
+
|
|
54
|
+
Returns
|
|
55
|
+
-------
|
|
56
|
+
dictionary with keys: x, y, z_rel, z_abs, alpha
|
|
57
|
+
All (n_spikes,) tensors
|
|
58
|
+
"""
|
|
59
|
+
# only monopole implemented in torch for now
|
|
60
|
+
# maybe this will become a wrapper function if we want more models.
|
|
61
|
+
# and, this is why we return a dict, different models will have different
|
|
62
|
+
# parameters
|
|
63
|
+
assert model in ("com", "pointsource", "dipole")
|
|
64
|
+
n_spikes, c = amplitude_vectors.shape
|
|
65
|
+
n_channels_tot = len(geom)
|
|
66
|
+
if channel_index is None:
|
|
67
|
+
assert c == n_channels_tot
|
|
68
|
+
channel_index = full_channel_index(n_channels_tot)
|
|
69
|
+
assert channel_index.shape == (n_channels_tot, c)
|
|
70
|
+
assert main_channels.shape == (n_spikes,)
|
|
71
|
+
# we'll return numpy if user sent numpy
|
|
72
|
+
is_numpy = not torch.is_tensor(amplitude_vectors)
|
|
73
|
+
|
|
74
|
+
# handle channel subsetting
|
|
75
|
+
if radius is not None or n_channels_subset is not None:
|
|
76
|
+
amplitude_vectors, channel_index = channel_subset_by_radius(
|
|
77
|
+
amplitude_vectors,
|
|
78
|
+
main_channels,
|
|
79
|
+
channel_index,
|
|
80
|
+
geom,
|
|
81
|
+
radius=radius,
|
|
82
|
+
n_channels_subset=n_channels_subset,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# torch everyone
|
|
86
|
+
amplitude_vectors = torch.as_tensor(
|
|
87
|
+
amplitude_vectors,
|
|
88
|
+
dtype=dtype,
|
|
89
|
+
)
|
|
90
|
+
device = amplitude_vectors.device
|
|
91
|
+
geom = torch.as_tensor(geom, dtype=dtype, device=device)
|
|
92
|
+
channel_index = torch.as_tensor(channel_index, device=device)
|
|
93
|
+
|
|
94
|
+
# nan to num to avoid some masking
|
|
95
|
+
amplitude_vectors = torch.nan_to_num(amplitude_vectors)
|
|
96
|
+
|
|
97
|
+
# figure out which chans are outside the probe
|
|
98
|
+
# so that we can avoid counting them in optimization
|
|
99
|
+
in_probe_channel_index = (channel_index < n_channels_tot).to(dtype)
|
|
100
|
+
in_probe_mask = in_probe_channel_index[main_channels]
|
|
101
|
+
|
|
102
|
+
# local geometries of each amplitude vector
|
|
103
|
+
geom_pad = F.pad(geom, (0, 0, 0, 1))
|
|
104
|
+
local_geoms = geom_pad[channel_index[main_channels]]
|
|
105
|
+
local_geoms[:, :, 1] -= geom[main_channels, 1][:, None]
|
|
106
|
+
|
|
107
|
+
# center of mass initialization
|
|
108
|
+
com = torch.divide(
|
|
109
|
+
(amplitude_vectors[:, :, None] * local_geoms).sum(1),
|
|
110
|
+
amplitude_vectors.sum(1)[:, None],
|
|
111
|
+
)
|
|
112
|
+
xcom, zcom = com.T
|
|
113
|
+
|
|
114
|
+
if model == "com":
|
|
115
|
+
z_abs_com = zcom + geom[main_channels, 1]
|
|
116
|
+
return dict(x=xcom, z_rel=zcom, z_abs=z_abs_com)
|
|
117
|
+
|
|
118
|
+
# normalized PTP vectors
|
|
119
|
+
# this helps to keep the objective in a similar range, so we can use
|
|
120
|
+
# fixed constants in regularizers like the log barrier
|
|
121
|
+
max_amplitudes = torch.max(amplitude_vectors, dim=1).values
|
|
122
|
+
normalized_amp_vecs = amplitude_vectors / max_amplitudes[:, None]
|
|
123
|
+
|
|
124
|
+
# -- torch optimize
|
|
125
|
+
if levenberg_marquardt_kwargs is None:
|
|
126
|
+
levenberg_marquardt_kwargs = {}
|
|
127
|
+
|
|
128
|
+
# initialize with center of mass
|
|
129
|
+
locs = torch.column_stack((xcom, torch.full_like(xcom, y0), zcom))
|
|
130
|
+
|
|
131
|
+
if model == "pointsource":
|
|
132
|
+
locs, i = batched_levenberg_marquardt(
|
|
133
|
+
locs,
|
|
134
|
+
vmap_point_source_grad_and_mse,
|
|
135
|
+
vmap_point_source_hessian,
|
|
136
|
+
extra_args=(normalized_amp_vecs, in_probe_mask, local_geoms),
|
|
137
|
+
**levenberg_marquardt_kwargs,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# finish: get alpha closed form
|
|
141
|
+
x, y0, z_rel = locs.T
|
|
142
|
+
y = F.softplus(y0)
|
|
143
|
+
alpha = vmap_point_source_find_alpha(
|
|
144
|
+
amplitude_vectors, in_probe_mask, x, y, z_rel, local_geoms
|
|
145
|
+
)
|
|
146
|
+
z_abs = z_rel + geom[main_channels, 1]
|
|
147
|
+
|
|
148
|
+
results = dict(x=x, y=y, z_rel=z_rel, z_abs=z_abs, alpha=alpha)
|
|
149
|
+
if is_numpy:
|
|
150
|
+
results = {k: v.numpy(force=True) for k, v in results.items()}
|
|
151
|
+
return results
|
|
152
|
+
|
|
153
|
+
elif model == "dipole":
|
|
154
|
+
locs, i = batched_levenberg_marquardt(
|
|
155
|
+
locs,
|
|
156
|
+
vmap_dipole_grad_and_mse,
|
|
157
|
+
vmap_dipole_hessian,
|
|
158
|
+
extra_args=(normalized_amp_vecs, local_geoms),
|
|
159
|
+
**levenberg_marquardt_kwargs,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
x, y0, z_rel = locs.T
|
|
163
|
+
y = F.softplus(y0)
|
|
164
|
+
projected_dist = vmap_dipole_find_projection_distance(
|
|
165
|
+
normalized_amp_vecs, x, y, z_rel, local_geoms
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
z_abs = z_rel + geom[main_channels, 1]
|
|
169
|
+
|
|
170
|
+
if is_numpy:
|
|
171
|
+
x = x.numpy(force=True)
|
|
172
|
+
y = y.numpy(force=True)
|
|
173
|
+
z_rel = z_rel.numpy(force=True)
|
|
174
|
+
z_abs = z_abs.numpy(force=True)
|
|
175
|
+
alpha = alpha.numpy(force=True)
|
|
176
|
+
|
|
177
|
+
return dict(x=x, y=y, z_rel=z_rel, z_abs=z_abs, alpha=projected_dist)
|
|
178
|
+
|
|
179
|
+
else:
|
|
180
|
+
assert False
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# -- point source / dipole model library functions
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def point_source_amplitude_at(x, y, z, alpha, local_geom):
|
|
187
|
+
"""Point source model predicted amplitude at local_geom given location"""
|
|
188
|
+
dxs = torch.square(x - local_geom[:, 0])
|
|
189
|
+
dzs = torch.square(z - local_geom[:, 1])
|
|
190
|
+
dys = torch.square(y)
|
|
191
|
+
return alpha / torch.sqrt(dxs + dzs + dys)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def point_source_find_alpha(amp_vec, channel_mask, x, y, z, local_geoms):
|
|
195
|
+
"""We can solve for the brightness (alpha) of the source in closed form given x,y,z"""
|
|
196
|
+
amp1_vec = point_source_amplitude_at(x, y, z, 1.0, local_geoms)
|
|
197
|
+
alpha = torch.divide(
|
|
198
|
+
(amp1_vec * amp_vec * channel_mask).sum(),
|
|
199
|
+
torch.square(amp1_vec * channel_mask).sum(),
|
|
200
|
+
)
|
|
201
|
+
return alpha
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def dipole_find_projection_distance(normalized_amp_vec, x, y, z, local_geom):
|
|
205
|
+
"""COmpute a value dist/dipole in x,z that tells us if dipole or monopole is better"""
|
|
206
|
+
|
|
207
|
+
dxs = x - local_geom[:, 0]
|
|
208
|
+
dzs = z - local_geom[:, 1]
|
|
209
|
+
dys = y.expand(dzs.size())
|
|
210
|
+
duv = torch.stack([dxs, dys, dzs], dim=1)
|
|
211
|
+
X = duv / torch.pow(torch.sum(torch.square(duv), dim=1), 3 / 2)[:, None]
|
|
212
|
+
beta = torch.matmul(
|
|
213
|
+
torch.linalg.pinv(torch.matmul(X.T, X)), torch.matmul(X.T, normalized_amp_vec)
|
|
214
|
+
)
|
|
215
|
+
beta /= torch.sqrt(torch.square(beta).sum())
|
|
216
|
+
dipole_planar_direction = torch.sqrt(torch.square(beta[[0, 2]]).sum())
|
|
217
|
+
closest_chan = torch.argmin(torch.sum(torch.square(duv), dim=1))
|
|
218
|
+
# workaround around vmap doesn't work for one dim tensor .item()
|
|
219
|
+
min_duv = duv[closest_chan[None]][0]
|
|
220
|
+
val_th = torch.sqrt(torch.square(min_duv).sum()) / dipole_planar_direction
|
|
221
|
+
return val_th
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def point_source_mse(loc, amplitude_vector, channel_mask, local_geom, logbarrier=True):
|
|
225
|
+
"""Objective in point source model
|
|
226
|
+
|
|
227
|
+
Arguments
|
|
228
|
+
---------
|
|
229
|
+
loc : tensor of shape (3,)
|
|
230
|
+
Here, this is the x, y0, z positions, where y = softplus(y0)
|
|
231
|
+
amplitude_vector : tensor of shape (n_chans,)
|
|
232
|
+
channel_mask : tensor of shape (n_chans,)
|
|
233
|
+
A binary mask. Channels with 0s here are excluded from the
|
|
234
|
+
objective, useful for amplitudes from waveforms extracted
|
|
235
|
+
on a sparse/incomplete set of channels.
|
|
236
|
+
local_geom : tensor of shape (n_chans, 2)
|
|
237
|
+
logbarrier : bool
|
|
238
|
+
|
|
239
|
+
Returns
|
|
240
|
+
-------
|
|
241
|
+
obj : scalar
|
|
242
|
+
The objective, to be minimized
|
|
243
|
+
"""
|
|
244
|
+
x, y0, z = loc
|
|
245
|
+
y = F.softplus(y0)
|
|
246
|
+
|
|
247
|
+
alpha = point_source_find_alpha(amplitude_vector, channel_mask, x, y, z, local_geom)
|
|
248
|
+
obj = torch.square(
|
|
249
|
+
amplitude_vector - point_source_amplitude_at(x, y, z, alpha, local_geom)
|
|
250
|
+
).mean()
|
|
251
|
+
if logbarrier:
|
|
252
|
+
obj -= torch.log(10.0 * y) / 10000.0
|
|
253
|
+
# idea for logbarrier on points which run away
|
|
254
|
+
# obj -= torch.log(1000.0 - torch.sqrt(torch.square(x) + torch.square(z))).sum() / 10000.0
|
|
255
|
+
return obj
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def dipole_mse(loc, amplitude_vector, local_geom, logbarrier=True):
|
|
259
|
+
"""Dipole model predicted amplitude at local_geom given location"""
|
|
260
|
+
|
|
261
|
+
x, y0, z = loc
|
|
262
|
+
y = F.softplus(y0)
|
|
263
|
+
|
|
264
|
+
dxs = x - local_geom[:, 0]
|
|
265
|
+
dzs = z - local_geom[:, 1]
|
|
266
|
+
dys = y.expand(dzs.size())
|
|
267
|
+
|
|
268
|
+
duv = torch.stack([dxs, dys, dzs], dim=1)
|
|
269
|
+
X = duv / torch.pow(torch.sum(torch.square(duv), dim=1), 3 / 2)[:, None]
|
|
270
|
+
# beta = torch.linalg.lstsq(X, amplitude_vector[:, None])[0]
|
|
271
|
+
# beta = torch.linalg.solve(torch.matmul(X.T, X), torch.matmul(X.T, amplitude_vector))
|
|
272
|
+
beta = torch.matmul(
|
|
273
|
+
torch.linalg.pinv(torch.matmul(X.T, X)), torch.matmul(X.T, amplitude_vector)
|
|
274
|
+
)
|
|
275
|
+
qtq = torch.matmul(X, beta)
|
|
276
|
+
|
|
277
|
+
obj = torch.square(amplitude_vector - qtq).mean()
|
|
278
|
+
if logbarrier:
|
|
279
|
+
obj -= torch.log(10.0 * y) / 10000.0
|
|
280
|
+
|
|
281
|
+
return obj
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# vmapped functions for use in the optimizer, and might be handy for users too
|
|
285
|
+
vmap_point_source_grad_and_mse = vmap(grad_and_value(point_source_mse))
|
|
286
|
+
vmap_point_source_hessian = vmap(hessian(point_source_mse))
|
|
287
|
+
vmap_point_source_find_alpha = vmap(point_source_find_alpha)
|
|
288
|
+
|
|
289
|
+
vmap_dipole_grad_and_mse = vmap(grad_and_value(dipole_mse))
|
|
290
|
+
vmap_dipole_hessian = vmap(hessian(dipole_mse))
|
|
291
|
+
vmap_dipole_find_projection_distance = vmap(dipole_find_projection_distance)
|