DARTsort 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort/__init__.py +68 -0
- dartsort/cli.py +105 -0
- dartsort/clustering/__init__.py +3 -0
- dartsort/clustering/cluster_util.py +392 -0
- dartsort/clustering/clustering.py +765 -0
- dartsort/clustering/clustering_features.py +202 -0
- dartsort/clustering/density.py +1299 -0
- dartsort/clustering/forward_backward.py +219 -0
- dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort/clustering/kmeans.py +448 -0
- dartsort/clustering/merge.py +501 -0
- dartsort/clustering/modes.py +214 -0
- dartsort/clustering/refine_util.py +191 -0
- dartsort/clustering/relocate.py +109 -0
- dartsort/clustering/split.py +1285 -0
- dartsort/config.py +288 -0
- dartsort/detect/__init__.py +2 -0
- dartsort/detect/detect.py +449 -0
- dartsort/detect/detect_filters.py +37 -0
- dartsort/evaluate/__init__.py +1 -0
- dartsort/evaluate/analysis.py +594 -0
- dartsort/evaluate/comparison.py +534 -0
- dartsort/evaluate/hybrid_util.py +599 -0
- dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort/evaluate/simkit.py +840 -0
- dartsort/evaluate/simlib.py +270 -0
- dartsort/localize/__init__.py +0 -0
- dartsort/localize/localize_torch.py +291 -0
- dartsort/localize/localize_util.py +313 -0
- dartsort/main.py +657 -0
- dartsort/peel/__init__.py +6 -0
- dartsort/peel/grab.py +208 -0
- dartsort/peel/matching.py +628 -0
- dartsort/peel/matching_util/__init__.py +3 -0
- dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort/peel/matching_util/drifty.py +791 -0
- dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort/peel/peel_base.py +1000 -0
- dartsort/peel/running_template.py +1498 -0
- dartsort/peel/subtract.py +1123 -0
- dartsort/peel/threshold.py +471 -0
- dartsort/peel/universal.py +191 -0
- dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort/pretrained/detect_np1.pt +0 -0
- dartsort/pretrained/detect_np2.pt +0 -0
- dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort/templates/__init__.py +11 -0
- dartsort/templates/get_templates.py +771 -0
- dartsort/templates/postprocess_util.py +555 -0
- dartsort/templates/realignment.py +466 -0
- dartsort/templates/superres_util.py +204 -0
- dartsort/templates/template_util.py +518 -0
- dartsort/templates/templates.py +528 -0
- dartsort/transform/__init__.py +2 -0
- dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort/transform/all_transformers.py +58 -0
- dartsort/transform/amortized_localization.py +503 -0
- dartsort/transform/amplitudes.py +197 -0
- dartsort/transform/decollider.py +635 -0
- dartsort/transform/enforce_decrease.py +158 -0
- dartsort/transform/localize.py +86 -0
- dartsort/transform/matching_denoiser.py +44 -0
- dartsort/transform/pipeline.py +391 -0
- dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort/transform/single_channel_templates.py +98 -0
- dartsort/transform/supervised_denoiser.py +486 -0
- dartsort/transform/temporal_pca.py +413 -0
- dartsort/transform/transform_base.py +255 -0
- dartsort/util/__init__.py +0 -0
- dartsort/util/cli_util.py +179 -0
- dartsort/util/data_util.py +1128 -0
- dartsort/util/drift_util.py +1054 -0
- dartsort/util/internal_config.py +1074 -0
- dartsort/util/interpolation_util.py +1012 -0
- dartsort/util/job_util.py +19 -0
- dartsort/util/logging_util.py +91 -0
- dartsort/util/main_util.py +235 -0
- dartsort/util/more_operators.py +269 -0
- dartsort/util/multiprocessing_util.py +211 -0
- dartsort/util/nn_util.py +461 -0
- dartsort/util/noise_util.py +1461 -0
- dartsort/util/peel_util.py +223 -0
- dartsort/util/py_util.py +213 -0
- dartsort/util/registration_util.py +128 -0
- dartsort/util/residual_util.py +30 -0
- dartsort/util/sparse_util.py +81 -0
- dartsort/util/spikeio.py +227 -0
- dartsort/util/spiketorch.py +1198 -0
- dartsort/util/testing_util/__init__.py +0 -0
- dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort/util/torch_optimization_util.py +156 -0
- dartsort/util/torch_util.py +75 -0
- dartsort/util/universal_util.py +269 -0
- dartsort/util/waveform_util.py +809 -0
- dartsort/vis/__init__.py +19 -0
- dartsort/vis/analysis_plots.py +392 -0
- dartsort/vis/colors.py +36 -0
- dartsort/vis/gt.py +392 -0
- dartsort/vis/layout.py +149 -0
- dartsort/vis/metrics_util.py +23 -0
- dartsort/vis/mixture.py +1489 -0
- dartsort/vis/over_time.py +514 -0
- dartsort/vis/recanim.py +429 -0
- dartsort/vis/scatterplots.py +779 -0
- dartsort/vis/sorting.py +163 -0
- dartsort/vis/unit.py +917 -0
- dartsort/vis/unit_comparison.py +1017 -0
- dartsort/vis/versus.py +300 -0
- dartsort/vis/vismain.py +537 -0
- dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5.dist-info/METADATA +93 -0
- dartsort-0.3.5.dist-info/RECORD +125 -0
- dartsort-0.3.5.dist-info/WHEEL +5 -0
- dartsort-0.3.5.dist-info/entry_points.txt +2 -0
- dartsort-0.3.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Utilities for making toy data and fitting PPCA or mixtures thereof to it"""
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import torch
|
|
7
|
+
import torch.nn.functional as F
|
|
8
|
+
from sklearn.metrics import adjusted_rand_score
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def simulate_moppca(
|
|
12
|
+
Nper=2**12,
|
|
13
|
+
rank=4,
|
|
14
|
+
nc=8,
|
|
15
|
+
M=2,
|
|
16
|
+
n_missing=2,
|
|
17
|
+
K=5,
|
|
18
|
+
t_mu: Literal["zero", "smooth", "random"] = "smooth",
|
|
19
|
+
t_cov: Literal["eye", "random", "eyesmall"] = "eye",
|
|
20
|
+
# zero, hot, random,
|
|
21
|
+
t_w: Literal["zero", "hot", "smooth", "random"] = "zero",
|
|
22
|
+
t_missing: Literal[
|
|
23
|
+
None, "random", "random_no_extrap", "skewed", "by_cluster"
|
|
24
|
+
] = None,
|
|
25
|
+
init_label_corruption: float = 0.0,
|
|
26
|
+
snr: float = 10.0,
|
|
27
|
+
rg: int | np.random.Generator = 0,
|
|
28
|
+
device=None,
|
|
29
|
+
):
|
|
30
|
+
from dartsort.transform import TemporalPCAFeaturizer
|
|
31
|
+
from dartsort.util.data_util import DARTsortSorting
|
|
32
|
+
from dartsort.util.noise_util import EmbeddedNoise
|
|
33
|
+
from dartsort.clustering.gmm.stable_features import (
|
|
34
|
+
StableSpikeDataset,
|
|
35
|
+
SpikeNeighborhoods,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
N = Nper * K
|
|
39
|
+
|
|
40
|
+
rg = np.random.default_rng(rg)
|
|
41
|
+
D = rank * nc
|
|
42
|
+
|
|
43
|
+
if device is None:
|
|
44
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
45
|
+
device = torch.device(device)
|
|
46
|
+
|
|
47
|
+
clus_neighbs = None
|
|
48
|
+
clus_mask = None
|
|
49
|
+
if t_missing == "by_cluster":
|
|
50
|
+
# in this case, missing chans per clus are never observed,
|
|
51
|
+
# so we should treat them as 0 in W and mu to avoid issues with metrics
|
|
52
|
+
clus_neighbs = [
|
|
53
|
+
rg.choice(nc, size=nc - n_missing, replace=False) for _ in range(K)
|
|
54
|
+
]
|
|
55
|
+
clus_neighbs = np.array(clus_neighbs)
|
|
56
|
+
clus_neighbs = np.sort(clus_neighbs, axis=1)
|
|
57
|
+
clus_mask = np.zeros((K, nc))
|
|
58
|
+
for j, n in enumerate(clus_neighbs):
|
|
59
|
+
clus_mask[j, n] = 1.0
|
|
60
|
+
if t_missing in (None, "none"):
|
|
61
|
+
n_missing = 0
|
|
62
|
+
|
|
63
|
+
if t_mu == "zero":
|
|
64
|
+
mu = np.zeros((K, rank, nc))
|
|
65
|
+
elif t_mu == "random":
|
|
66
|
+
mu = snr * rg.normal(size=(K, rank, nc))
|
|
67
|
+
elif t_mu == "smooth":
|
|
68
|
+
phase = rg.uniform(0.0, 2 * np.pi, size=(K, rank, 1))
|
|
69
|
+
freq = rg.uniform(0.2, 1.0, size=(K, rank, 1))
|
|
70
|
+
amp = rg.uniform(1.0, 2.0, size=(K, rank, 1))
|
|
71
|
+
domain = np.linspace(0, 2 * np.pi, endpoint=False, num=nc)
|
|
72
|
+
mu = snr * amp * np.sin(phase + freq * domain)
|
|
73
|
+
else:
|
|
74
|
+
assert False
|
|
75
|
+
if clus_mask is not None:
|
|
76
|
+
mu = mu * clus_mask[:, None]
|
|
77
|
+
|
|
78
|
+
if t_cov == "eye":
|
|
79
|
+
cov = np.eye(D)
|
|
80
|
+
noise = EmbeddedNoise(
|
|
81
|
+
rank,
|
|
82
|
+
nc,
|
|
83
|
+
cov_kind="scalar",
|
|
84
|
+
global_std=torch.tensor(1.0, dtype=torch.float, device=device),
|
|
85
|
+
)
|
|
86
|
+
elif t_cov == "eyesmall":
|
|
87
|
+
cov = 1e-4 * np.eye(D)
|
|
88
|
+
noise = EmbeddedNoise(
|
|
89
|
+
rank,
|
|
90
|
+
nc,
|
|
91
|
+
cov_kind="scalar",
|
|
92
|
+
global_std=torch.tensor(0.01, dtype=torch.float, device=device),
|
|
93
|
+
)
|
|
94
|
+
elif t_cov == "random":
|
|
95
|
+
_c = rg.normal(size=(D, 10 * D))
|
|
96
|
+
cov = _c @ _c.T / (10 * D)
|
|
97
|
+
full_cov = torch.asarray(cov, dtype=torch.float, device=device).view(
|
|
98
|
+
rank, nc, rank, nc
|
|
99
|
+
)
|
|
100
|
+
noise = EmbeddedNoise(rank, nc, cov_kind="full", full_cov=full_cov)
|
|
101
|
+
else:
|
|
102
|
+
assert False
|
|
103
|
+
|
|
104
|
+
if t_w == "hot":
|
|
105
|
+
W = np.zeros((K, rank, nc, M))
|
|
106
|
+
for j in range(K):
|
|
107
|
+
for m in range(M):
|
|
108
|
+
W[j, m, j, m] = 10 - m
|
|
109
|
+
elif t_w == "random":
|
|
110
|
+
W = rg.normal(size=(K, rank, nc, M))
|
|
111
|
+
elif t_w == "smooth":
|
|
112
|
+
phase = rg.uniform(0.0, 2 * np.pi, size=(K, rank, 1, M))
|
|
113
|
+
freq = rg.uniform(0.2, 1.0, size=(K, rank, 1, M))
|
|
114
|
+
amp = rg.uniform(1.0, 2.0, size=(K, rank, 1, M))
|
|
115
|
+
domain = np.linspace(0, 2 * np.pi, endpoint=False, num=nc)
|
|
116
|
+
W = amp * np.exp(-(np.sin(phase + freq * domain[:, None])**2))
|
|
117
|
+
elif t_w == "zero":
|
|
118
|
+
W = np.zeros((K, rank, nc, M))
|
|
119
|
+
else:
|
|
120
|
+
assert False
|
|
121
|
+
if clus_mask is not None:
|
|
122
|
+
W = W * clus_mask[:, None, :, None]
|
|
123
|
+
|
|
124
|
+
labels = rg.integers(K, size=N)
|
|
125
|
+
u = rg.normal(size=(N, M))
|
|
126
|
+
eps = rg.multivariate_normal(mean=np.zeros(D), cov=cov, size=N)
|
|
127
|
+
|
|
128
|
+
labels = torch.asarray(labels)
|
|
129
|
+
cov = noise.marginal_covariance()
|
|
130
|
+
u = torch.asarray(u, dtype=torch.float)
|
|
131
|
+
eps = torch.asarray(eps, dtype=torch.float)
|
|
132
|
+
mu = torch.asarray(mu, dtype=torch.float)
|
|
133
|
+
W = torch.asarray(W, dtype=torch.float)
|
|
134
|
+
|
|
135
|
+
y = mu[labels] + torch.einsum("nrcm,nm->nrc", W[labels], u) + eps.view(N, rank, nc)
|
|
136
|
+
|
|
137
|
+
x = None
|
|
138
|
+
if not t_missing:
|
|
139
|
+
x = y
|
|
140
|
+
channels = torch.arange(nc).unsqueeze(0).broadcast_to(N, nc)
|
|
141
|
+
elif t_missing.startswith("random"):
|
|
142
|
+
assert n_missing == 2
|
|
143
|
+
no_extrap = int(t_missing == "random_no_extrap")
|
|
144
|
+
possible_neighbs = np.zeros(nc - n_missing, dtype=np.int64)[None]
|
|
145
|
+
npair = ((nc - 2 * no_extrap) * (nc - 1 - 2 * no_extrap)) // 2
|
|
146
|
+
possible_neighbs = np.broadcast_to(possible_neighbs, (npair, nc - n_missing))
|
|
147
|
+
possible_neighbs = np.ascontiguousarray(possible_neighbs)
|
|
148
|
+
mask = np.ones(nc, dtype=bool)
|
|
149
|
+
ct = 0
|
|
150
|
+
for i in range(no_extrap, nc - no_extrap):
|
|
151
|
+
mask[i] = False
|
|
152
|
+
for j in range(i + 1, nc - no_extrap):
|
|
153
|
+
mask[j] = False
|
|
154
|
+
possible_neighbs[ct] = np.flatnonzero(mask)
|
|
155
|
+
mask[j] = True
|
|
156
|
+
ct += 1
|
|
157
|
+
mask[i] = True
|
|
158
|
+
assert ct == npair
|
|
159
|
+
channels = possible_neighbs[rg.integers(npair, size=N)]
|
|
160
|
+
elif t_missing == "skewed":
|
|
161
|
+
nc_miss = min(2, nc - 2)
|
|
162
|
+
assert nc_miss > 0
|
|
163
|
+
n0 = np.arange(nc - nc_miss)
|
|
164
|
+
n1 = nc_miss + n0
|
|
165
|
+
neighbs = np.stack((n0, n1), axis=0)
|
|
166
|
+
ns_missing = int(N / 3)
|
|
167
|
+
choices = np.concatenate(
|
|
168
|
+
(
|
|
169
|
+
np.zeros(ns_missing, dtype=np.int64),
|
|
170
|
+
np.ones(N - ns_missing, dtype=np.int64),
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
rg.shuffle(choices)
|
|
174
|
+
channels = neighbs[choices]
|
|
175
|
+
elif t_missing == "by_cluster":
|
|
176
|
+
assert clus_neighbs is not None
|
|
177
|
+
channels = clus_neighbs[labels]
|
|
178
|
+
else:
|
|
179
|
+
assert False
|
|
180
|
+
channels = torch.asarray(channels, dtype=torch.long)
|
|
181
|
+
|
|
182
|
+
if t_missing:
|
|
183
|
+
x = torch.take_along_dim(y, channels.unsqueeze(1), dim=2)
|
|
184
|
+
assert x is not None
|
|
185
|
+
|
|
186
|
+
neighbs = SpikeNeighborhoods.from_channels(channels, nc, device=device)
|
|
187
|
+
|
|
188
|
+
init_labels = labels.clone()
|
|
189
|
+
if init_label_corruption:
|
|
190
|
+
to_corrupt = rg.binomial(1, init_label_corruption, size=N).astype(bool)
|
|
191
|
+
init_labels[to_corrupt] = torch.from_numpy(
|
|
192
|
+
rg.integers(K, size=to_corrupt.sum())
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
init_sorting = DARTsortSorting(
|
|
196
|
+
times_samples=np.arange(N),
|
|
197
|
+
channels=np.zeros(N, dtype=np.int64),
|
|
198
|
+
labels=init_labels.numpy(force=True),
|
|
199
|
+
sampling_frequency=100.0,
|
|
200
|
+
ephemeral_features=dict(times_seconds=np.arange(N) / 100.0),
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
_tpca = TemporalPCAFeaturizer(
|
|
204
|
+
channel_index=torch.zeros(nc, nc - n_missing, dtype=torch.long),
|
|
205
|
+
rank=rank,
|
|
206
|
+
)
|
|
207
|
+
_tpca = _tpca.to(device)
|
|
208
|
+
splits = rg.binomial(1, p=0.3, size=N)
|
|
209
|
+
|
|
210
|
+
prgeom = 15.0 * torch.arange(nc, dtype=torch.float)
|
|
211
|
+
prgeom = torch.stack((torch.zeros(nc), prgeom), dim=1)
|
|
212
|
+
prgeom = F.pad(prgeom, (0, 0, 0, 1), value=torch.nan)
|
|
213
|
+
|
|
214
|
+
data = StableSpikeDataset(
|
|
215
|
+
original_sorting=init_sorting,
|
|
216
|
+
kept_indices=np.arange(N),
|
|
217
|
+
prgeom=prgeom,
|
|
218
|
+
tpca=_tpca,
|
|
219
|
+
extract_channels=channels,
|
|
220
|
+
core_channels=channels,
|
|
221
|
+
core_features=x,
|
|
222
|
+
train_extract_features=x[splits == 0],
|
|
223
|
+
split_names=["train", "val"],
|
|
224
|
+
split_mask=torch.from_numpy(splits),
|
|
225
|
+
device=device,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
data = data.to(device)
|
|
229
|
+
x = x.to(device)
|
|
230
|
+
noise = noise.to(device)
|
|
231
|
+
neighbs = neighbs.to(device)
|
|
232
|
+
noise_log_priors = noise.detection_prior_log_prob(mu)
|
|
233
|
+
noise_log_priors = noise_log_priors[labels]
|
|
234
|
+
|
|
235
|
+
return dict(
|
|
236
|
+
data=data,
|
|
237
|
+
init_sorting=init_sorting,
|
|
238
|
+
neighborhoods=neighbs,
|
|
239
|
+
channels=channels,
|
|
240
|
+
noise=noise,
|
|
241
|
+
M=M,
|
|
242
|
+
mu=mu,
|
|
243
|
+
W=W,
|
|
244
|
+
labels=labels,
|
|
245
|
+
K=K,
|
|
246
|
+
cov=cov,
|
|
247
|
+
x=x,
|
|
248
|
+
channel_observed_by_unit=clus_mask,
|
|
249
|
+
noise_log_priors=noise_log_priors,
|
|
250
|
+
n_channels=nc,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def compare_subspaces(mu, W, umu=None, uW=None, gmm=None, k=None):
|
|
256
|
+
if k is not None:
|
|
257
|
+
mu = mu[k]
|
|
258
|
+
W = W[k]
|
|
259
|
+
else:
|
|
260
|
+
if mu.ndim == 3:
|
|
261
|
+
assert mu.shape[0] == 1
|
|
262
|
+
mu = mu[0]
|
|
263
|
+
W = W[0]
|
|
264
|
+
if torch.is_tensor(mu):
|
|
265
|
+
mu = mu.numpy(force=True)
|
|
266
|
+
W = W.numpy(force=True)
|
|
267
|
+
|
|
268
|
+
if umu is None:
|
|
269
|
+
unit = {}
|
|
270
|
+
if k in gmm: # type: ignore
|
|
271
|
+
unit = gmm[k] # type: ignore
|
|
272
|
+
uW = getattr(unit, "W", np.zeros_like(W))
|
|
273
|
+
umu = getattr(unit, "mean", np.zeros_like(mu))
|
|
274
|
+
M = 0
|
|
275
|
+
werr = None
|
|
276
|
+
if torch.is_tensor(uW):
|
|
277
|
+
uW = uW.numpy(force=True)
|
|
278
|
+
assert uW.shape == W.shape, f"{uW.shape=} {W.shape=}"
|
|
279
|
+
rank, nc, M = W.shape
|
|
280
|
+
WTW = W.reshape(rank * nc, M)
|
|
281
|
+
WTW = WTW @ WTW.T
|
|
282
|
+
uWTW = uW.reshape(rank * nc, M)
|
|
283
|
+
uWTW = uWTW @ uWTW.T
|
|
284
|
+
werr = WTW - uWTW
|
|
285
|
+
if torch.is_tensor(umu):
|
|
286
|
+
umu = umu.numpy(force=True)
|
|
287
|
+
assert umu.shape == mu.shape, f"{umu.shape=} {mu.shape=}"
|
|
288
|
+
|
|
289
|
+
muerr = mu - umu
|
|
290
|
+
return muerr, werr
|
|
291
|
+
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def batched_levenberg_marquardt(
|
|
5
|
+
x0,
|
|
6
|
+
vgradfunc,
|
|
7
|
+
vhess,
|
|
8
|
+
extra_args=(),
|
|
9
|
+
lambd=100.0,
|
|
10
|
+
nu=10.0,
|
|
11
|
+
scale_problem="hessian",
|
|
12
|
+
max_steps=250,
|
|
13
|
+
convergence_g=1e-10,
|
|
14
|
+
convergence_err=1e-10,
|
|
15
|
+
min_scale=1e-2,
|
|
16
|
+
tikhonov=0.0,
|
|
17
|
+
):
|
|
18
|
+
"""A batched torch implementation of Levenberg-Marquardt
|
|
19
|
+
|
|
20
|
+
Solve a batch of minimization problems, starting from the batch
|
|
21
|
+
of initial conditions x0 with shape (n, p). Here, n is the number
|
|
22
|
+
of problems (batch size) and p is the problem size (number of features).
|
|
23
|
+
|
|
24
|
+
vgradfunc and vhess should be functions such that
|
|
25
|
+
- vgradfunc(x, *extra_args) returns a tuple of the (n, p) gradient at x and the
|
|
26
|
+
(n,) objective at x
|
|
27
|
+
- vhess(x, *extra_args) returns the (n, p, p) tensor of Hessians at x
|
|
28
|
+
You can easily produce these using torch's vmap, as done elsewhere
|
|
29
|
+
in dartsort. extra_args is a tuple of (n, ...) tensors which you can
|
|
30
|
+
use to maintain other per-problem variables that are needed to evaluate
|
|
31
|
+
your objective, gradient, or Hessian.
|
|
32
|
+
|
|
33
|
+
Arguments
|
|
34
|
+
---------
|
|
35
|
+
x : tensor, shape (n, p) as described above
|
|
36
|
+
The initial condition. This will not be written to.
|
|
37
|
+
vgradfunc, vhess : functions
|
|
38
|
+
Batched (grad, objective) and Hessian functions as described above
|
|
39
|
+
extra_args : tuple
|
|
40
|
+
Extra arguments passed to vgradfunc and vhess as described above
|
|
41
|
+
lambd : float, default 100.0
|
|
42
|
+
Initial Levenberg-Marquardt damping factor
|
|
43
|
+
nu : float, default 10.0
|
|
44
|
+
Damping update multiplier for bad subproblems, or inverse multiplier
|
|
45
|
+
for good problems (lambd <- lambd * nu or lambd / nu)
|
|
46
|
+
scale_problem : one of ("hessian", "none")
|
|
47
|
+
If scale_problem == "hessian", then the Hessian's diagonal is used to
|
|
48
|
+
inform the damping.
|
|
49
|
+
max_steps : int, default 250
|
|
50
|
+
Hard stopping condition
|
|
51
|
+
convergence_g, convergence_err : floats
|
|
52
|
+
Stopping conditions
|
|
53
|
+
min_scale : float, default 0.01
|
|
54
|
+
Stability term, this is the minimum amount of damping. So it sets the
|
|
55
|
+
minimum Tikhonov-style damping that is always used. Can be set to 0
|
|
56
|
+
for nice problems.
|
|
57
|
+
tikhonov : float, default 0.0
|
|
58
|
+
Extra damping
|
|
59
|
+
|
|
60
|
+
Returns
|
|
61
|
+
-------
|
|
62
|
+
x : (n, p) tensor
|
|
63
|
+
Optimization result
|
|
64
|
+
nsteps : (n,) LongTensor
|
|
65
|
+
Number of steps taken for each problem
|
|
66
|
+
"""
|
|
67
|
+
x = x0.clone()
|
|
68
|
+
n, p = x.shape
|
|
69
|
+
min_scale = torch.tensor(min_scale, dtype=x.dtype, device=x.device)
|
|
70
|
+
|
|
71
|
+
# initial objective
|
|
72
|
+
g, f = vgradfunc(x, *extra_args)
|
|
73
|
+
|
|
74
|
+
# state variables
|
|
75
|
+
# number of steps taken in each of the n problems
|
|
76
|
+
nsteps = torch.zeros(n, dtype=torch.int, device=x.device)
|
|
77
|
+
# current indices of active problems (all to start)
|
|
78
|
+
# we don't run all problems at every step, only those that haven't
|
|
79
|
+
# yet converged
|
|
80
|
+
active = torch.arange(n, device=x.device)
|
|
81
|
+
# the Levenberg-Marquardt damping factor, which changes dynamically
|
|
82
|
+
lambd = torch.full_like(f, lambd)
|
|
83
|
+
|
|
84
|
+
# these variable names
|
|
85
|
+
xa = x
|
|
86
|
+
eargsa = extra_args
|
|
87
|
+
|
|
88
|
+
for it in range(max_steps):
|
|
89
|
+
# get hessians for all active problems
|
|
90
|
+
H = vhess(xa, *eargsa)
|
|
91
|
+
|
|
92
|
+
# determine damping factor
|
|
93
|
+
if scale_problem == "hessian":
|
|
94
|
+
diag = H.diagonal(dim1=1, dim2=2).abs()
|
|
95
|
+
H.diagonal(dim1=1, dim2=2).add_(
|
|
96
|
+
lambd[:, None] * torch.maximum(diag, min_scale)
|
|
97
|
+
)
|
|
98
|
+
elif scale_problem == "none":
|
|
99
|
+
H.diagonal(dim1=1, dim2=2).add_(lambd[:, None])
|
|
100
|
+
else:
|
|
101
|
+
assert False
|
|
102
|
+
|
|
103
|
+
# extra damping
|
|
104
|
+
if tikhonov > 0:
|
|
105
|
+
H.diagonal(dim1=1, dim2=2).add_(tikhonov)
|
|
106
|
+
|
|
107
|
+
# find the search direction `d`
|
|
108
|
+
# use cholesky to find which problems that are pos def
|
|
109
|
+
# cholesky_solve to get search directions for those,
|
|
110
|
+
# and use gradient elsewhere
|
|
111
|
+
L, info = torch.linalg.cholesky_ex(H)
|
|
112
|
+
# initialize search direction with neg gradient
|
|
113
|
+
d = -g
|
|
114
|
+
ill_conditioned = info != 0
|
|
115
|
+
nice = info == 0
|
|
116
|
+
# cholesky solve nice problems
|
|
117
|
+
if nice.any():
|
|
118
|
+
d[nice] = torch.cholesky_solve(d[nice][..., None], L[nice])[..., 0]
|
|
119
|
+
# problems with issues need to be damped
|
|
120
|
+
if ill_conditioned.any():
|
|
121
|
+
d[ill_conditioned] /= lambd[ill_conditioned, None]
|
|
122
|
+
|
|
123
|
+
# check success, step, update lambd
|
|
124
|
+
gnew, fnew = vgradfunc(xa + d, *eargsa)
|
|
125
|
+
df = fnew - f
|
|
126
|
+
shrink = df <= 0
|
|
127
|
+
ill_conditioned = ill_conditioned | ~shrink
|
|
128
|
+
|
|
129
|
+
# accept shrink steps
|
|
130
|
+
if shrink.any():
|
|
131
|
+
x[active[shrink]] += d[shrink]
|
|
132
|
+
f[shrink] = fnew[shrink]
|
|
133
|
+
g[shrink] = gnew[shrink]
|
|
134
|
+
|
|
135
|
+
# check stopping conditions
|
|
136
|
+
converged_g = g.square().sum(dim=1) < convergence_g
|
|
137
|
+
converged_f = df.abs() < convergence_err
|
|
138
|
+
converged = shrink & converged_f & converged_g
|
|
139
|
+
|
|
140
|
+
# update damping factor based on ill conditioned-ness and
|
|
141
|
+
# whether the objective decreased
|
|
142
|
+
lambd = torch.where(ill_conditioned, lambd * nu, lambd / nu)
|
|
143
|
+
|
|
144
|
+
# update active set
|
|
145
|
+
remain = ~converged
|
|
146
|
+
active = active[remain]
|
|
147
|
+
if not active.numel():
|
|
148
|
+
break
|
|
149
|
+
xa = x[active]
|
|
150
|
+
eargsa = tuple(ea[remain] for ea in eargsa)
|
|
151
|
+
lambd = lambd[remain]
|
|
152
|
+
f = f[remain]
|
|
153
|
+
g = g[remain]
|
|
154
|
+
nsteps[active] += 1
|
|
155
|
+
|
|
156
|
+
return x, nsteps
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from torch import Tensor
|
|
2
|
+
from torch.nn import Module
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class BModule(Module):
|
|
6
|
+
"""This class only exists to silence some pyright messages
|
|
7
|
+
|
|
8
|
+
You can access buffers by `obj.b.buffername`. They will be known to be
|
|
9
|
+
Tensors (not Modules), and pyright will stop painting the town red.
|
|
10
|
+
|
|
11
|
+
Also, sometimes you want to have a tensor which does not get .to()d
|
|
12
|
+
around. It's too big. It can stay on cpu with register_cpu_buffer().
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, *args, **kwargs):
|
|
16
|
+
super().__init__(*args, **kwargs)
|
|
17
|
+
|
|
18
|
+
# bypassing torch Module's hooks
|
|
19
|
+
self.___cpu_buffers: dict[str, Tensor | None] = {}
|
|
20
|
+
self._init_bgetter()
|
|
21
|
+
|
|
22
|
+
def del_none_buffer(self, bufname: str):
|
|
23
|
+
assert bufname in self.___cpu_buffers
|
|
24
|
+
assert hasattr(self, bufname)
|
|
25
|
+
assert self.___cpu_buffers[bufname] is None
|
|
26
|
+
delattr(self, bufname)
|
|
27
|
+
del self.___cpu_buffers[bufname]
|
|
28
|
+
|
|
29
|
+
def _init_bgetter(self):
|
|
30
|
+
self.___bgetter = BufGetter(self._buffers, self.___cpu_buffers)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def b(self) -> "BufGetter":
|
|
34
|
+
return self.___bgetter
|
|
35
|
+
|
|
36
|
+
def register_cpu_buffer(self, name: str, buf: Tensor | None):
|
|
37
|
+
self.___cpu_buffers[name] = buf
|
|
38
|
+
|
|
39
|
+
def register_buffer_or_none(
|
|
40
|
+
self, name: str, buf: Tensor | None, on_device: bool = True
|
|
41
|
+
):
|
|
42
|
+
if buf is None:
|
|
43
|
+
setattr(self, name, None)
|
|
44
|
+
self.register_cpu_buffer(name, None)
|
|
45
|
+
elif on_device:
|
|
46
|
+
self.register_buffer(name, buf)
|
|
47
|
+
else:
|
|
48
|
+
self.register_cpu_buffer(name, buf)
|
|
49
|
+
|
|
50
|
+
def __getstate__(self):
|
|
51
|
+
try:
|
|
52
|
+
del self.___bgetter
|
|
53
|
+
except AttributeError:
|
|
54
|
+
pass
|
|
55
|
+
state = super().__getstate__()
|
|
56
|
+
return state
|
|
57
|
+
|
|
58
|
+
def __setstate__(self, state):
|
|
59
|
+
super().__setstate__(state)
|
|
60
|
+
self._init_bgetter()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class BufGetter:
|
|
64
|
+
__slots__ = "buffers", "cpu_buffers"
|
|
65
|
+
|
|
66
|
+
def __init__(self, buffers, cpu_buffers):
|
|
67
|
+
self.buffers = buffers
|
|
68
|
+
self.cpu_buffers = cpu_buffers
|
|
69
|
+
|
|
70
|
+
def __getattr__(self, key) -> Tensor:
|
|
71
|
+
if key in self.cpu_buffers:
|
|
72
|
+
return self.cpu_buffers[key]
|
|
73
|
+
elif key in self.buffers:
|
|
74
|
+
return self.buffers[key]
|
|
75
|
+
raise AttributeError(f"BufGetter didn't find {key=}.")
|