DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,291 @@
1
+ """Utilities for making toy data and fitting PPCA or mixtures thereof to it"""
2
+
3
+ from typing import Literal
4
+
5
+ import numpy as np
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from sklearn.metrics import adjusted_rand_score
9
+
10
+
11
+ def simulate_moppca(
12
+ Nper=2**12,
13
+ rank=4,
14
+ nc=8,
15
+ M=2,
16
+ n_missing=2,
17
+ K=5,
18
+ t_mu: Literal["zero", "smooth", "random"] = "smooth",
19
+ t_cov: Literal["eye", "random", "eyesmall"] = "eye",
20
+ # zero, hot, random,
21
+ t_w: Literal["zero", "hot", "smooth", "random"] = "zero",
22
+ t_missing: Literal[
23
+ None, "random", "random_no_extrap", "skewed", "by_cluster"
24
+ ] = None,
25
+ init_label_corruption: float = 0.0,
26
+ snr: float = 10.0,
27
+ rg: int | np.random.Generator = 0,
28
+ device=None,
29
+ ):
30
+ from dartsort.transform import TemporalPCAFeaturizer
31
+ from dartsort.util.data_util import DARTsortSorting
32
+ from dartsort.util.noise_util import EmbeddedNoise
33
+ from dartsort.clustering.gmm.stable_features import (
34
+ StableSpikeDataset,
35
+ SpikeNeighborhoods,
36
+ )
37
+
38
+ N = Nper * K
39
+
40
+ rg = np.random.default_rng(rg)
41
+ D = rank * nc
42
+
43
+ if device is None:
44
+ device = "cuda" if torch.cuda.is_available() else "cpu"
45
+ device = torch.device(device)
46
+
47
+ clus_neighbs = None
48
+ clus_mask = None
49
+ if t_missing == "by_cluster":
50
+ # in this case, missing chans per clus are never observed,
51
+ # so we should treat them as 0 in W and mu to avoid issues with metrics
52
+ clus_neighbs = [
53
+ rg.choice(nc, size=nc - n_missing, replace=False) for _ in range(K)
54
+ ]
55
+ clus_neighbs = np.array(clus_neighbs)
56
+ clus_neighbs = np.sort(clus_neighbs, axis=1)
57
+ clus_mask = np.zeros((K, nc))
58
+ for j, n in enumerate(clus_neighbs):
59
+ clus_mask[j, n] = 1.0
60
+ if t_missing in (None, "none"):
61
+ n_missing = 0
62
+
63
+ if t_mu == "zero":
64
+ mu = np.zeros((K, rank, nc))
65
+ elif t_mu == "random":
66
+ mu = snr * rg.normal(size=(K, rank, nc))
67
+ elif t_mu == "smooth":
68
+ phase = rg.uniform(0.0, 2 * np.pi, size=(K, rank, 1))
69
+ freq = rg.uniform(0.2, 1.0, size=(K, rank, 1))
70
+ amp = rg.uniform(1.0, 2.0, size=(K, rank, 1))
71
+ domain = np.linspace(0, 2 * np.pi, endpoint=False, num=nc)
72
+ mu = snr * amp * np.sin(phase + freq * domain)
73
+ else:
74
+ assert False
75
+ if clus_mask is not None:
76
+ mu = mu * clus_mask[:, None]
77
+
78
+ if t_cov == "eye":
79
+ cov = np.eye(D)
80
+ noise = EmbeddedNoise(
81
+ rank,
82
+ nc,
83
+ cov_kind="scalar",
84
+ global_std=torch.tensor(1.0, dtype=torch.float, device=device),
85
+ )
86
+ elif t_cov == "eyesmall":
87
+ cov = 1e-4 * np.eye(D)
88
+ noise = EmbeddedNoise(
89
+ rank,
90
+ nc,
91
+ cov_kind="scalar",
92
+ global_std=torch.tensor(0.01, dtype=torch.float, device=device),
93
+ )
94
+ elif t_cov == "random":
95
+ _c = rg.normal(size=(D, 10 * D))
96
+ cov = _c @ _c.T / (10 * D)
97
+ full_cov = torch.asarray(cov, dtype=torch.float, device=device).view(
98
+ rank, nc, rank, nc
99
+ )
100
+ noise = EmbeddedNoise(rank, nc, cov_kind="full", full_cov=full_cov)
101
+ else:
102
+ assert False
103
+
104
+ if t_w == "hot":
105
+ W = np.zeros((K, rank, nc, M))
106
+ for j in range(K):
107
+ for m in range(M):
108
+ W[j, m, j, m] = 10 - m
109
+ elif t_w == "random":
110
+ W = rg.normal(size=(K, rank, nc, M))
111
+ elif t_w == "smooth":
112
+ phase = rg.uniform(0.0, 2 * np.pi, size=(K, rank, 1, M))
113
+ freq = rg.uniform(0.2, 1.0, size=(K, rank, 1, M))
114
+ amp = rg.uniform(1.0, 2.0, size=(K, rank, 1, M))
115
+ domain = np.linspace(0, 2 * np.pi, endpoint=False, num=nc)
116
+ W = amp * np.exp(-(np.sin(phase + freq * domain[:, None])**2))
117
+ elif t_w == "zero":
118
+ W = np.zeros((K, rank, nc, M))
119
+ else:
120
+ assert False
121
+ if clus_mask is not None:
122
+ W = W * clus_mask[:, None, :, None]
123
+
124
+ labels = rg.integers(K, size=N)
125
+ u = rg.normal(size=(N, M))
126
+ eps = rg.multivariate_normal(mean=np.zeros(D), cov=cov, size=N)
127
+
128
+ labels = torch.asarray(labels)
129
+ cov = noise.marginal_covariance()
130
+ u = torch.asarray(u, dtype=torch.float)
131
+ eps = torch.asarray(eps, dtype=torch.float)
132
+ mu = torch.asarray(mu, dtype=torch.float)
133
+ W = torch.asarray(W, dtype=torch.float)
134
+
135
+ y = mu[labels] + torch.einsum("nrcm,nm->nrc", W[labels], u) + eps.view(N, rank, nc)
136
+
137
+ x = None
138
+ if not t_missing:
139
+ x = y
140
+ channels = torch.arange(nc).unsqueeze(0).broadcast_to(N, nc)
141
+ elif t_missing.startswith("random"):
142
+ assert n_missing == 2
143
+ no_extrap = int(t_missing == "random_no_extrap")
144
+ possible_neighbs = np.zeros(nc - n_missing, dtype=np.int64)[None]
145
+ npair = ((nc - 2 * no_extrap) * (nc - 1 - 2 * no_extrap)) // 2
146
+ possible_neighbs = np.broadcast_to(possible_neighbs, (npair, nc - n_missing))
147
+ possible_neighbs = np.ascontiguousarray(possible_neighbs)
148
+ mask = np.ones(nc, dtype=bool)
149
+ ct = 0
150
+ for i in range(no_extrap, nc - no_extrap):
151
+ mask[i] = False
152
+ for j in range(i + 1, nc - no_extrap):
153
+ mask[j] = False
154
+ possible_neighbs[ct] = np.flatnonzero(mask)
155
+ mask[j] = True
156
+ ct += 1
157
+ mask[i] = True
158
+ assert ct == npair
159
+ channels = possible_neighbs[rg.integers(npair, size=N)]
160
+ elif t_missing == "skewed":
161
+ nc_miss = min(2, nc - 2)
162
+ assert nc_miss > 0
163
+ n0 = np.arange(nc - nc_miss)
164
+ n1 = nc_miss + n0
165
+ neighbs = np.stack((n0, n1), axis=0)
166
+ ns_missing = int(N / 3)
167
+ choices = np.concatenate(
168
+ (
169
+ np.zeros(ns_missing, dtype=np.int64),
170
+ np.ones(N - ns_missing, dtype=np.int64),
171
+ )
172
+ )
173
+ rg.shuffle(choices)
174
+ channels = neighbs[choices]
175
+ elif t_missing == "by_cluster":
176
+ assert clus_neighbs is not None
177
+ channels = clus_neighbs[labels]
178
+ else:
179
+ assert False
180
+ channels = torch.asarray(channels, dtype=torch.long)
181
+
182
+ if t_missing:
183
+ x = torch.take_along_dim(y, channels.unsqueeze(1), dim=2)
184
+ assert x is not None
185
+
186
+ neighbs = SpikeNeighborhoods.from_channels(channels, nc, device=device)
187
+
188
+ init_labels = labels.clone()
189
+ if init_label_corruption:
190
+ to_corrupt = rg.binomial(1, init_label_corruption, size=N).astype(bool)
191
+ init_labels[to_corrupt] = torch.from_numpy(
192
+ rg.integers(K, size=to_corrupt.sum())
193
+ )
194
+
195
+ init_sorting = DARTsortSorting(
196
+ times_samples=np.arange(N),
197
+ channels=np.zeros(N, dtype=np.int64),
198
+ labels=init_labels.numpy(force=True),
199
+ sampling_frequency=100.0,
200
+ ephemeral_features=dict(times_seconds=np.arange(N) / 100.0),
201
+ )
202
+
203
+ _tpca = TemporalPCAFeaturizer(
204
+ channel_index=torch.zeros(nc, nc - n_missing, dtype=torch.long),
205
+ rank=rank,
206
+ )
207
+ _tpca = _tpca.to(device)
208
+ splits = rg.binomial(1, p=0.3, size=N)
209
+
210
+ prgeom = 15.0 * torch.arange(nc, dtype=torch.float)
211
+ prgeom = torch.stack((torch.zeros(nc), prgeom), dim=1)
212
+ prgeom = F.pad(prgeom, (0, 0, 0, 1), value=torch.nan)
213
+
214
+ data = StableSpikeDataset(
215
+ original_sorting=init_sorting,
216
+ kept_indices=np.arange(N),
217
+ prgeom=prgeom,
218
+ tpca=_tpca,
219
+ extract_channels=channels,
220
+ core_channels=channels,
221
+ core_features=x,
222
+ train_extract_features=x[splits == 0],
223
+ split_names=["train", "val"],
224
+ split_mask=torch.from_numpy(splits),
225
+ device=device,
226
+ )
227
+
228
+ data = data.to(device)
229
+ x = x.to(device)
230
+ noise = noise.to(device)
231
+ neighbs = neighbs.to(device)
232
+ noise_log_priors = noise.detection_prior_log_prob(mu)
233
+ noise_log_priors = noise_log_priors[labels]
234
+
235
+ return dict(
236
+ data=data,
237
+ init_sorting=init_sorting,
238
+ neighborhoods=neighbs,
239
+ channels=channels,
240
+ noise=noise,
241
+ M=M,
242
+ mu=mu,
243
+ W=W,
244
+ labels=labels,
245
+ K=K,
246
+ cov=cov,
247
+ x=x,
248
+ channel_observed_by_unit=clus_mask,
249
+ noise_log_priors=noise_log_priors,
250
+ n_channels=nc,
251
+ )
252
+
253
+
254
+
255
+ def compare_subspaces(mu, W, umu=None, uW=None, gmm=None, k=None):
256
+ if k is not None:
257
+ mu = mu[k]
258
+ W = W[k]
259
+ else:
260
+ if mu.ndim == 3:
261
+ assert mu.shape[0] == 1
262
+ mu = mu[0]
263
+ W = W[0]
264
+ if torch.is_tensor(mu):
265
+ mu = mu.numpy(force=True)
266
+ W = W.numpy(force=True)
267
+
268
+ if umu is None:
269
+ unit = {}
270
+ if k in gmm: # type: ignore
271
+ unit = gmm[k] # type: ignore
272
+ uW = getattr(unit, "W", np.zeros_like(W))
273
+ umu = getattr(unit, "mean", np.zeros_like(mu))
274
+ M = 0
275
+ werr = None
276
+ if torch.is_tensor(uW):
277
+ uW = uW.numpy(force=True)
278
+ assert uW.shape == W.shape, f"{uW.shape=} {W.shape=}"
279
+ rank, nc, M = W.shape
280
+ WTW = W.reshape(rank * nc, M)
281
+ WTW = WTW @ WTW.T
282
+ uWTW = uW.reshape(rank * nc, M)
283
+ uWTW = uWTW @ uWTW.T
284
+ werr = WTW - uWTW
285
+ if torch.is_tensor(umu):
286
+ umu = umu.numpy(force=True)
287
+ assert umu.shape == mu.shape, f"{umu.shape=} {mu.shape=}"
288
+
289
+ muerr = mu - umu
290
+ return muerr, werr
291
+
@@ -0,0 +1,156 @@
1
+ import torch
2
+
3
+
4
+ def batched_levenberg_marquardt(
5
+ x0,
6
+ vgradfunc,
7
+ vhess,
8
+ extra_args=(),
9
+ lambd=100.0,
10
+ nu=10.0,
11
+ scale_problem="hessian",
12
+ max_steps=250,
13
+ convergence_g=1e-10,
14
+ convergence_err=1e-10,
15
+ min_scale=1e-2,
16
+ tikhonov=0.0,
17
+ ):
18
+ """A batched torch implementation of Levenberg-Marquardt
19
+
20
+ Solve a batch of minimization problems, starting from the batch
21
+ of initial conditions x0 with shape (n, p). Here, n is the number
22
+ of problems (batch size) and p is the problem size (number of features).
23
+
24
+ vgradfunc and vhess should be functions such that
25
+ - vgradfunc(x, *extra_args) returns a tuple of the (n, p) gradient at x and the
26
+ (n,) objective at x
27
+ - vhess(x, *extra_args) returns the (n, p, p) tensor of Hessians at x
28
+ You can easily produce these using torch's vmap, as done elsewhere
29
+ in dartsort. extra_args is a tuple of (n, ...) tensors which you can
30
+ use to maintain other per-problem variables that are needed to evaluate
31
+ your objective, gradient, or Hessian.
32
+
33
+ Arguments
34
+ ---------
35
+ x : tensor, shape (n, p) as described above
36
+ The initial condition. This will not be written to.
37
+ vgradfunc, vhess : functions
38
+ Batched (grad, objective) and Hessian functions as described above
39
+ extra_args : tuple
40
+ Extra arguments passed to vgradfunc and vhess as described above
41
+ lambd : float, default 100.0
42
+ Initial Levenberg-Marquardt damping factor
43
+ nu : float, default 10.0
44
+ Damping update multiplier for bad subproblems, or inverse multiplier
45
+ for good problems (lambd <- lambd * nu or lambd / nu)
46
+ scale_problem : one of ("hessian", "none")
47
+ If scale_problem == "hessian", then the Hessian's diagonal is used to
48
+ inform the damping.
49
+ max_steps : int, default 250
50
+ Hard stopping condition
51
+ convergence_g, convergence_err : floats
52
+ Stopping conditions
53
+ min_scale : float, default 0.01
54
+ Stability term, this is the minimum amount of damping. So it sets the
55
+ minimum Tikhonov-style damping that is always used. Can be set to 0
56
+ for nice problems.
57
+ tikhonov : float, default 0.0
58
+ Extra damping
59
+
60
+ Returns
61
+ -------
62
+ x : (n, p) tensor
63
+ Optimization result
64
+ nsteps : (n,) LongTensor
65
+ Number of steps taken for each problem
66
+ """
67
+ x = x0.clone()
68
+ n, p = x.shape
69
+ min_scale = torch.tensor(min_scale, dtype=x.dtype, device=x.device)
70
+
71
+ # initial objective
72
+ g, f = vgradfunc(x, *extra_args)
73
+
74
+ # state variables
75
+ # number of steps taken in each of the n problems
76
+ nsteps = torch.zeros(n, dtype=torch.int, device=x.device)
77
+ # current indices of active problems (all to start)
78
+ # we don't run all problems at every step, only those that haven't
79
+ # yet converged
80
+ active = torch.arange(n, device=x.device)
81
+ # the Levenberg-Marquardt damping factor, which changes dynamically
82
+ lambd = torch.full_like(f, lambd)
83
+
84
+ # these variable names
85
+ xa = x
86
+ eargsa = extra_args
87
+
88
+ for it in range(max_steps):
89
+ # get hessians for all active problems
90
+ H = vhess(xa, *eargsa)
91
+
92
+ # determine damping factor
93
+ if scale_problem == "hessian":
94
+ diag = H.diagonal(dim1=1, dim2=2).abs()
95
+ H.diagonal(dim1=1, dim2=2).add_(
96
+ lambd[:, None] * torch.maximum(diag, min_scale)
97
+ )
98
+ elif scale_problem == "none":
99
+ H.diagonal(dim1=1, dim2=2).add_(lambd[:, None])
100
+ else:
101
+ assert False
102
+
103
+ # extra damping
104
+ if tikhonov > 0:
105
+ H.diagonal(dim1=1, dim2=2).add_(tikhonov)
106
+
107
+ # find the search direction `d`
108
+ # use cholesky to find which problems that are pos def
109
+ # cholesky_solve to get search directions for those,
110
+ # and use gradient elsewhere
111
+ L, info = torch.linalg.cholesky_ex(H)
112
+ # initialize search direction with neg gradient
113
+ d = -g
114
+ ill_conditioned = info != 0
115
+ nice = info == 0
116
+ # cholesky solve nice problems
117
+ if nice.any():
118
+ d[nice] = torch.cholesky_solve(d[nice][..., None], L[nice])[..., 0]
119
+ # problems with issues need to be damped
120
+ if ill_conditioned.any():
121
+ d[ill_conditioned] /= lambd[ill_conditioned, None]
122
+
123
+ # check success, step, update lambd
124
+ gnew, fnew = vgradfunc(xa + d, *eargsa)
125
+ df = fnew - f
126
+ shrink = df <= 0
127
+ ill_conditioned = ill_conditioned | ~shrink
128
+
129
+ # accept shrink steps
130
+ if shrink.any():
131
+ x[active[shrink]] += d[shrink]
132
+ f[shrink] = fnew[shrink]
133
+ g[shrink] = gnew[shrink]
134
+
135
+ # check stopping conditions
136
+ converged_g = g.square().sum(dim=1) < convergence_g
137
+ converged_f = df.abs() < convergence_err
138
+ converged = shrink & converged_f & converged_g
139
+
140
+ # update damping factor based on ill conditioned-ness and
141
+ # whether the objective decreased
142
+ lambd = torch.where(ill_conditioned, lambd * nu, lambd / nu)
143
+
144
+ # update active set
145
+ remain = ~converged
146
+ active = active[remain]
147
+ if not active.numel():
148
+ break
149
+ xa = x[active]
150
+ eargsa = tuple(ea[remain] for ea in eargsa)
151
+ lambd = lambd[remain]
152
+ f = f[remain]
153
+ g = g[remain]
154
+ nsteps[active] += 1
155
+
156
+ return x, nsteps
@@ -0,0 +1,75 @@
1
+ from torch import Tensor
2
+ from torch.nn import Module
3
+
4
+
5
+ class BModule(Module):
6
+ """This class only exists to silence some pyright messages
7
+
8
+ You can access buffers by `obj.b.buffername`. They will be known to be
9
+ Tensors (not Modules), and pyright will stop painting the town red.
10
+
11
+ Also, sometimes you want to have a tensor which does not get .to()d
12
+ around. It's too big. It can stay on cpu with register_cpu_buffer().
13
+ """
14
+
15
+ def __init__(self, *args, **kwargs):
16
+ super().__init__(*args, **kwargs)
17
+
18
+ # bypassing torch Module's hooks
19
+ self.___cpu_buffers: dict[str, Tensor | None] = {}
20
+ self._init_bgetter()
21
+
22
+ def del_none_buffer(self, bufname: str):
23
+ assert bufname in self.___cpu_buffers
24
+ assert hasattr(self, bufname)
25
+ assert self.___cpu_buffers[bufname] is None
26
+ delattr(self, bufname)
27
+ del self.___cpu_buffers[bufname]
28
+
29
+ def _init_bgetter(self):
30
+ self.___bgetter = BufGetter(self._buffers, self.___cpu_buffers)
31
+
32
+ @property
33
+ def b(self) -> "BufGetter":
34
+ return self.___bgetter
35
+
36
+ def register_cpu_buffer(self, name: str, buf: Tensor | None):
37
+ self.___cpu_buffers[name] = buf
38
+
39
+ def register_buffer_or_none(
40
+ self, name: str, buf: Tensor | None, on_device: bool = True
41
+ ):
42
+ if buf is None:
43
+ setattr(self, name, None)
44
+ self.register_cpu_buffer(name, None)
45
+ elif on_device:
46
+ self.register_buffer(name, buf)
47
+ else:
48
+ self.register_cpu_buffer(name, buf)
49
+
50
+ def __getstate__(self):
51
+ try:
52
+ del self.___bgetter
53
+ except AttributeError:
54
+ pass
55
+ state = super().__getstate__()
56
+ return state
57
+
58
+ def __setstate__(self, state):
59
+ super().__setstate__(state)
60
+ self._init_bgetter()
61
+
62
+
63
+ class BufGetter:
64
+ __slots__ = "buffers", "cpu_buffers"
65
+
66
+ def __init__(self, buffers, cpu_buffers):
67
+ self.buffers = buffers
68
+ self.cpu_buffers = cpu_buffers
69
+
70
+ def __getattr__(self, key) -> Tensor:
71
+ if key in self.cpu_buffers:
72
+ return self.cpu_buffers[key]
73
+ elif key in self.buffers:
74
+ return self.buffers[key]
75
+ raise AttributeError(f"BufGetter didn't find {key=}.")