braincoder 0.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,533 @@
1
+ """Backend-agnostic utilities for Keras 3 multi-backend support.
2
+
3
+ Provides pure ``keras.ops`` implementations of operations that were
4
+ previously handled by TensorFlow-Probability, plus a thin abstraction
5
+ over gradient computation so the rest of the codebase is not coupled
6
+ to ``tf.GradientTape``.
7
+ """
8
+
9
+ import numpy as np
10
+ import contextlib
11
+
12
+ import keras
13
+ from keras import ops
14
+
15
+
16
+ # ---------------------------------------------------------------------------
17
+ # Robust Cholesky
18
+ # ---------------------------------------------------------------------------
19
+
20
+ def _torch_on_mps():
21
+ if keras.backend.backend() != 'torch':
22
+ return False
23
+ import torch
24
+ return torch.backends.mps.is_available() and ops.convert_to_tensor(0.).device.type == 'mps'
25
+
26
+
27
+ def float64_device():
28
+ """Scope for float64 linear algebra: Apple GPUs (MPS) have no float64, so use the CPU there."""
29
+ return keras.device('cpu') if _torch_on_mps() else contextlib.nullcontext()
30
+
31
+
32
+ def to_float64(x):
33
+ """Cast to float64 inside :func:`float64_device`, moving the tensor there first."""
34
+ return ops.cast(ops.convert_to_tensor(x), 'float64')
35
+
36
+
37
+ def safe_cholesky(M, jitter=1e-4, max_attempts=6):
38
+ """Cholesky factor of ``M`` with adaptive diagonal jitter + retries.
39
+
40
+ Fitted covariance matrices coming out of the residual fitter (and
41
+ similar optimisation loops) can slip below the PSD boundary by a
42
+ small amount due to numerical drift in the Adam updates (e.g. α
43
+ going slightly negative, β shrinking to ~0, or ρ saturating). A
44
+ plain ``ops.cholesky`` then returns NaN and crashes the caller.
45
+
46
+ Strategy:
47
+ 1. Symmetrise ``M`` (kills tiny asymmetry from accumulated rounding).
48
+ 2. Add ``(jitter * mean(diag(M)) + 1e-9) * I`` and try to factorise.
49
+ 3. On failure, multiply jitter by 10× and retry, up to
50
+ ``max_attempts`` times. The jitter scales with the matrix's own
51
+ diagonal so the same code path works for residual covariances
52
+ on the order of ~1 and on the order of ~1e4.
53
+
54
+ Most calls succeed on the first attempt with the default jitter
55
+ (~1e-4 relative). Subjects whose ResidualFitter pushed Ω deeper
56
+ below PSD need 1e-3 or 1e-2; the retries handle that without
57
+ inflating the default perturbation on healthy subjects.
58
+
59
+ Parameters
60
+ ----------
61
+ M : tensor or array, shape (n, n)
62
+ Square matrix that should be PSD but may be slightly off.
63
+ jitter : float, optional
64
+ Initial relative jitter (multiplier on ``mean(diag(M))``).
65
+ Default ``1e-4``.
66
+ max_attempts : int, optional
67
+ How many jitter levels to try before giving up. Each retry uses
68
+ 10× the previous jitter. Default 6 — covers
69
+ ``1e-4 → 10`` relative to mean(diag), which always succeeds
70
+ unless the input contains NaN/Inf.
71
+
72
+ Returns
73
+ -------
74
+ L : tensor, shape (n, n)
75
+ Lower-triangular Cholesky factor of the jittered matrix.
76
+
77
+ Raises
78
+ ------
79
+ RuntimeError
80
+ If all attempts fail. The input is then not rescuable by
81
+ diagonal jitter alone (NaN/Inf or many strongly negative
82
+ eigenvalues) and the caller should drop the fold / subject
83
+ rather than silently propagate NaN.
84
+
85
+ Notes
86
+ -----
87
+ The output preserves the caller's numpy dtype (float32 in →
88
+ float32 out, float64 in → float64 out). On backends that don't
89
+ natively support float64 (JAX without ``jax_enable_x64``), the
90
+ factorisation is computed in float32 internally and cast back at
91
+ the end — so the dtype contract holds even though the precision
92
+ of intermediate arithmetic is bounded by the backend.
93
+ """
94
+ # Remember caller's dtype so the contract "dtype in == dtype out"
95
+ # holds even on backends that silently downcast (e.g. JAX default
96
+ # mode coerces float64 → float32 inside ``ops.convert_to_tensor``).
97
+ in_dtype = getattr(M, 'dtype', None)
98
+
99
+ if str(in_dtype) == 'float64' and _torch_on_mps():
100
+ with float64_device():
101
+ return safe_cholesky(M, jitter=jitter, max_attempts=max_attempts)
102
+
103
+ M_t = ops.convert_to_tensor(M)
104
+ M_sym = 0.5 * (M_t + ops.transpose(M_t))
105
+ n = ops.shape(M_sym)[0]
106
+ diag_mean = ops.mean(ops.diag(M_sym))
107
+ eye = ops.eye(n, dtype=M_sym.dtype)
108
+ epsilon = ops.cast(1e-9, M_sym.dtype)
109
+
110
+ current = float(jitter)
111
+ last_exc = None
112
+ L = None
113
+ for _ in range(max_attempts):
114
+ scale = ops.cast(current, M_sym.dtype) * diag_mean + epsilon
115
+ try:
116
+ L_try = ops.cholesky(M_sym + scale * eye)
117
+ except Exception as exc:
118
+ last_exc = exc
119
+ current *= 10.0
120
+ continue
121
+ # JAX silently returns a NaN-filled matrix when the input is not
122
+ # PSD (the Keras wrapper raises in eager mode but only on concrete
123
+ # arrays; under tracing or with older Keras, NaN propagates).
124
+ # Treat a NaN result the same as a raised exception: bump jitter
125
+ # and retry. The bool() cast forces concrete evaluation; this
126
+ # function is therefore not safe to call inside `jax.jit` —
127
+ # callers that need a jittable version should pre-symmetrise and
128
+ # jitter the matrix themselves.
129
+ try:
130
+ has_nan = bool(ops.convert_to_numpy(ops.any(ops.isnan(L_try))))
131
+ except Exception:
132
+ # If we can't materialise the check (e.g. inside a trace),
133
+ # trust the cholesky result; the exception path above will
134
+ # still catch the eager-mode failures.
135
+ has_nan = False
136
+ if has_nan:
137
+ last_exc = RuntimeError(
138
+ f"cholesky returned NaN (input not PSD at jitter={current:.1e})")
139
+ current *= 10.0
140
+ continue
141
+ L = L_try
142
+ break
143
+ if L is None:
144
+ raise RuntimeError(
145
+ f"safe_cholesky: failed after {max_attempts} jitter levels "
146
+ f"(final jitter ~{current / 10:.1e}). Input is not rescuable "
147
+ f"by diagonal jitter — check for NaN/Inf or strongly negative "
148
+ f"eigenvalues. Last error: {last_exc}")
149
+
150
+ # Restore caller's dtype if it was a concrete numpy dtype (e.g.
151
+ # float64 in, but JAX default mode computed in float32). We only
152
+ # do this when the caller explicitly passed a numpy array; tensor
153
+ # inputs keep the backend's native dtype.
154
+ #
155
+ # Under JAX default mode, ``ops.cast`` to float64 silently keeps
156
+ # float32, so we round-trip through numpy to actually upcast. For
157
+ # tensor backends (torch / tf) this is a no-op since the dtype
158
+ # already matches.
159
+ if in_dtype is not None and isinstance(in_dtype, np.dtype):
160
+ if str(L.dtype) != str(np.dtype(in_dtype)):
161
+ L = np.asarray(ops.convert_to_numpy(L)).astype(in_dtype)
162
+ return L
163
+
164
+
165
+ def strict_cholesky(M):
166
+ """Cholesky factor of the symmetrised ``M``, with no jitter.
167
+
168
+ For matrices that are positive definite by construction. A numerical
169
+ failure is not papered over: it raises (TF, torch, eager JAX) or returns
170
+ NaN (JAX under tracing), which ``ResidualFitter.fit`` counts as a failed
171
+ step and rolls back, so what is reported in ``n_failed_steps`` is what
172
+ happened. Contrast :func:`safe_cholesky`, which always adds at least
173
+ ``1e-4 * mean(diag(M))`` to the diagonal, so the likelihood it gives is
174
+ that of a different matrix from the one returned to the caller.
175
+ """
176
+ M_t = ops.convert_to_tensor(M)
177
+ return ops.cholesky(0.5 * (M_t + ops.transpose(M_t)))
178
+
179
+
180
+ # ---------------------------------------------------------------------------
181
+ # Inverse transforms
182
+ # ---------------------------------------------------------------------------
183
+
184
+ def softplus_inverse(x):
185
+ """Numerically stable inverse of softplus: log(exp(x) - 1).
186
+
187
+ For large ``x`` (> 20), softplus(x) ≈ x so the inverse is also ≈ x.
188
+ """
189
+ return ops.where(x > 20.0, x, ops.log(ops.exp(x) - 1.0 + 1e-7))
190
+
191
+
192
+ # ---------------------------------------------------------------------------
193
+ # Multivariate Normal log-probability (replaces tfd.MultivariateNormalTriL)
194
+ # ---------------------------------------------------------------------------
195
+
196
+ def mvn_log_prob(x, L):
197
+ """Log-probability under MVN(0, L Lᵀ) for each row of ``x``.
198
+
199
+ Parameters
200
+ ----------
201
+ x : tensor, shape (n_timepoints, n_voxels)
202
+ Residuals (mean assumed zero).
203
+ L : tensor, shape (n_voxels, n_voxels)
204
+ Lower-triangular Cholesky factor of the covariance matrix.
205
+
206
+ Returns
207
+ -------
208
+ log_probs : tensor, shape (n_timepoints,)
209
+ """
210
+ k = ops.cast(ops.shape(x)[1], 'float32')
211
+ # Solve L y = xᵀ → y shape (n_voxels, n_timepoints)
212
+ y = ops.solve_triangular(L, ops.transpose(x), lower=True)
213
+ # log|Sigma| = 2 * sum(log(diag(L)))
214
+ log_det = 2.0 * ops.sum(ops.log(ops.diag(L)))
215
+ mahal = ops.sum(y ** 2, axis=0) # (n_timepoints,)
216
+ log_2pi = ops.cast(ops.log(ops.convert_to_tensor(2.0 * np.pi)), 'float32')
217
+ return -0.5 * (k * log_2pi + log_det + mahal)
218
+
219
+
220
+ # ---------------------------------------------------------------------------
221
+ # Multivariate Student-T log-probability
222
+ # (replaces tfd.MultivariateStudentTLinearOperator)
223
+ # ---------------------------------------------------------------------------
224
+
225
+ def _lgamma(x):
226
+ """Log-gamma (differentiable, backend-native).
227
+
228
+ Delegates to the public ``lgamma`` (defined below) so that the
229
+ autograd graph is preserved on every backend. The previous
230
+ implementation pulled scalars out via ``float(ops.convert_to_numpy(...))``
231
+ and called ``scipy.special.gammaln`` — that path:
232
+
233
+ * Crashes on JAX under tracing (``ConcretizationTypeError`` whenever
234
+ ``mvt_log_prob`` is called with ``method='t'`` and the dof is a
235
+ ``value_and_grad`` tracer).
236
+ * Silently breaks autograd on TF/torch, because scipy's gammaln has
237
+ no autograd connection: ``∂nll/∂dof`` is missing the gammaln
238
+ contribution and dof-fitting drifts to a biased optimum.
239
+
240
+ Routing through ``lgamma`` (which uses ``tf.math.lgamma`` /
241
+ ``jax.scipy.special.gammaln`` / ``torch.lgamma`` depending on the
242
+ backend) fixes both. Kept as ``_lgamma`` for backwards compatibility
243
+ of existing call sites.
244
+ """
245
+ return lgamma(ops.cast(x, 'float32'))
246
+
247
+
248
+ def mvt_log_prob(x, L, dof):
249
+ """Log-probability under multivariate Student-T(dof, 0, L Lᵀ) for each row.
250
+
251
+ Parameters
252
+ ----------
253
+ x : tensor, shape (n_timepoints, n_voxels)
254
+ L : tensor, shape (n_voxels, n_voxels) — Cholesky factor
255
+ dof : scalar tensor — degrees of freedom
256
+
257
+ Returns
258
+ -------
259
+ log_probs : tensor, shape (n_timepoints,)
260
+ """
261
+ k = ops.cast(ops.shape(x)[1], 'float32')
262
+ nu = ops.cast(dof, 'float32')
263
+
264
+ y = ops.solve_triangular(L, ops.transpose(x), lower=True) # (k, T)
265
+ mahal = ops.sum(y ** 2, axis=0) # (T,)
266
+ log_det = 2.0 * ops.sum(ops.log(ops.diag(L)))
267
+
268
+ lg_nu_k = _lgamma((nu + k) / 2.0)
269
+ lg_nu = _lgamma(nu / 2.0)
270
+ log_pi = ops.cast(ops.log(ops.convert_to_tensor(np.pi)), 'float32')
271
+
272
+ log_prob = (lg_nu_k - lg_nu
273
+ - 0.5 * k * ops.log(nu)
274
+ - 0.5 * k * log_pi
275
+ - 0.5 * log_det
276
+ - 0.5 * (nu + k) * ops.log(1.0 + mahal / nu))
277
+ return log_prob
278
+
279
+
280
+ # ---------------------------------------------------------------------------
281
+ # Sampling (replaces tfd sampling)
282
+ # ---------------------------------------------------------------------------
283
+
284
+ def sample_mvn(L, shape, seed=None):
285
+ """Draw samples from MVN(0, L Lᵀ).
286
+
287
+ Parameters
288
+ ----------
289
+ L : tensor, shape (n_voxels, n_voxels)
290
+ shape : tuple — leading batch dimensions, e.g. (n_batches, n_timepoints)
291
+ seed : optional int
292
+
293
+ Returns
294
+ -------
295
+ samples : tensor, shape (*shape, n_voxels)
296
+ """
297
+ n_voxels = ops.shape(L)[0]
298
+ flat_shape = (int(np.prod(shape)), n_voxels)
299
+ z = keras.random.normal(flat_shape, seed=seed) # (N, k)
300
+ samples = ops.matmul(z, ops.transpose(L)) # (N, k)
301
+ return ops.reshape(samples, (*shape, n_voxels))
302
+
303
+
304
+ def _split_seed(seed, n):
305
+ """Return ``n`` distinct seeds derived from a single integer ``seed``.
306
+
307
+ Background: ``keras.random.normal(..., seed=int)`` on the JAX backend
308
+ converts the integer into a ``jax.random.PRNGKey``, and that key is
309
+ *deterministic in the integer*. Two calls with the same ``seed=int``
310
+ therefore produce the **identical** sequence on JAX, while on
311
+ TF/torch a global counter advances between calls and the streams
312
+ differ. Calling ``sample_mvn(seed=42)`` followed by
313
+ ``keras.random.normal(seed=42)`` in the Student-T construction
314
+ therefore makes the numerator ``z`` and the χ²-scaler ``v``
315
+ perfectly correlated on JAX, so the resulting "Student-T sample"
316
+ collapses to ``sign(z) * sqrt(dof)`` — heavy-tailed structure gone.
317
+
318
+ Fix: derive sub-seeds (``seed+1, seed+2, …``) so each downstream
319
+ ``keras.random`` call has its own key. This is the simplest
320
+ backend-agnostic split that preserves backwards compatibility for
321
+ the ``seed=None`` case (fresh randomness from the backend's global
322
+ counter).
323
+ """
324
+ if seed is None:
325
+ return [None] * n
326
+ return [int(seed) + i for i in range(n)]
327
+
328
+
329
+ def sample_mvt(L, dof, shape, seed=None):
330
+ """Draw samples from multivariate Student-T(dof, 0, L Lᵀ).
331
+
332
+ Uses the representation: x = z / sqrt(v/dof) where z ~ MVN(0, LLᵀ)
333
+ and v ~ chi2(dof). The two random draws (z and v) MUST be
334
+ independent; see ``_split_seed`` for the JAX PRNG subtlety this
335
+ function had to defend against.
336
+ """
337
+ n_voxels = ops.shape(L)[0]
338
+ flat_n = int(np.prod(shape))
339
+ seed_z, seed_v = _split_seed(seed, 2)
340
+ z = sample_mvn(L, (flat_n,), seed=seed_z) # (N, k)
341
+ # chi2(dof) = Gamma(dof/2, 2), sample via normal: v = sum of dof normals^2
342
+ dof_int = max(1, int(round(float(ops.convert_to_numpy(ops.convert_to_tensor(dof))))))
343
+ normals = keras.random.normal((flat_n, dof_int), seed=seed_v)
344
+ v = ops.sum(normals ** 2, axis=1, keepdims=True) # (N, 1)
345
+ samples = z / ops.sqrt(v / ops.cast(dof, 'float32'))
346
+ return ops.reshape(samples, (*shape, n_voxels))
347
+
348
+
349
+ def sample_student_t(dof, scale, shape, seed=None):
350
+ """Draw i.i.d. samples from Student-T(dof, 0, scale).
351
+
352
+ Same independence requirement as ``sample_mvt`` — see ``_split_seed``.
353
+ """
354
+ n = int(np.prod(shape))
355
+ dof_int = max(1, int(round(float(ops.convert_to_numpy(ops.convert_to_tensor(dof))))))
356
+ seed_z, seed_v = _split_seed(seed, 2)
357
+ z = keras.random.normal((n,), seed=seed_z)
358
+ v = ops.sum(keras.random.normal((n, dof_int), seed=seed_v) ** 2, axis=1)
359
+ samples = z * scale / ops.sqrt(v / ops.cast(dof, 'float32'))
360
+ return ops.reshape(samples, shape)
361
+
362
+
363
+ # ---------------------------------------------------------------------------
364
+ # Backend-specific differentiable math ops
365
+ # ---------------------------------------------------------------------------
366
+
367
+ def lgamma(x):
368
+ """Backend-agnostic log-gamma function (differentiable)."""
369
+ backend = keras.backend.backend()
370
+ if backend == 'tensorflow':
371
+ import tensorflow as tf
372
+ return tf.math.lgamma(x)
373
+ elif backend == 'jax':
374
+ import jax.scipy.special as jss
375
+ return jss.gammaln(x)
376
+ elif backend == 'torch':
377
+ import torch
378
+ return torch.lgamma(ops.convert_to_tensor(x, dtype='float32'))
379
+ else:
380
+ from scipy.special import gammaln
381
+ return ops.convert_to_tensor(gammaln(np.array(x)), dtype='float32')
382
+
383
+
384
+ def bessel_i0(x):
385
+ """Backend-agnostic modified Bessel function I₀ (differentiable)."""
386
+ backend = keras.backend.backend()
387
+ if backend == 'tensorflow':
388
+ import tensorflow as tf
389
+ return tf.math.bessel_i0(x)
390
+ elif backend == 'jax':
391
+ import jax.scipy.special as jss
392
+ return jss.i0(x)
393
+ elif backend == 'torch':
394
+ import torch
395
+ return torch.special.i0(ops.convert_to_tensor(x, dtype='float32'))
396
+ else:
397
+ from scipy.special import i0
398
+ return ops.convert_to_tensor(i0(np.array(x)), dtype='float32')
399
+
400
+
401
+ def interp_regular_1d_grid(x, x_ref_min, x_ref_max, y_ref):
402
+ """Backend-agnostic linear interpolation on a regular 1-D grid.
403
+
404
+ Drop-in replacement for ``tfp.math.interp_regular_1d_grid`` with
405
+ ``fill_value='constant_extension'`` (clamp to boundary).
406
+
407
+ Parameters
408
+ ----------
409
+ x : tensor, shape (n_query,) or (n_query, 1)
410
+ x_ref_min : scalar — lower end of reference grid
411
+ x_ref_max : scalar — upper end of reference grid
412
+ y_ref : tensor, shape (n_grid, n_cols)
413
+
414
+ Returns
415
+ -------
416
+ tensor, shape (n_query, n_cols)
417
+ """
418
+ n = ops.cast(ops.shape(y_ref)[0], 'float32')
419
+
420
+ alpha = (ops.reshape(x, (-1,)) - x_ref_min) / (x_ref_max - x_ref_min) * (n - 1.0)
421
+ alpha = ops.clip(alpha, 0.0, n - 1.0) # constant extension
422
+
423
+ idx_lo = ops.cast(ops.floor(alpha), 'int32')
424
+ idx_hi = ops.minimum(idx_lo + 1, ops.cast(n, 'int32') - 1)
425
+ frac = alpha - ops.cast(idx_lo, 'float32') # (n_query,)
426
+
427
+ y_lo = ops.take(y_ref, idx_lo, axis=0) # (n_query, n_cols)
428
+ y_hi = ops.take(y_ref, idx_hi, axis=0)
429
+
430
+ return y_lo * (1.0 - frac[:, None]) + y_hi * frac[:, None]
431
+
432
+
433
+ # ---------------------------------------------------------------------------
434
+ # Jacobian computation
435
+ # ---------------------------------------------------------------------------
436
+
437
+ def compute_jacobian(fn, inputs):
438
+ """Compute the Jacobian of ``fn(inputs)`` w.r.t. ``inputs``.
439
+
440
+ Returns a tensor of shape ``(*output_shape, *input_shape)``.
441
+ """
442
+ backend = keras.backend.backend()
443
+ if backend == 'tensorflow':
444
+ import tensorflow as tf
445
+ inputs_var = tf.Variable(ops.convert_to_tensor(inputs))
446
+ # experimental_use_pfor=False avoids the default vectorized (pfor) path,
447
+ # whose UnsortedSegmentSum builds an int32 `num_segments` that overflows
448
+ # to negative on large Jacobians (e.g. n_voxels*n_values in Fisher-info),
449
+ # raising "num_segments must not be negative". The while_loop path is
450
+ # slower but returns an identical Jacobian. In eager mode pfor=False
451
+ # requires a persistent tape.
452
+ with tf.GradientTape(persistent=True) as tape:
453
+ outputs = fn(inputs_var)
454
+ return tape.jacobian(outputs, inputs_var, experimental_use_pfor=False)
455
+ elif backend == 'jax':
456
+ import jax
457
+ return jax.jacobian(fn)(inputs)
458
+ elif backend == 'torch':
459
+ import torch
460
+ return torch.autograd.functional.jacobian(fn, inputs)
461
+ else:
462
+ raise NotImplementedError(f"compute_jacobian not implemented for backend: {backend!r}")
463
+
464
+
465
+ # ---------------------------------------------------------------------------
466
+ # Gradient computation abstraction
467
+ # ---------------------------------------------------------------------------
468
+
469
+ def compute_gradients(loss_fn, variables):
470
+ """Compute gradients of ``loss_fn()`` w.r.t. ``variables``.
471
+
472
+ Returns
473
+ -------
474
+ loss : scalar tensor
475
+ grads : list of gradient tensors aligned with ``variables``
476
+ """
477
+ backend = keras.backend.backend()
478
+ if backend == 'tensorflow':
479
+ import tensorflow as tf
480
+ with tf.GradientTape() as tape:
481
+ loss = loss_fn()
482
+ grads = tape.gradient(loss, variables)
483
+ return loss, grads
484
+ elif backend == 'jax':
485
+ import jax
486
+ # Snapshot each Variable's current value as a concrete JAX array
487
+ # so the trace operates on pure inputs, not on the live Variable.
488
+ var_arrays = [ops.convert_to_tensor(v.value) for v in variables]
489
+
490
+ # CRITICAL: do NOT `var.assign(<tracer>)` inside the closure.
491
+ # `assign` writes the tracer into the Variable's backing array,
492
+ # and the reference escapes the trace boundary — at the end of
493
+ # ``jax.value_and_grad`` the Variable holds a ``LinearizeTracer``
494
+ # instead of a concrete ``ArrayImpl``. Subsequent reads then
495
+ # leak the entire forward graph one iteration at a time
496
+ # (analogous to the torch ``v.grad``-accumulation bug fixed in
497
+ # e5671a1). The user-visible symptom is monotonic host-memory
498
+ # growth across optimizer iterations and an eventual OOM on long
499
+ # runs.
500
+ #
501
+ # Fix: rebind the Variables *inside* a ``StatelessScope``, which
502
+ # is Keras 3's official mechanism for purifying Variable reads
503
+ # during a JAX trace. The scope binds (variable → traced array)
504
+ # only for the body of the closure; outside, the Variables keep
505
+ # their concrete values untouched. No tracer ever escapes.
506
+ from keras.src.backend.common.stateless_scope import StatelessScope
507
+
508
+ def _loss(*arrays):
509
+ mapping = list(zip(variables, arrays))
510
+ with StatelessScope(state_mapping=mapping):
511
+ return loss_fn()
512
+
513
+ argnums = tuple(range(len(variables)))
514
+ loss_val, grads = jax.value_and_grad(_loss, argnums=argnums)(*var_arrays)
515
+ return ops.convert_to_tensor(loss_val), [ops.convert_to_tensor(g) for g in grads]
516
+ elif backend == 'torch':
517
+ import torch
518
+ # Use the functional `torch.autograd.grad` rather than
519
+ # `.backward()` + read `.grad`. The latter accumulates into the
520
+ # variables' `.grad` attribute across optimizer iterations and
521
+ # returns live graph references — both leak the autograd graph
522
+ # one iteration at a time, OOM'ing an 80 GiB A100 by ~iter 7
523
+ # of the PRF gradient-descent loop. autograd.grad is the
524
+ # idiomatic one-shot API: no `.grad` mutation, returns
525
+ # standalone tensors that can be detached cleanly.
526
+ for v in variables:
527
+ v.value.requires_grad_(True)
528
+ loss = loss_fn()
529
+ inputs = [v.value for v in variables]
530
+ grads = torch.autograd.grad(loss, inputs)
531
+ return loss.detach(), [g.detach() for g in grads]
532
+ else:
533
+ raise NotImplementedError(f"compute_gradients not implemented for backend: {backend!r}")