torchzero 0.3.10__py3-none-any.whl → 0.3.13__py3-none-any.whl

torchzero/modules/conjugate_gradient/cg.py

@@ -0,0 +1,355 @@
+from abc import ABC, abstractmethod
+from typing import Literal
+
+import torch
+
+from ...core import (
+    Chainable,
+    Modular,
+    Module,
+    Transform,
+    Var,
+    apply_transform,
+)
+from ...utils import TensorList, as_tensorlist, unpack_dicts, unpack_states
+from ..line_search import LineSearchBase
+from ..quasi_newton.quasi_newton import HessianUpdateStrategy
+from ..functional import safe_clip
+
+
+class ConguateGradientBase(Transform, ABC):
+    """Base class for conjugate gradient methods. The only difference between them is how beta is calculated.
+
+    This is an abstract class, to use it, subclass it and override `get_beta`.
+
+
+    Args:
+        defaults (dict | None, optional): dictionary of settings defaults. Defaults to None.
+        clip_beta (bool, optional): whether to clip beta to be no less than 0. Defaults to False.
+        restart_interval (int | None | Literal["auto"], optional):
+            interval between resetting the search direction.
+            "auto" means number of dimensions + 1, None means no reset. Defaults to None.
+        inner (Chainable | None, optional): previous direction is added to the output of this module. Defaults to None.
+
+    Example:
+
+    ```python
+    class PolakRibiere(ConguateGradientBase):
+        def __init__(
+            self,
+            clip_beta=True,
+            restart_interval: int | None = None,
+            inner: Chainable | None = None
+        ):
+            super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+        def get_beta(self, p, g, prev_g, prev_d):
+            denom = prev_g.dot(prev_g)
+            if denom.abs() <= torch.finfo(g[0].dtype).eps: return 0
+            return g.dot(g - prev_g) / denom
+    ```
+
+    """
+    def __init__(self, defaults = None, clip_beta: bool = False, restart_interval: int | None | Literal['auto'] = None, inner: Chainable | None = None):
+        if defaults is None: defaults = {}
+        defaults['restart_interval'] = restart_interval
+        defaults['clip_beta'] = clip_beta
+        super().__init__(defaults, uses_grad=False)
+
+        if inner is not None:
+            self.set_child('inner', inner)
+
+
+    def reset_for_online(self):
+        super().reset_for_online()
+        self.clear_state_keys('prev_grad')
+        self.global_state.pop('stage', None)
+        self.global_state['step'] = self.global_state.get('step', 1) - 1
+
+    def initialize(self, p: TensorList, g: TensorList):
+        """runs on first step when prev_grads and prev_dir are not available"""
+
+    @abstractmethod
+    def get_beta(self, p: TensorList, g: TensorList, prev_g: TensorList, prev_d: TensorList) -> float | torch.Tensor:
+        """returns beta"""
+
+    @torch.no_grad
+    def update_tensors(self, tensors, params, grads, loss, states, settings):
+        tensors = as_tensorlist(tensors)
+        params = as_tensorlist(params)
+
+        step = self.global_state.get('step', 0) + 1
+        self.global_state['step'] = step
+
+        # initialize on first step
+        if self.global_state.get('stage', 0) == 0:
+            g_prev, d_prev = unpack_states(states, tensors, 'g_prev', 'd_prev', cls=TensorList)
+            d_prev.copy_(tensors)
+            g_prev.copy_(tensors)
+            self.initialize(params, tensors)
+            self.global_state['stage'] = 1
+
+        else:
+            # if `update_tensors` was called multiple times before `apply_tensors`,
+            # stage becomes 2
+            self.global_state['stage'] = 2
+
+    @torch.no_grad
+    def apply_tensors(self, tensors, params, grads, loss, states, settings):
+        tensors = as_tensorlist(tensors)
+        step = self.global_state['step']
+
+        if 'inner' in self.children:
+            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads))
+
+        assert self.global_state['stage'] != 0
+        if self.global_state['stage'] == 1:
+            self.global_state['stage'] = 2
+            return tensors
+
+        params = as_tensorlist(params)
+        g_prev, d_prev = unpack_states(states, tensors, 'g_prev', 'd_prev', cls=TensorList)
+
+        # get beta
+        beta = self.get_beta(params, tensors, g_prev, d_prev)
+        if settings[0]['clip_beta']: beta = max(0, beta) # pyright:ignore[reportArgumentType]
+
+        # inner step
+        # calculate new direction with beta
+        dir = tensors.add_(d_prev.mul_(beta))
+        d_prev.copy_(dir)
+
+        # resetting
+        restart_interval = settings[0]['restart_interval']
+        if restart_interval == 'auto': restart_interval = tensors.global_numel() + 1
+        if restart_interval is not None and step % restart_interval == 0:
+            self.state.clear()
+            self.global_state.clear()
+
+        return dir
+
+# ------------------------------- Polak-Ribière ------------------------------ #
+def polak_ribiere_beta(g: TensorList, prev_g: TensorList):
+    denom = prev_g.dot(prev_g)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+    return g.dot(g - prev_g) / denom
+
+class PolakRibiere(ConguateGradientBase):
+    """Polak-Ribière-Polyak nonlinear conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, clip_beta=True, restart_interval: int | None = None, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return polak_ribiere_beta(g, prev_g)
+
+# ------------------------------ Fletcher–Reeves ----------------------------- #
+def fletcher_reeves_beta(gg: torch.Tensor, prev_gg: torch.Tensor):
+    if prev_gg.abs() <= torch.finfo(gg.dtype).tiny * 2: return 0
+    return gg / prev_gg
+
+class FletcherReeves(ConguateGradientBase):
+    """Fletcher–Reeves nonlinear conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = 'auto', clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def initialize(self, p, g):
+        self.global_state['prev_gg'] = g.dot(g)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        gg = g.dot(g)
+        beta = fletcher_reeves_beta(gg, self.global_state['prev_gg'])
+        self.global_state['prev_gg'] = gg
+        return beta
+
+# ----------------------------- Hestenes–Stiefel ----------------------------- #
+def hestenes_stiefel_beta(g: TensorList, prev_d: TensorList,prev_g: TensorList):
+    grad_diff = g - prev_g
+    denom = prev_d.dot(grad_diff)
+    if denom.abs() < torch.finfo(g[0].dtype).tiny * 2: return 0
+    return (g.dot(grad_diff) / denom).neg()
+
+
+class HestenesStiefel(ConguateGradientBase):
+    """Hestenes–Stiefel nonlinear conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return hestenes_stiefel_beta(g, prev_d, prev_g)
+
+
+# --------------------------------- Dai–Yuan --------------------------------- #
+def dai_yuan_beta(g: TensorList, prev_d: TensorList,prev_g: TensorList):
+    denom = prev_d.dot(g - prev_g)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+    return (g.dot(g) / denom).neg()
+
+class DaiYuan(ConguateGradientBase):
+    """Dai–Yuan nonlinear conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1)`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return dai_yuan_beta(g, prev_d, prev_g)
+
+
+# -------------------------------- Liu-Storey -------------------------------- #
+def liu_storey_beta(g:TensorList, prev_d:TensorList, prev_g:TensorList, ):
+    denom = prev_g.dot(prev_d)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+    return g.dot(g - prev_g) / denom
+
+class LiuStorey(ConguateGradientBase):
+    """Liu-Storey nonlinear conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return liu_storey_beta(g, prev_d, prev_g)
+
+# ----------------------------- Conjugate Descent ---------------------------- #
+def conjugate_descent_beta(g:TensorList, prev_d:TensorList, prev_g:TensorList):
+    denom = prev_g.dot(prev_d)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+    return g.dot(g) / denom
+
+class ConjugateDescent(ConguateGradientBase):
+    """Conjugate Descent (CD).
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return conjugate_descent_beta(g, prev_d, prev_g)
+
+
+# -------------------------------- Hager-Zhang ------------------------------- #
+def hager_zhang_beta(g:TensorList, prev_d:TensorList, prev_g:TensorList,):
+    g_diff = g - prev_g
+    denom = prev_d.dot(g_diff)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+
+    term1 = 1/denom
+    # term2
+    term2 = (g_diff - (2 * prev_d * (g_diff.pow(2).global_sum()/denom))).dot(g)
+    return (term1 * term2).neg()
+
+
+class HagerZhang(ConguateGradientBase):
+    """Hager-Zhang nonlinear conjugate gradient method,
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return hager_zhang_beta(g, prev_d, prev_g)
+
+
+# ----------------------------------- DYHS ---------------------------------- #
+def dyhs_beta(g: TensorList, prev_d: TensorList,prev_g: TensorList):
+    grad_diff = g - prev_g
+    denom = prev_d.dot(grad_diff)
+    if denom.abs() <= torch.finfo(g[0].dtype).tiny * 2: return 0
+
+    # Dai-Yuan
+    dy_beta = (g.dot(g) / denom).neg().clamp(min=0)
+
+    # Hestenes–Stiefel
+    hs_beta = (g.dot(grad_diff) / denom).neg().clamp(min=0)
+
+    return max(0, min(dy_beta, hs_beta)) # type:ignore
+
+class DYHS(ConguateGradientBase):
+    """Dai-Yuan - Hestenes–Stiefel hybrid conjugate gradient method.
+
+    Note:
+        This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+    """
+    def __init__(self, restart_interval: int | None | Literal['auto'] = None, clip_beta=False, inner: Chainable | None = None):
+        super().__init__(clip_beta=clip_beta, restart_interval=restart_interval, inner=inner)
+
+    def get_beta(self, p, g, prev_g, prev_d):
+        return dyhs_beta(g, prev_d, prev_g)
+
+
+def projected_gradient_(H:torch.Tensor, y:torch.Tensor):
+    Hy = H @ y
+    yHy = safe_clip(y.dot(Hy))
+    H -= (Hy.outer(y) @ H) / yHy
+    return H
+
+class ProjectedGradientMethod(HessianUpdateStrategy): # this doesn't maintain hessian
+    """Projected gradient method. Directly projects the gradient onto subspace conjugate to past directions.
+
+    Notes:
+        - This method uses N^2 memory.
+        - This requires step size to be determined via a line search, so put a line search like ``tz.m.StrongWolfe(c2=0.1, a_init="first-order")`` after this.
+        - This is not the same as projected gradient descent.
+
+    Reference:
+        Pearson, J. D. (1969). Variable metric methods of minimisation. The Computer Journal, 12(2), 171–178. doi:10.1093/comjnl/12.2.171.  (algorithm 5 in section 6)
+
+    """
+
+    def __init__(
+        self,
+        init_scale: float | Literal["auto"] = 1,
+        tol: float = 1e-32,
+        ptol: float | None = 1e-32,
+        ptol_restart: bool = False,
+        gtol: float | None = 1e-32,
+        restart_interval: int | None | Literal['auto'] = 'auto',
+        beta: float | None = None,
+        update_freq: int = 1,
+        scale_first: bool = False,
+        concat_params: bool = True,
+        # inverse: bool = True,
+        inner: Chainable | None = None,
+    ):
+        super().__init__(
+            defaults=None,
+            init_scale=init_scale,
+            tol=tol,
+            ptol=ptol,
+            ptol_restart=ptol_restart,
+            gtol=gtol,
+            restart_interval=restart_interval,
+            beta=beta,
+            update_freq=update_freq,
+            scale_first=scale_first,
+            concat_params=concat_params,
+            inverse=True,
+            inner=inner,
+        )
+
+
+
+    def update_H(self, H, s, y, p, g, p_prev, g_prev, state, setting):
+        return projected_gradient_(H=H, y=y)
+

torchzero/modules/experimental/__init__.py

@@ -1,24 +1,18 @@
-from .absoap import ABSOAP
-from .adadam import Adadam
-from .adamY import AdamY
-from .adasoap import AdaSOAP
+"""Those are various ideas of mine plus some other modules that I decided not to move to other sub-packages for whatever reason. This is generally less tested and shouldn't be used."""
 from .curveball import CurveBall
-from .eigendescent import EigenDescent
-from .etf import (
-    ExponentialTrajectoryFit,
-    ExponentialTrajectoryFitV2,
-    PointwiseExponential,
-)
+
+# from dct import DCTProjection
+from .fft import FFTProjection
 from .gradmin import GradMin
+from .l_infinity import InfinityNormTrustRegion
+from .momentum import (
+    CoordinateMomentum,
+    NesterovEMASquared,
+    PrecenteredEMASquared,
+    SqrtNesterovEMASquared,
+)
 from .newton_solver import NewtonSolver
 from .newtonnewton import NewtonNewton
 from .reduce_outward_lr import ReduceOutwardLR
-from .soapy import SOAPY
-from .spectral import SpectralPreconditioner
-from .structured_newton import StructuredNewton
-from .subspace_preconditioners import (
-    HistorySubspacePreconditioning,
-    RandomSubspacePreconditioning,
-)
-from .tada import TAda
-from .diagonal_higher_order_newton import DiagonalHigherOrderNewton
+from .scipy_newton_cg import ScipyNewtonCG
+from .structural_projections import BlockPartition, TensorizeProjection

torchzero/modules/{projections → experimental}/dct.py

@@ -1,13 +1,13 @@
 from typing import Literal
 import torch
 import torch_dct
-from .projection import Projection
+from ..projections import ProjectionBase
 from ...core import Chainable
 
 def reverse_dims(t:torch.Tensor):
     return t.permute(*reversed(range(t.ndim)))
 
-class DCTProjection(Projection):
+class DCTProjection(ProjectionBase):
     # norm description copied from pytorch docstring
     """Project update into Discrete Cosine Transform space, requires `torch_dct` library.
 
@@ -34,8 +34,8 @@ class DCTProjection(Projection):
         super().__init__(modules, project_update=project_update, project_params=project_params, project_grad=project_grad, defaults=defaults)
 
     @torch.no_grad
-    def project(self, tensors, var, current):
-        settings = self.settings[var.params[0]]
+    def project(self, tensors, params, grads, loss, states, settings, current):
+        settings = settings[0]
         dims = settings['dims']
         norm = settings['norm']
 
@@ -54,18 +54,18 @@ class DCTProjection(Projection):
         return projected
 
     @torch.no_grad
-    def unproject(self, tensors, var, current):
-        settings = self.settings[var.params[0]]
+    def unproject(self, projected_tensors, params, grads, loss, states, settings, current):
+        settings = settings[0]
         dims = settings['dims']
         norm = settings['norm']
 
         unprojected = []
-        for u in tensors:
-            dim = min(u.ndim, dims)
+        for t in projected_tensors:
+            dim = min(t.ndim, dims)
 
-            if dim == 1: idct = torch_dct.idct(u, norm = norm)
-            elif dim == 2: idct = torch_dct.idct_2d(u, norm=norm)
-            elif dim == 3: idct = torch_dct.idct_3d(u, norm=norm)
+            if dim == 1: idct = torch_dct.idct(t, norm = norm)
+            elif dim == 2: idct = torch_dct.idct_2d(t, norm=norm)
+            elif dim == 3: idct = torch_dct.idct_3d(t, norm=norm)
             else: raise ValueError(f"Unsupported number of dimensions {dim}")
 
             unprojected.append(reverse_dims(idct))

torchzero/modules/{projections → experimental}/fft.py

@@ -2,12 +2,12 @@ import torch
 
 from ...core import Chainable
 from ...utils import vec_to_tensors
-from .projection import Projection
+from ..projections import ProjectionBase
 
 
-class FFTProjection(Projection):
+class FFTProjection(ProjectionBase):
     # norm description copied from pytorch docstring
-    """Project update into Fourrier space of real-valued inputs.
+    """Project update into Fourier space of real-valued inputs.
 
     Args:
         modules (Chainable): modules that will optimize the projected update.
@@ -45,8 +45,8 @@ class FFTProjection(Projection):
         super().__init__(modules, project_update=project_update, project_params=project_params, project_grad=project_grad, defaults=defaults)
 
     @torch.no_grad
-    def project(self, tensors, var, current):
-        settings = self.settings[var.params[0]]
+    def project(self, tensors, params, grads, loss, states, settings, current):
+        settings = settings[0]
         one_d = settings['one_d']
         norm = settings['norm']
 
@@ -60,14 +60,14 @@ class FFTProjection(Projection):
         return [torch.view_as_real(torch.fft.rfftn(t, norm=norm)) if t.numel() > 1 else t for t in tensors] # pylint:disable=not-callable
 
     @torch.no_grad
-    def unproject(self, tensors, var, current):
-        settings = self.settings[var.params[0]]
+    def unproject(self, projected_tensors, params, grads, loss, states, settings, current):
+        settings = settings[0]
         one_d = settings['one_d']
         norm = settings['norm']
 
         if one_d:
-            vec = torch.view_as_complex(tensors[0])
+            vec = torch.view_as_complex(projected_tensors[0])
             unprojected_vec = torch.fft.irfft(vec, n=self.global_state['length'], norm=norm) # pylint:disable=not-callable
-            return vec_to_tensors(unprojected_vec, reference=var.params)
+            return vec_to_tensors(unprojected_vec, reference=params)
 
-        return [torch.fft.irfftn(torch.view_as_complex(t.contiguous()), s=p.shape, norm=norm) if t.numel() > 1 else t for t, p in zip(tensors, var.params)] # pylint:disable=not-callable
+        return [torch.fft.irfftn(torch.view_as_complex(t.contiguous()), s=p.shape, norm=norm) if t.numel() > 1 else t for t, p in zip(projected_tensors, params)] # pylint:disable=not-callable

torchzero/modules/experimental/gradmin.py

@@ -5,11 +5,11 @@ from typing import Literal
 
 import torch
 
-from ...core import Module, Var
+from ...core import Module, Var, Chainable
 from ...utils import NumberList, TensorList
 from ...utils.derivatives import jacobian_wrt
 from ..grad_approximation import GradApproximator, GradTarget
-from ..smoothing.gaussian import Reformulation
+from ..smoothing.sampling import Reformulation
 
 
 
@@ -28,6 +28,7 @@ class GradMin(Reformulation):
     """
     def __init__(
         self,
+        modules: Chainable,
         loss_term: float | None = 0,
         relative: Literal['loss_to_grad', 'grad_to_loss'] | None = None,
         graft: Literal['loss_to_grad', 'grad_to_loss'] | None = None,
@@ -39,7 +40,7 @@ class GradMin(Reformulation):
     ):
         if (relative is not None) and (graft is not None): warnings.warn('both relative and graft loss are True, they will clash with each other')
         defaults = dict(loss_term=loss_term, relative=relative, graft=graft, square=square, mean=mean, maximize_grad=maximize_grad, create_graph=create_graph, modify_loss=modify_loss)
-        super().__init__(defaults)
+        super().__init__(defaults, modules=modules)
 
     @torch.no_grad
     def closure(self, backward, closure, params, var):

torchzero/modules/experimental/l_infinity.py

@@ -0,0 +1,111 @@
+
+import numpy as np
+import torch
+from scipy.optimize import lsq_linear
+
+from ...core import Chainable, Module
+from ..trust_region.trust_region import _RADIUS_KEYS, TrustRegionBase, _RadiusStrategy
+
+
+class InfinityNormTrustRegion(TrustRegionBase):
+    """Trust region with L-infinity norm via ``scipy.optimize.lsq_linear``.
+
+    Args:
+        hess_module (Module | None, optional):
+            A module that maintains a hessian approximation (not hessian inverse!).
+            This includes all full-matrix quasi-newton methods, ``tz.m.Newton`` and ``tz.m.GaussNewton``.
+            When using quasi-newton methods, set `inverse=False` when constructing them.
+        eta (float, optional):
+            if ratio of actual to predicted rediction is larger than this, step is accepted.
+            When :code:`hess_module` is GaussNewton, this can be set to 0. Defaults to 0.15.
+        nplus (float, optional): increase factor on successful steps. Defaults to 1.5.
+        nminus (float, optional): decrease factor on unsuccessful steps. Defaults to 0.75.
+        rho_good (float, optional):
+            if ratio of actual to predicted rediction is larger than this, trust region size is multiplied by `nplus`.
+        rho_bad (float, optional):
+            if ratio of actual to predicted rediction is less than this, trust region size is multiplied by `nminus`.
+        init (float, optional): Initial trust region value. Defaults to 1.
+        update_freq (int, optional): frequency of updating the hessian. Defaults to 1.
+        max_attempts (max_attempts, optional):
+            maximum number of trust region size size reductions per step. A zero update vector is returned when
+            this limit is exceeded. Defaults to 10.
+        boundary_tol (float | None, optional):
+            The trust region only increases when suggested step's norm is at least `(1-boundary_tol)*trust_region`.
+            This prevents increasing trust region when solution is not on the boundary. Defaults to 1e-2.
+        tol (float | None, optional): tolerance for least squares solver.
+        fallback (bool, optional):
+            if ``True``, when ``hess_module`` maintains hessian inverse which can't be inverted efficiently, it will
+            be inverted anyway. When ``False`` (default), a ``RuntimeError`` will be raised instead.
+        inner (Chainable | None, optional): preconditioning is applied to output of thise module. Defaults to None.
+
+    Examples:
+        BFGS with infinity-norm trust region
+
+        .. code-block:: python
+
+            opt = tz.Modular(
+                model.parameters(),
+                tz.m.InfinityNormTrustRegion(hess_module=tz.m.BFGS(inverse=False)),
+            )
+    """
+    def __init__(
+        self,
+        hess_module: Module,
+        prefer_dense:bool=True,
+        tol: float = 1e-10,
+        eta: float= 0.0,
+        nplus: float = 3.5,
+        nminus: float = 0.25,
+        rho_good: float = 0.99,
+        rho_bad: float = 1e-4,
+        boundary_tol: float | None = None,
+        init: float = 1,
+        max_attempts: int = 10,
+        radius_strategy: _RadiusStrategy | _RADIUS_KEYS = 'default',
+        update_freq: int = 1,
+        inner: Chainable | None = None,
+    ):
+        defaults = dict(tol=tol, prefer_dense=prefer_dense)
+        super().__init__(
+            defaults=defaults,
+            hess_module=hess_module,
+            eta=eta,
+            nplus=nplus,
+            nminus=nminus,
+            rho_good=rho_good,
+            rho_bad=rho_bad,
+            boundary_tol=boundary_tol,
+            init=init,
+            max_attempts=max_attempts,
+            radius_strategy=radius_strategy,
+            update_freq=update_freq,
+            inner=inner,
+
+            radius_fn=torch.amax,
+        )
+
+    def trust_solve(self, f, g, H, radius, params, closure, settings):
+        if settings['prefer_dense'] and H.is_dense():
+            # convert to array if possible to avoid many conversions
+            # between torch and numpy, plus it seems that it uses
+            # a better solver
+            A = H.to_tensor().numpy(force=True).astype(np.float64)
+        else:
+            # memory efficient linear operator (is this still faster on CUDA?)
+            A = H.scipy_linop()
+
+        try:
+            d_np = lsq_linear(
+                A,
+                g.numpy(force=True).astype(np.float64),
+                tol=settings['bounds'],
+                bounds=(-radius, radius),
+            ).x
+            return torch.as_tensor(d_np, device=g.device, dtype=g.dtype)
+
+        except np.linalg.LinAlgError:
+            self.children['hess_module'].reset()
+            g_max = g.amax()
+            if g_max > radius:
+                g = g * (radius / g_max)
+            return g