torchzero 0.3.11__py3-none-any.whl → 0.3.14__py3-none-any.whl

torchzero/modules/misc/switch.py

@@ -53,7 +53,7 @@ class Alternate(Module):
         var = module.step(var.clone(clone_update=False))
 
         # number of steps until next module
-        steps = self.settings[var.params[0]]['steps']
+        steps = self.defaults['steps']
         if isinstance(steps, int): steps = [steps]*len(self.children)
 
         if 'steps_to_next' not in self.global_state:

torchzero/modules/momentum/__init__.py

@@ -6,9 +6,5 @@ from .cautious import (
     ScaleModulesByCosineSimilarity,
     UpdateGradientSignConsistency,
 )
-from .ema import EMA, Debias, Debias2, EMASquared, SqrtEMASquared, CenteredEMASquared, CenteredSqrtEMASquared
-from .experimental import CoordinateMomentum
-# from .matrix_momentum import MatrixMomentum
 
-from .momentum import NAG, HeavyBall
-from .matrix_momentum import MatrixMomentum, AdaptiveMatrixMomentum
+from .momentum import NAG, HeavyBall, EMA

torchzero/modules/momentum/averaging.py

@@ -10,7 +10,7 @@ from ...utils import tolist
 
 
 class Averaging(TensorwiseTransform):
-    """Average of past :code:`history_size` updates.
+    """Average of past ``history_size`` updates.
 
     Args:
         history_size (int): Number of past updates to average
@@ -35,7 +35,7 @@ class Averaging(TensorwiseTransform):
         return average / len(history)
 
 class WeightedAveraging(TensorwiseTransform):
-    """Weighted average of past :code:`len(weights)` updates.
+    """Weighted average of past ``len(weights)`` updates.
 
     Args:
         weights (Sequence[float]): a sequence of weights from oldest to newest.
@@ -69,7 +69,7 @@ class WeightedAveraging(TensorwiseTransform):
 
 
 class MedianAveraging(TensorwiseTransform):
-    """Median of past :code:`history_size` updates.
+    """Median of past ``history_size`` updates.
 
     Args:
         history_size (int): Number of past updates to average

torchzero/modules/momentum/cautious.py

@@ -48,24 +48,22 @@ class Cautious(Transform):
         eps (float, optional): epsilon for normalization. Defaults to 1e-6.
         mode (str, optional):
             what to do with updates with inconsistent signs.
+            - "zero" - set them to zero (as in paper)
+            - "grad" - set them to the gradient (same as using update magnitude and gradient sign)
+            - "backtrack" - negate them
 
-            "zero" - set them to zero (as in paper)
+    ## Examples:
 
-            "grad" - set them to the gradient
+    Cautious Adam
 
-            "backtrack" - negate them (same as using update magnitude and gradient sign)
-
-    Examples:
-        Cautious Adam
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                bench.parameters(),
-                tz.m.Adam(),
-                tz.m.Cautious(),
-                tz.m.LR(1e-2)
-            )
+    ```python
+    opt = tz.Modular(
+        bench.parameters(),
+        tz.m.Adam(),
+        tz.m.Cautious(),
+        tz.m.LR(1e-2)
+    )
+    ```
 
     References:
         Cautious Optimizers: Improving Training with One Line of Code. Kaizhao Liang, Lizhang Chen, Bo Liu, Qiang Liu
@@ -120,12 +118,9 @@ class IntermoduleCautious(Module):
         eps (float, optional): epsilon for normalization. Defaults to 1e-6.
         mode (str, optional):
             what to do with updates with inconsistent signs.
-
-            "zero" - set them to zero (as in paper)
-
-            "grad" - set them to the gradient
-
-            "backtrack" - negate them (same as using update magnitude and gradient sign)
+            - "zero" - set them to zero (as in paper)
+            - "grad" - set them to the gradient (same as using update magnitude and gradient sign)
+            - "backtrack" - negate them
     """
     def __init__(
         self,
@@ -153,7 +148,7 @@ class IntermoduleCautious(Module):
         compare_var = compare.step(var.clone(clone_update=True))
         var.update_attrs_from_clone_(compare_var)
 
-        mode, normalize, eps = itemgetter('mode', 'normalize', 'eps')(self.settings[var.params[0]])
+        mode, normalize, eps = itemgetter('mode', 'normalize', 'eps')(self.defaults)
         var.update = cautious_(
             TensorList(main_var.get_update()),
             TensorList(compare_var.get_update()),
@@ -171,17 +166,17 @@ class ScaleByGradCosineSimilarity(Transform):
     Args:
         eps (float, optional): epsilon for division. Defaults to 1e-6.
 
-    Examples:
-        Scaled Adam
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                bench.parameters(),
-                tz.m.Adam(),
-                tz.m.ScaleByGradCosineSimilarity(),
-                tz.m.LR(1e-2)
-            )
+    ## Examples:
+
+    Scaled Adam
+    ```python
+    opt = tz.Modular(
+        bench.parameters(),
+        tz.m.Adam(),
+        tz.m.ScaleByGradCosineSimilarity(),
+        tz.m.LR(1e-2)
+    )
+    ```
     """
     def __init__(
         self,
@@ -209,19 +204,19 @@ class ScaleModulesByCosineSimilarity(Module):
         compare (Chainable): module or sequence of modules to compare to
         eps (float, optional): epsilon for division. Defaults to 1e-6.
 
-    Example:
-        Adam scaled by similarity to RMSprop
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                bench.parameters(),
-                tz.m.ScaleModulesByCosineSimilarity(
-                    main = tz.m.Adam(),
-                    compare = tz.m.RMSprop(0.999, debiased=True),
-                ),
-                tz.m.LR(1e-2)
-            )
+    ## Examples:
+
+    Adam scaled by similarity to RMSprop
+    ```python
+    opt = tz.Modular(
+        bench.parameters(),
+        tz.m.ScaleModulesByCosineSimilarity(
+            main = tz.m.Adam(),
+            compare = tz.m.RMSprop(0.999, debiased=True),
+        ),
+        tz.m.LR(1e-2)
+    )
+    ```
     """
     def __init__(
         self,
@@ -248,7 +243,7 @@ class ScaleModulesByCosineSimilarity(Module):
 
         m = TensorList(main_var.get_update())
         c = TensorList(compare_var.get_update())
-        eps = self.settings[var.params[0]]['eps']
+        eps = self.defaults['eps']
 
         cos_sim = m.dot(c) / (m.global_vector_norm() * c.global_vector_norm()).clip(min=eps)
 

torchzero/modules/momentum/momentum.py

@@ -1,10 +1,44 @@
+from collections import deque
+from operator import itemgetter
 from typing import Literal
 
 import torch
 
 from ...core import Target, Transform
 from ...utils import NumberList, TensorList, unpack_dicts, unpack_states
-from .ema import EMA
+from ..functional import debias, ema_
+
+
+class EMA(Transform):
+    """Maintains an exponential moving average of update.
+
+    Args:
+        momentum (float, optional): momentum (beta). Defaults to 0.9.
+        dampening (float, optional): momentum dampening. Defaults to 0.
+        debiased (bool, optional): whether to debias the EMA like in Adam. Defaults to False.
+        lerp (bool, optional): whether to use linear interpolation. Defaults to True.
+        ema_init (str, optional): initial values for the EMA, "zeros" or "update".
+        target (Target, optional): target to apply EMA to. Defaults to 'update'.
+    """
+    def __init__(self, momentum:float=0.9, dampening:float=0, debiased: bool = False, lerp=True, ema_init: Literal['zeros', 'update'] = 'zeros', target: Target = 'update'):
+        defaults = dict(momentum=momentum,dampening=dampening,debiased=debiased,lerp=lerp,ema_init=ema_init)
+        super().__init__(defaults, uses_grad=False, target=target)
+
+    @torch.no_grad
+    def apply_tensors(self, tensors, params, grads, loss, states, settings):
+        step = self.global_state['step'] = self.global_state.get('step', 0) + 1
+
+        debiased, lerp, ema_init = itemgetter('debiased','lerp','ema_init')(settings[0])
+
+        exp_avg = unpack_states(states, tensors, 'exp_avg',
+                                init=torch.zeros_like if ema_init=='zeros' else tensors, cls=TensorList)
+        momentum, dampening = unpack_dicts(settings, 'momentum','dampening', cls=NumberList)
+
+        exp_avg = ema_(TensorList(tensors), exp_avg_=exp_avg,beta=momentum,dampening=dampening,lerp=lerp)
+
+        if debiased: return debias(exp_avg, step=step, beta1=momentum, alpha=1, inplace=False)
+        else: return exp_avg.clone() # this has exp_avg storage so needs to be cloned
+
 
 
 class HeavyBall(EMA):

torchzero/modules/ops/__init__.py

@@ -27,6 +27,14 @@ from .binary import (
     Sub,
     Threshold,
 )
+from .higher_level import (
+    CenteredEMASquared,
+    CenteredSqrtEMASquared,
+    Debias,
+    Debias2,
+    EMASquared,
+    SqrtEMASquared,
+)
 from .multi import (
     ClipModules,
     DivModules,
@@ -64,7 +72,7 @@ from .utility import (
     Grad,
     GradToNone,
     Identity,
-    NoOp,
+    Noop,
     Ones,
     Params,
     Randn,

torchzero/modules/ops/binary.py

@@ -57,8 +57,8 @@ class Add(BinaryOperationBase):
 
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], other: float | list[torch.Tensor]):
-        if isinstance(other, (int,float)): torch._foreach_add_(update, other * self.settings[var.params[0]]['alpha'])
-        else: torch._foreach_add_(update, other, alpha=self.settings[var.params[0]]['alpha'])
+        if isinstance(other, (int,float)): torch._foreach_add_(update, other * self.defaults['alpha'])
+        else: torch._foreach_add_(update, other, alpha=self.defaults['alpha'])
         return update
 
 class Sub(BinaryOperationBase):
@@ -72,8 +72,8 @@ class Sub(BinaryOperationBase):
 
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], other: float | list[torch.Tensor]):
-        if isinstance(other, (int,float)): torch._foreach_sub_(update, other * self.settings[var.params[0]]['alpha'])
-        else: torch._foreach_sub_(update, other, alpha=self.settings[var.params[0]]['alpha'])
+        if isinstance(other, (int,float)): torch._foreach_sub_(update, other * self.defaults['alpha'])
+        else: torch._foreach_sub_(update, other, alpha=self.defaults['alpha'])
         return update
 
 class RSub(BinaryOperationBase):
@@ -219,7 +219,7 @@ class Graft(BinaryOperationBase):
 
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], magnitude: list[torch.Tensor]):
-        tensorwise, ord, eps = itemgetter('tensorwise','ord','eps')(self.settings[var.params[0]])
+        tensorwise, ord, eps = itemgetter('tensorwise','ord','eps')(self.defaults)
         return TensorList(update).graft_(magnitude, tensorwise=tensorwise, ord=ord, eps=eps)
 
 class RGraft(BinaryOperationBase):
@@ -231,7 +231,7 @@ class RGraft(BinaryOperationBase):
 
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], direction: list[torch.Tensor]):
-        tensorwise, ord, eps = itemgetter('tensorwise','ord','eps')(self.settings[var.params[0]])
+        tensorwise, ord, eps = itemgetter('tensorwise','ord','eps')(self.defaults)
         return TensorList(direction).graft_(update, tensorwise=tensorwise, ord=ord, eps=eps)
 
 GraftToUpdate = RGraft
@@ -265,7 +265,8 @@ class GramSchimdt(BinaryOperationBase):
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], other: list[torch.Tensor]):
         update = TensorList(update); other = TensorList(other)
-        return update - (other*update) / ((other*other) + 1e-8)
+        min = torch.finfo(update[0].dtype).tiny * 2
+        return update - (other*update) / (other*other).clip(min=min)
 
 
 class Threshold(BinaryOperationBase):
@@ -276,7 +277,7 @@ class Threshold(BinaryOperationBase):
 
     @torch.no_grad
     def transform(self, var, update: list[torch.Tensor], threshold: list[torch.Tensor] | float, value: list[torch.Tensor] | float):
-        update_above = self.settings[var.params[0]]['update_above']
+        update_above = self.defaults['update_above']
         update = TensorList(update)
         if update_above:
             if isinstance(value, list): return update.where_(update>threshold, value)

torchzero/modules/{momentum/ema.py → ops/higher_level.py}

@@ -5,39 +5,16 @@ from typing import Literal
 import torch
 
 from ...core import Target, Transform
-from ...utils import TensorList, NumberList, unpack_dicts, unpack_states
-from ..functional import debias, ema_, ema_sq_, sqrt_ema_sq_, centered_ema_sq_, sqrt_centered_ema_sq_, debias_second_momentum
-
-
-class EMA(Transform):
-    """Maintains an exponential moving average of update.
-
-    Args:
-        momentum (float, optional): momentum (beta). Defaults to 0.9.
-        dampening (float, optional): momentum dampening. Defaults to 0.
-        debiased (bool, optional): whether to debias the EMA like in Adam. Defaults to False.
-        lerp (bool, optional): whether to use linear interpolation. Defaults to True.
-        ema_init (str, optional): initial values for the EMA, "zeros" or "update".
-        target (Target, optional): target to apply EMA to. Defaults to 'update'.
-    """
-    def __init__(self, momentum:float=0.9, dampening:float=0, debiased: bool = False, lerp=True, ema_init: Literal['zeros', 'update'] = 'zeros', target: Target = 'update'):
-        defaults = dict(momentum=momentum,dampening=dampening,debiased=debiased,lerp=lerp,ema_init=ema_init)
-        super().__init__(defaults, uses_grad=False, target=target)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        step = self.global_state['step'] = self.global_state.get('step', 0) + 1
-
-        debiased, lerp, ema_init = itemgetter('debiased','lerp','ema_init')(settings[0])
-
-        exp_avg = unpack_states(states, tensors, 'exp_avg',
-                                init=torch.zeros_like if ema_init=='zeros' else tensors, cls=TensorList)
-        momentum, dampening = unpack_dicts(settings, 'momentum','dampening', cls=NumberList)
-
-        exp_avg = ema_(TensorList(tensors), exp_avg_=exp_avg,beta=momentum,dampening=dampening,lerp=lerp)
-
-        if debiased: return debias(exp_avg, step=step, beta1=momentum, alpha=1, inplace=False)
-        else: return exp_avg.clone() # this has exp_avg storage so needs to be cloned
+from ...utils import NumberList, TensorList, unpack_dicts, unpack_states
+from ..functional import (
+    centered_ema_sq_,
+    debias,
+    debias_second_momentum,
+    ema_,
+    ema_sq_,
+    sqrt_centered_ema_sq_,
+    sqrt_ema_sq_,
+)
 
 
 class EMASquared(Transform):

torchzero/modules/ops/multi.py

@@ -8,7 +8,7 @@ from typing import Any, Literal
 import torch
 
 from ...core import Chainable, Module, Target, Var, maybe_chain
-from ...utils import TensorList, tensorlist
+from ...utils import TensorList, tensorlist, Metrics
 
 
 class MultiOperationBase(Module, ABC):
@@ -59,7 +59,7 @@ class SubModules(MultiOperationBase):
 
     @torch.no_grad
     def transform(self, var: Var, input: float | list[torch.Tensor], other: float | list[torch.Tensor]) -> list[torch.Tensor]:
-        alpha = self.settings[var.params[0]]['alpha']
+        alpha = self.defaults['alpha']
 
         if isinstance(input, (int,float)):
             assert isinstance(other, list)
@@ -112,7 +112,7 @@ class LerpModules(MultiOperationBase):
 
     @torch.no_grad
     def transform(self, var: Var, input: list[torch.Tensor], end: list[torch.Tensor]) -> list[torch.Tensor]:
-        torch._foreach_lerp_(input, end, weight=self.settings[var.params[0]]['weight'])
+        torch._foreach_lerp_(input, end, weight=self.defaults['weight'])
         return input
 
 class ClipModules(MultiOperationBase):
@@ -154,45 +154,45 @@ class GraftModules(MultiOperationBase):
     Reference:
         Agarwal, N., Anil, R., Hazan, E., Koren, T., & Zhang, C. (2020). Disentangling adaptive gradient methods from learning rates. arXiv preprint arXiv:2002.11803. https://arxiv.org/pdf/2002.11803
     """
-    def __init__(self, direction: Chainable, magnitude: Chainable, tensorwise:bool=True, ord:float=2, eps:float = 1e-6, strength:float=1):
+    def __init__(self, direction: Chainable, magnitude: Chainable, tensorwise:bool=True, ord:Metrics=2, eps:float = 1e-6, strength:float=1):
         defaults = dict(tensorwise=tensorwise, ord=ord, eps=eps, strength=strength)
         super().__init__(defaults, direction=direction, magnitude=magnitude)
 
     @torch.no_grad
     def transform(self, var, magnitude: list[torch.Tensor], direction:list[torch.Tensor]):
-        tensorwise, ord, eps, strength = itemgetter('tensorwise','ord','eps', 'strength')(self.settings[var.params[0]])
+        tensorwise, ord, eps, strength = itemgetter('tensorwise','ord','eps', 'strength')(self.defaults)
         return TensorList(direction).graft_(magnitude, tensorwise=tensorwise, ord=ord, eps=eps, strength=strength)
 
 class MultiplyByModuleNorm(MultiOperationBase):
     """Outputs :code:`input` multiplied by norm of the :code:`norm` output."""
-    def __init__(self, input: Chainable, norm: Chainable, tensorwise:bool=True, ord:float|Literal['mean_abs']=2):
+    def __init__(self, input: Chainable, norm: Chainable, tensorwise:bool=True, ord:Metrics=2):
         defaults = dict(tensorwise=tensorwise, ord=ord)
         super().__init__(defaults, input=input, norm=norm)
 
     @torch.no_grad
     def transform(self, var, input: list[torch.Tensor], norm:list[torch.Tensor]):
-        tensorwise, ord = itemgetter('tensorwise','ord')(self.settings[var.params[0]])
+        tensorwise, ord = itemgetter('tensorwise','ord')(self.defaults)
         if tensorwise:
-            if ord == 'mean_abs': n = [t.mean() for t in torch._foreach_abs(norm)]
-            else: n = torch._foreach_norm(norm, ord)
-        else: n = TensorList(norm).global_vector_norm(ord)
+            n = TensorList(norm).metric(ord)
+        else:
+            n = TensorList(norm).global_metric(ord)
 
         torch._foreach_mul_(input, n)
         return input
 
 class DivideByModuleNorm(MultiOperationBase):
     """Outputs :code:`input` divided by norm of the :code:`norm` output."""
-    def __init__(self, input: Chainable, norm: Chainable, tensorwise:bool=True, ord:float|Literal['mean_abs']=2):
+    def __init__(self, input: Chainable, norm: Chainable, tensorwise:bool=True, ord:Metrics=2):
         defaults = dict(tensorwise=tensorwise, ord=ord)
         super().__init__(defaults, input=input, norm=norm)
 
     @torch.no_grad
     def transform(self, var, input: list[torch.Tensor], norm:list[torch.Tensor]):
-        tensorwise, ord = itemgetter('tensorwise','ord')(self.settings[var.params[0]])
+        tensorwise, ord = itemgetter('tensorwise','ord')(self.defaults)
         if tensorwise:
-            if ord == 'mean_abs': n = [t.mean().clip(min=1e-8) for t in torch._foreach_abs(norm)]
-            else: n = torch._foreach_clamp_min(torch._foreach_norm(norm, ord), 1e-8)
-        else: n = TensorList(norm).global_vector_norm(ord).clip(min=1e-8)
+            n = TensorList(norm).metric(ord)
+        else:
+            n = TensorList(norm).global_metric(ord)
 
         torch._foreach_div_(input, n)
         return input

torchzero/modules/ops/reduce.py

@@ -81,7 +81,7 @@ class WeightedSum(ReduceOperationBase):
     @torch.no_grad
     def transform(self, var: Var, *inputs: float | list[torch.Tensor]) -> list[torch.Tensor]:
         sorted_inputs = sorted(inputs, key=lambda x: isinstance(x, float))
-        weights = self.settings[var.params[0]]['weights']
+        weights = self.defaults['weights']
         sum = cast(list, sorted_inputs[0])
         torch._foreach_mul_(sum, weights[0])
         if len(sorted_inputs) > 1:

torchzero/modules/ops/utility.py

@@ -4,7 +4,7 @@ import torch
 
 from ...core import Module, Target, Transform
 from ...utils.tensorlist import Distributions, TensorList
-
+from ...utils.linalg.linear_operator import ScaledIdentity
 
 class Clone(Module):
     """Clones input. May be useful to store some intermediate result and make sure it doesn't get affected by in-place operations"""
@@ -64,15 +64,15 @@ class Fill(Module):
 
 class RandomSample(Module):
     """Outputs tensors filled with random numbers from distribution depending on value of :code:`distribution`."""
-    def __init__(self, eps: float = 1, distribution: Distributions = 'normal'):
-        defaults = dict(eps=eps, distribution=distribution)
+    def __init__(self, distribution: Distributions = 'normal', variance:float | None = None):
+        defaults = dict(distribution=distribution, variance=variance)
         super().__init__(defaults)
 
     @torch.no_grad
     def step(self, var):
-        var.update = TensorList(var.params).sample_like(
-            eps=[self.settings[p]['eps'] for p in var.params], distribution=self.settings[var.params[0]]['distribution']
-        )
+        distribution = self.defaults['distribution']
+        variance = self.get_settings(var.params, 'variance')
+        var.update = TensorList(var.params).sample_like(distribution=distribution, variance=variance)
         return var
 
 class Randn(Module):
@@ -112,9 +112,13 @@ class UpdateToNone(Module):
         return var
 
 class Identity(Module):
-    """A placeholder identity operator that is argument-insensitive."""
+    """Identity operator that is argument-insensitive. This also can be used as identity hessian for trust region methods."""
     def __init__(self, *args, **kwargs): super().__init__()
     def step(self, var): return var
+    def get_H(self, var):
+        n = sum(p.numel() for p in var.params)
+        p = var.params[0]
+        return ScaledIdentity(shape=(n,n), device=p.device, dtype=p.dtype)
 
-NoOp = Identity
+Noop = Identity
 """A placeholder identity operator that is argument-insensitive."""

torchzero/modules/projections/projection.py

@@ -1,7 +1,7 @@
 import math
 import warnings
 from abc import ABC, abstractmethod
-from collections import defaultdict, ChainMap
+from collections import ChainMap, defaultdict
 from collections.abc import Iterable, Mapping, Sequence
 from functools import partial
 from typing import Any, Literal
@@ -9,7 +9,7 @@ from typing import Any, Literal
 import torch
 
 from ...core import Chainable, Module, Var
-from ...utils import vec_to_tensors, set_storage_
+from ...utils import set_storage_, vec_to_tensors
 
 
 def _make_projected_closure(closure, project_fn, unproject_fn,
@@ -166,7 +166,7 @@ class ProjectionBase(Module, ABC):
                 current=current,
             ))
 
-        projected_var = var.clone(clone_update=False)
+        projected_var = var.clone(clone_update=False, parent=var)
 
         closure = var.closure
 
@@ -278,7 +278,7 @@ class ProjectionBase(Module, ABC):
         unprojected_var = projected_var.clone(clone_update=False)
         unprojected_var.closure = var.closure
         unprojected_var.params = var.params
-        unprojected_var.grad = var.grad
+        unprojected_var.grad = var.grad # this may also be set by projected_var since it has var as parent
 
         if self._project_update:
             assert projected_var.update is not None

torchzero/modules/quasi_newton/__init__.py

@@ -1,14 +1,3 @@
-from .cg import (
-    ConjugateDescent,
-    DaiYuan,
-    FletcherReeves,
-    HagerZhang,
-    HestenesStiefel,
-    HybridHS_DY,
-    LiuStorey,
-    PolakRibiere,
-    ProjectedGradientMethod,
-)
 from .diagonal_quasi_newton import (
     DNRTR,
     DiagonalBFGS,
@@ -19,9 +8,6 @@ from .diagonal_quasi_newton import (
 )
 from .lbfgs import LBFGS
 from .lsr1 import LSR1
-# from .olbfgs import OnlineLBFGS
-
-# from .experimental import ModularLBFGS
 from .quasi_newton import (
     BFGS,
     DFP,
@@ -40,7 +26,6 @@ from .quasi_newton import (
     NewSSM,
     Pearson,
     ProjectedNewtonRaphson,
-    ThomasOptimalMethod,
     ShorR,
+    ThomasOptimalMethod,
 )
-from .trust_region import CubicRegularization, TrustCG, TrustRegionBase