torchzero 0.3.11__py3-none-any.whl → 0.3.13__py3-none-any.whl

torchzero/modules/experimental/adaptive_step_size.py

@@ -1,90 +0,0 @@
-from operator import itemgetter
-
-import torch
-
-from ..line_search import LineSearchBase
-
-
-class AdaptiveStepSize(LineSearchBase):
-    """Basic first order step size adaptation method. Re-evaluates the function after stepping, if value decreased sufficiently,
-    step size is increased. If value increased, step size is decreased.
-
-    .. note::
-        This works well in some cases, but it is often prone to collapsing.
-        For a more robust alternative use :code:`tz.m.AdaptiveBacktracking`.
-
-    Args:
-        nplus (float, optional): multiplier to step size on successful steps. Defaults to 1.5.
-        nminus (float, optional): multiplier to step size on unsuccessful steps. Defaults to 0.75.
-        c (float, optional): descent condition. Defaults to 1e-4.
-        init (float, optional): initial step size. Defaults to 1.
-        backtrack (bool, optional): whether to undo the step if value increased. Defaults to True.
-        adaptive (bool, optional):
-            If enabled, when multiple consecutive steps have been successful or unsuccessful,
-            the corresponding multipliers are increased, otherwise they are reset. Defaults to True.
-
-
-    Examples:
-        Adagrad with trust region:
-
-        .. code-block:: python
-
-            opt = tz.Modular(
-                model.parameters(),
-                tz.m.Adagrad(),
-                tz.m.TrustRegion()
-            )
-
-    """
-    def __init__(self, nplus: float=1.5, nminus: float=0.75, c: float=1e-4, init: float = 1, backtrack: bool = True, adaptive: bool = True):
-        defaults = dict(nplus=nplus, nminus=nminus, c=c, init=init, backtrack=backtrack, adaptive=adaptive)
-        super().__init__(defaults)
-
-    @torch.no_grad
-    def search(self, update, var):
-
-        nplus, nminus, c, init, backtrack, adaptive = itemgetter('nplus','nminus','c','init','backtrack', 'adaptive')(self.settings[var.params[0]])
-        step_size = self.global_state.setdefault('step_size', init)
-        previous_success = self.global_state.setdefault('previous_success', False)
-        nplus_mul =  self.global_state.setdefault('nplus_mul', 1)
-        nminus_mul = self.global_state.setdefault('nminus_mul', 1)
-
-
-        f_0 = self.evaluate_step_size(0, var, backward=False)
-
-        # directional derivative (0 if c = 0 because it is not needed)
-        if c == 0: d = 0
-        else: d = -sum(t.sum() for t in torch._foreach_mul(var.get_grad(), update))
-
-        # test step size
-        sufficient_f = f_0 + c * step_size * min(d, 0) # pyright:ignore[reportArgumentType]
-
-        f_1 = self.evaluate_step_size(step_size, var, backward=False)
-
-        proposed = step_size
-
-        # very good step
-        if f_1 < sufficient_f:
-            self.global_state['step_size'] *= nplus * nplus_mul
-
-            # two very good steps in a row - increase nplus_mul
-            if adaptive:
-                if previous_success: self.global_state['nplus_mul'] *= nplus
-                else: self.global_state['nplus_mul'] = 1
-
-        # acceptable step step
-        #elif f_1 <= f_0: pass
-
-        # bad step
-        if f_1 >= f_0:
-            self.global_state['step_size'] *= nminus * nminus_mul
-
-            # two bad steps in a row - decrease nminus_mul
-            if adaptive:
-                if previous_success: self.global_state['nminus_mul'] *= nminus
-                else: self.global_state['nminus_mul'] = 1
-
-            if backtrack: proposed = 0
-            else: proposed *= nminus * nminus_mul
-
-        return proposed

torchzero/modules/experimental/adasoap.py

@@ -1,177 +0,0 @@
-from operator import itemgetter
-
-import torch
-
-from ...core import Chainable, Transform
-from ...modules.optimizers.shampoo import _merge_small_dims, _unmerge_small_dims
-from ..optimizers.soap import (
-    get_orthogonal_matrix,
-    get_orthogonal_matrix_QR,
-    project,
-    project_back,
-)
-
-
-@torch.no_grad
-def update_adasoap_covariances_(
-    grad: torch.Tensor,
-    GGs_: list[torch.Tensor | None],
-    GG_sqs: list[torch.Tensor | None],
-    beta: float | None,
-    precond_beta: float | None,
-):
-    for i, (GG, GG_sq) in enumerate(zip(GGs_, GG_sqs)):
-        if GG is None: continue
-        assert GG_sq is not None
-
-        if precond_beta is None: GG_sq.addcmul_(GG, GG)
-        else: GG_sq.mul_(precond_beta).addcmul_(GG, GG, value=1-precond_beta)
-
-        axes = list(range(i)) + list(range(i + 1, grad.ndim)) # this works fine with 1d params
-        if beta is None: GG.add_(torch.tensordot(grad, grad, (axes, axes))) # pyright:ignore[reportArgumentType]
-        else: GG.lerp_(torch.tensordot(grad, grad, (axes, axes)), 1-beta) # pyright:ignore[reportArgumentType]
-
-
-class AdaSOAP(Transform):
-    """SOAP with diagonally preconditioned GG^Ts.
-
-    .. warning::
-        Experimental.
-
-    precond_beta - beta for GG^T squares
-
-    Verdict: It works, but it is about the same performance as Adam, but maybe more tuning potential?
-    """
-    def __init__(
-        self,
-        beta1: float = 0.95,
-        beta2: float = 0.95,
-        shampoo_beta: float | None = 0.95,
-        precond_beta: float | None = 0.95,
-        precond_freq: int = 10,
-        merge_small: bool = True,
-        max_dim: int = 2_000,
-        precondition_1d: bool = True,
-        eps: float = 1e-8,
-        decay: float | None = None,
-        alpha: float = 1,
-        unprojected_exp_avg: bool = True,
-        bias_correction: bool = True,
-    ):
-        defaults = dict(
-            beta1=beta1,
-            beta2=beta2,
-            shampoo_beta=shampoo_beta,
-            precond_beta=precond_beta,
-            precond_freq=precond_freq,
-            merge_small=merge_small,
-            max_dim=max_dim,
-            precondition_1d=precondition_1d,
-            eps=eps,
-            decay=decay,
-            unprojected_exp_avg=unprojected_exp_avg,
-            bias_correction=bias_correction,
-            alpha=alpha,
-        )
-        super().__init__(defaults, uses_grad=False)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        updates = []
-        # update preconditioners
-        for i,(p,t, state, setting) in enumerate(zip(params, tensors, states, settings)):
-
-            beta1, beta2, shampoo_beta, merge_small, max_dim, precondition_1d, eps, unprojected_exp_avg,alpha = itemgetter(
-                'beta1', 'beta2', 'shampoo_beta', 'merge_small', 'max_dim', 'precondition_1d', 'eps', 'unprojected_exp_avg','alpha')(setting)
-            precond_beta = setting['precond_beta']
-
-            if merge_small:
-                t, state['flat_sizes'], state['sort_idxs'] = _merge_small_dims(t, max_dim)
-
-            # initialize state on 1st step
-            if 'GG' not in state:
-                state["exp_avg"] = torch.zeros_like(t)
-                state["exp_avg_sq"] = torch.zeros_like(t)
-
-                if not precondition_1d and t.ndim <= 1:
-                    state['GG'] = []
-                    state['GG_sq'] = []
-
-                else:
-                    state['GG'] = [torch.zeros(s, s, dtype=t.dtype, device=t.device) if 1<s<max_dim else None for s in t.shape]
-                    state['GG_sq'] = [torch.zeros(s, s, dtype=t.dtype, device=t.device) if 1<s<max_dim else None for s in t.shape]
-
-                # either scalar parameter, 1d with precondition_1d=False, or all dims are too big.
-                if len([i is not None for i in state['GG']]) == 0:
-                    state['GG'] = None
-                    state['GG_sq'] = None
-
-                if state['GG'] is not None:
-                    assert state['GG_sq'] is not None
-                    update_adasoap_covariances_(t, GGs_=state['GG'], GG_sqs=state['GG_sq'], beta=shampoo_beta, precond_beta=precond_beta)
-                    GG_precond = [GG / (GG_sq+1e-8) if GG is not None and GG_sq is not None else None for GG, GG_sq in zip(state['GG'], state['GG_sq'])]
-                    state['Q'] = get_orthogonal_matrix(GG_precond)
-
-                state['step'] = 0
-                updates.append(tensors[i].clip(-0.1,0.1))
-                continue  # skip 1st step as in https://github.com/nikhilvyas/SOAP/blob/main/soap.py ?
-                # that can mess with other modules scaling
-
-            # Projecting gradients to the eigenbases of Shampoo's preconditioner
-            # i.e. projecting to the eigenbases of matrices in state['GG']
-            t_projected = None
-            if state['GG'] is not None:
-                t_projected = project(t, state['Q'])
-
-            # exponential moving averages
-            # this part could be foreached but I will do that at some point its not a big difference compared to preconditioning
-            exp_avg: torch.Tensor = state["exp_avg"]
-            exp_avg_sq: torch.Tensor = state["exp_avg_sq"]
-
-            if unprojected_exp_avg or t_projected is None:
-                exp_avg.lerp_(t, 1-beta1)
-            else:
-                exp_avg.lerp_(t_projected, 1-beta1)
-
-            if t_projected is None:
-                exp_avg_sq.mul_(beta2).addcmul_(t, t, value=1-beta2)
-            else:
-                exp_avg_sq.mul_(beta2).addcmul_(t_projected, t_projected, value=1-beta2)
-
-            # project exponential moving averages if they are accumulated unprojected
-            exp_avg_projected = exp_avg
-            if unprojected_exp_avg and t_projected is not None:
-                exp_avg_projected = project(exp_avg, state['Q'])
-
-            exp_avg_sq_projected = exp_avg_sq
-
-            denom = exp_avg_sq_projected.sqrt().add_(eps)
-            # print(f'{t_projected = }, {exp_avg = }, {exp_avg_projected = }, {exp_avg_sq = }, {exp_avg_sq_projected = }, {denom = }')
-
-            # Projecting back the preconditioned (by Adam) exponential moving average of gradients
-            # to the original space
-            update = exp_avg_projected / denom
-            if t_projected is not None:
-                update = project_back(update, state["Q"])
-
-            if setting['bias_correction']:
-                bias_correction1 = 1.0 - beta1 ** (state["step"]+1)
-                bias_correction2 = 1.0 - beta2 ** (state["step"]+1)
-                update *= ((bias_correction2 ** .5) / bias_correction1) * alpha
-            elif alpha is not None:
-                update *= alpha
-
-            if merge_small:
-                update = _unmerge_small_dims(update, state['flat_sizes'], state['sort_idxs'])
-
-            updates.append(update)
-            state["step"] += 1
-
-            # Update is done after the gradient step to avoid using current gradients in the projection.
-            if state['GG'] is not None:
-                update_adasoap_covariances_(t, GGs_=state['GG'], GG_sqs=state['GG_sq'], beta=shampoo_beta, precond_beta=precond_beta)
-                GG_precond = [GG / (GG_sq+1e-8) if GG is not None and GG_sq is not None else None for GG, GG_sq in zip(state['GG'], state['GG_sq'])]
-                if state['step'] % setting['precond_freq'] == 0:
-                    state['Q'], state['exp_avg_sq'] = get_orthogonal_matrix_QR(exp_avg_sq, GG_precond, state['Q'])
-
-        return updates

torchzero/modules/experimental/cosine.py

@@ -1,214 +0,0 @@
-"""A bunch of useless modules that I hate and that didn't work"""
-import torch
-
-from ...core import Chainable, Transform, apply_transform
-from ...utils import NumberList, TensorList, as_tensorlist, unpack_dicts, unpack_states
-
-
-class CosineStepSize(Transform):
-    """Adaptive step size based on cosine similarity
-
-    VERDICT: Useless. This is too unstable.
-
-    Args:
-        scale (float, optional): cosine similarity multiplier. Defaults to 0.95.
-        init (float, optional): initial step size. Defaults to 1.
-        eps (float, optional): epsilon for division stability. Defaults to 1e-12.
-        target_cossim (float, optional): cosine similarity needs to be above this to increase step size. Defaults to 1e-8.
-        inner (Chainable | None, optional):
-            inner modules applied after calculating cosine similarity and before step size correction. Defaults to None.
-    """
-    def __init__(self, scale:float = 0.95, init:float=1, eps:float=1e-12, inner:Chainable | None = None):
-        defaults = dict(scale=scale, init=init, eps=eps)
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        scale, init = unpack_dicts(settings, 'scale', 'init', cls=NumberList)
-        unpack_states(states, tensors, 'alpha', init=init, cls=NumberList) # initializes alpha to init
-        eps = settings[0]['eps']
-
-        tensors = as_tensorlist(tensors)
-        prev = unpack_states(states, tensors, 'prev', init=tensors, cls=TensorList)
-
-        tensors_norm = tensors.global_vector_norm()
-        cos_sim = (tensors.dot(prev) / (tensors_norm * prev.global_vector_norm()).clip(min=eps)).item()
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        new_alpha = []
-        for s, sc in zip(states, scale):
-            s['alpha'] *= 1 + cos_sim * sc
-            new_alpha.append(s['alpha'])
-
-        tensors.mul_(new_alpha)
-        prev.copy_(tensors)
-
-        return tensors
-
-
-
-class CosineDebounce(Transform):
-    """Debouncing when cosine similarity is less than 0.
-
-    VERDICT: Useless. This doesn't help at all.
-
-    Args:
-        scale (float, optional): cosine similarity multiplier. Defaults to 0.95.
-        eps (float, optional): epsilon for division stability. Defaults to 1e-12.
-        inner (Chainable | None, optional):
-            inner modules applied after calculating cosine similarity and before debouncing correction. Defaults to None.
-    """
-    def __init__(self, scale:float = 0.95, eps:float=1e-12, damping:float=0.95, inner:Chainable | None = None):
-        defaults = dict(scale=scale, eps=eps, damping=damping)
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        scale, damping = unpack_dicts(settings, 'scale', 'damping', cls=NumberList)
-        eps = settings[0]['eps']
-
-        tensors = as_tensorlist(tensors)
-        prev = unpack_states(states, tensors, 'prev', init=tensors, cls=TensorList).mul_(damping)
-
-        tensors_norm = tensors.global_vector_norm()
-        cos_sim = (tensors.dot(prev) / (tensors_norm * prev.global_vector_norm()).clip(min=eps)).item()
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        if cos_sim < -eps:
-            undo = prev.neg().mul_(-cos_sim * scale)
-            comb = prev.graft(tensors).add_(tensors).graft_(prev).mul_(-cos_sim*scale)
-            tensors = undo.add_(comb)
-
-        prev.copy_(tensors)
-        return tensors
-
-
-
-class CosineMomentum(Transform):
-    """Beta depends on cosine similarity. At cossim=1, beta is 0. At cossim=-1, beta is 2^power. This basically removes oscillations.
-
-    VERDICT: Useless. Worse than all other momentums.
-
-    Args:
-        scale (float, optional): cosine similarity multiplier. Defaults to 1.
-        nesterov (float, optional): whether to use nesterov momentum. Defaults to False.
-        power (float, optional): power for beta. Defaults to 1.
-        eps (float, optional): epsilon for division stability. Defaults to 1e-12.
-        inner (Chainable | None, optional):
-            inner modules applied after calculating cosine similarity and before updating exponential moving average. Defaults to None.
-    """
-    def __init__(self, scale:float = 1, nesterov: bool = False, power: float = 1, eps:float=1e-12, inner:Chainable | None = None):
-        defaults = dict(scale=scale, eps=eps, nesterov=nesterov, power=power)
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        scale, power = unpack_dicts(settings, 'scale', 'power', cls=NumberList)
-        eps = settings[0]['eps']
-        nesterov = settings[0]['nesterov']
-        exp_avg = unpack_states(states, tensors, 'exp_avg', cls=TensorList)
-
-        tensors = as_tensorlist(tensors)
-
-        tensors_norm = tensors.global_vector_norm()
-        cos_sim = (tensors.dot(exp_avg) / (tensors_norm * exp_avg.global_vector_norm()).clip(min=eps)).item()
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        beta = (1 - (cos_sim*scale)) ** power
-        if nesterov:
-            exp_avg.add_(tensors.mul(beta))
-            return tensors.add_(exp_avg)
-        else:
-            exp_avg.add_(tensors.mul_(beta))
-            return exp_avg.clone()
-
-
-class AdaptiveDifference(Transform):
-    """VERDICT: Useless. Doesn't help (sort of to be expected)."""
-    def __init__(self, inner:Chainable | None = None):
-        defaults = dict()
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        tensors = as_tensorlist(tensors)
-        prev = unpack_states(states, tensors, 'prev', init=tensors, cls=TensorList)
-
-        diff = tensors - prev.graft_(tensors)
-        prev.copy_(tensors)
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        tensors.add_(diff.graft_(tensors))
-
-        return tensors
-
-class AdaptiveDifferenceEMA(Transform):
-    """VERDICT: better than non-EMA but still useless."""
-    def __init__(self, beta=0.99, inner:Chainable | None = None):
-        defaults = dict(beta=beta)
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        tensors = as_tensorlist(tensors)
-        beta = unpack_dicts(settings, 'beta', cls=NumberList)
-        prev, diff_exp_avg = unpack_states(states, tensors, 'prev', 'diff_exp_avg', init=[tensors,torch.zeros_like], cls=TensorList)
-
-        diff = (tensors - prev.graft_(tensors)).graft_(tensors)
-        diff_exp_avg.lerp_(diff, 1-beta)
-        prev.copy_(tensors)
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        tensors.add_(diff_exp_avg.graft(tensors))
-
-        return tensors
-
-
-class ScaledAdaptiveDifference(Transform):
-    """VERDICT: Useless and doesn't help."""
-    def __init__(self, scale=0.95, damping:float=0.99, inner:Chainable | None = None):
-        defaults = dict(scale=scale, damping=damping)
-        super().__init__(defaults, uses_grad=False)
-        if inner is not None: self.set_child('inner', inner)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        tensors = as_tensorlist(tensors)
-        scale, damping = unpack_dicts(settings, 'scale', 'damping', cls=NumberList)
-        prev_tensors, prev_update = unpack_states(states, tensors, 'prev', 'prev_update', init=[tensors,tensors], cls=TensorList)
-
-        cos_sim = (tensors.dot(prev_update) / (tensors.global_vector_norm() * prev_update.global_vector_norm()).clip(min=1e-10)).item()
-
-        if 'inner' in self.children:
-            tensors = as_tensorlist(apply_transform(self.children['inner'], tensors, params, grads, loss))
-
-        if cos_sim > 0:
-            tensors.add_(prev_tensors*(cos_sim*scale))
-
-        else:
-            undo = prev_tensors.neg().mul_(-cos_sim*scale)
-            comb = prev_tensors.graft(tensors).add_(tensors).graft_(prev_tensors).mul_(-cos_sim*scale)
-            tensors = undo.add_(comb).graft_((tensors-prev_tensors).mul_(damping))
-
-        diff = tensors - prev_tensors.graft_(tensors)
-        prev_tensors.copy_(tensors)
-        diff.graft_(tensors)
-        tensors.add_(diff)
-        prev_update.copy_(tensors)
-
-        return tensors

torchzero/modules/experimental/cubic_adam.py

@@ -1,97 +0,0 @@
-import torch
-
-from ...core import Transform
-from ...utils import NumberList, TensorList, unpack_dicts, unpack_states
-
-
-def signed_cbrt(x: TensorList) -> TensorList:
-    return x.sign() * x.abs().pow(1/3)
-
-def cubic_adam_(
-    tensors: TensorList,
-    exp_avg_: TensorList,
-    exp_avg_sq_: TensorList,
-    exp_avg_cu_: TensorList,
-    alpha: float | NumberList,
-    beta1: float | NumberList,
-    beta2: float | NumberList,
-    beta3: float | NumberList,
-    eps: float | NumberList,
-    debiased: bool,
-    step: int,
-):
-    exp_avg_.lerp_(tensors, 1-beta1)
-    exp_avg_sq_.lerp_(tensors**2, 1-beta2)
-    exp_avg_cu_.lerp_(tensors**3, 1-beta3)
-
-    if debiased:
-        m1 = exp_avg_ / (1 - beta1 ** step)
-        m2 = exp_avg_sq_ / (1 - beta2 ** step)
-        m3 = exp_avg_cu_ / (1 - beta3 ** step)
-    else:
-        m1, m2, m3 = exp_avg_, exp_avg_sq_, exp_avg_cu_
-
-    # adam minimizes ax^2 + bx
-    # we are going to minimize ax^3 + bx^2 + cx
-    A = signed_cbrt(m3)
-    B = m2.sqrt()
-    C = m1
-    discriminant = B.pow(2) - 4 * A * C
-
-    denom = 2 * A
-    root = discriminant.clamp(min=0).sqrt_()
-
-    x0 = (-B + root) / (denom + eps)
-    x1 = (-B - root) / (denom + eps)
-
-    f0 = (A/3)*x0**3 + (B/2)*x0**2 + C*x0
-    f1 = (A/3)*x1**3 + (B/2)*x1**2 + C*x1
-
-    x_star = x0.where(f0 < f1, x1)
-
-    adam = -C / (B + eps)
-    x_star = adam.where(discriminant < 0, x_star)
-
-    return x_star.mul_(-alpha)
-
-class CubicAdam(Transform):
-    """Adam which has 3rd momentum and minimizes a cubic polynomial.
-
-    VERDICT: can outperform Adam very slightly. Usually very similar performance.
-
-    .. warning::
-        Experimental.
-
-    """
-    def __init__(
-        self,
-        beta1: float = 0.9,
-        beta2: float = 0.99,
-        beta3: float = 0.99,
-        eps: float = 1e-8,
-        debiased:bool=True,
-        alpha: float = 1.,
-    ):
-        defaults=dict(beta1=beta1,beta2=beta2,beta3=beta3,eps=eps,debiased=debiased,alpha=alpha)
-        super().__init__(defaults, uses_grad=False)
-
-    @torch.no_grad
-    def apply_tensors(self, tensors, params, grads, loss, states, settings):
-        step = self.global_state['step'] = self.global_state.get('step', 0) + 1
-
-        beta1,beta2,beta3,eps,alpha=unpack_dicts(settings, 'beta1','beta2','beta3','eps','alpha', cls=NumberList)
-        exp_avg, exp_avg_sq, exp_avg_cu = unpack_states(states, tensors, 'exp_avg', 'exp_avg_sq', 'exp_avg_cu', cls=TensorList)
-
-        return cubic_adam_(
-            tensors=TensorList(tensors),
-            exp_avg_=exp_avg,
-            exp_avg_sq_=exp_avg_sq,
-            exp_avg_cu_=exp_avg_cu,
-            alpha=alpha,
-            beta1=beta1,
-            beta2=beta2,
-            beta3=beta3,
-            eps=eps,
-            debiased=settings[0]['debiased'],
-            step=step,
-        )