torchzero 0.1.7__py3-none-any.whl → 0.3.1__py3-none-any.whl

torchzero/modules/line_search/line_search.py

@@ -0,0 +1,181 @@
+import math
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from functools import partial
+from operator import itemgetter
+from typing import Any
+
+import numpy as np
+import torch
+
+from ...core import Module, Target, Vars
+from ...utils import tofloat
+
+
+class MaxLineSearchItersReached(Exception): pass
+
+
+class LineSearch(Module, ABC):
+    """Base class for line searches.
+    This is an abstract class, to use it, subclass it and override `search`.
+
+    Args:
+        defaults (dict[str, Any] | None): dictionary with defaults.
+        maxiter (int | None, optional):
+            if this is specified, the search method will terminate upon evaluating
+            the objective this many times, and step size with the lowest loss value will be used.
+            This is useful when passing `make_objective` to an external library which
+            doesn't have a maxiter option. Defaults to None.
+    """
+    def __init__(self, defaults: dict[str, Any] | None, maxiter: int | None = None):
+        super().__init__(defaults)
+        self._maxiter = maxiter
+        self._reset()
+
+    def _reset(self):
+        self._current_step_size: float = 0
+        self._lowest_loss = float('inf')
+        self._best_step_size: float = 0
+        self._current_iter = 0
+
+    def set_step_size_(
+        self,
+        step_size: float,
+        params: list[torch.Tensor],
+        update: list[torch.Tensor],
+    ):
+        if not math.isfinite(step_size): return
+        step_size = max(min(tofloat(step_size), 1e36), -1e36) # fixes overflow when backtracking keeps increasing alpha after converging
+        alpha = self._current_step_size - step_size
+        if alpha != 0:
+            torch._foreach_add_(params, update, alpha=alpha)
+        self._current_step_size = step_size
+
+    def _set_per_parameter_step_size_(
+        self,
+        step_size: Sequence[float],
+        params: list[torch.Tensor],
+        update: list[torch.Tensor],
+    ):
+        if not np.isfinite(step_size): step_size = [0 for _ in step_size]
+        alpha = [self._current_step_size - s for s in step_size]
+        if any(a!=0 for a in alpha):
+            torch._foreach_add_(params, torch._foreach_mul(update, alpha))
+
+    def _loss(self, step_size: float, vars: Vars, closure, params: list[torch.Tensor],
+              update: list[torch.Tensor], backward:bool=False) -> float:
+
+        # if step_size is 0, we might already know the loss
+        if (vars.loss is not None) and (step_size == 0):
+            return tofloat(vars.loss)
+
+        # check max iter
+        if self._maxiter is not None and self._current_iter >= self._maxiter: raise MaxLineSearchItersReached
+        self._current_iter += 1
+
+        # set new lr and evaluate loss with it
+        self.set_step_size_(step_size, params=params, update=update)
+        if backward:
+            with torch.enable_grad(): loss = closure()
+        else:
+            loss = closure(False)
+
+        # if it is the best so far, record it
+        if loss < self._lowest_loss:
+            self._lowest_loss = tofloat(loss)
+            self._best_step_size = step_size
+
+        # if evaluated loss at step size 0, set it to vars.loss
+        if step_size == 0:
+            vars.loss = loss
+            if backward: vars.grad = [p.grad if p.grad is not None else torch.zeros_like(p) for p in params]
+
+        return tofloat(loss)
+
+    def _loss_derivative(self, step_size: float, vars: Vars, closure,
+                         params: list[torch.Tensor], update: list[torch.Tensor]):
+        # if step_size is 0, we might already know the derivative
+        if (vars.grad is not None) and (step_size == 0):
+            loss = self._loss(step_size=step_size,vars=vars,closure=closure,params=params,update=update,backward=False)
+            derivative = - sum(t.sum() for t in torch._foreach_mul(vars.grad, update))
+
+        else:
+            # loss with a backward pass sets params.grad
+            loss = self._loss(step_size=step_size,vars=vars,closure=closure,params=params,update=update,backward=True)
+
+            # directional derivative
+            derivative = - sum(t.sum() for t in torch._foreach_mul([p.grad if p.grad is not None
+                                                                    else torch.zeros_like(p) for p in params], update))
+
+        return loss, tofloat(derivative)
+
+    def evaluate_step_size(self, step_size: float, vars: Vars, backward:bool=False):
+        closure = vars.closure
+        if closure is None: raise RuntimeError('line search requires closure')
+        return self._loss(step_size=step_size, vars=vars, closure=closure, params=vars.params,update=vars.get_update(),backward=backward)
+
+    def evaluate_step_size_loss_and_derivative(self, step_size: float, vars: Vars):
+        closure = vars.closure
+        if closure is None: raise RuntimeError('line search requires closure')
+        return self._loss_derivative(step_size=step_size, vars=vars, closure=closure, params=vars.params,update=vars.get_update())
+
+    def make_objective(self, vars: Vars, backward:bool=False):
+        closure = vars.closure
+        if closure is None: raise RuntimeError('line search requires closure')
+        return partial(self._loss, vars=vars, closure=closure, params=vars.params, update=vars.get_update(), backward=backward)
+
+    def make_objective_with_derivative(self, vars: Vars):
+        closure = vars.closure
+        if closure is None: raise RuntimeError('line search requires closure')
+        return partial(self._loss_derivative, vars=vars, closure=closure, params=vars.params, update=vars.get_update())
+
+    @abstractmethod
+    def search(self, update: list[torch.Tensor], vars: Vars) -> float:
+        """Finds the step size to use"""
+
+    @torch.no_grad
+    def step(self, vars: Vars) -> Vars:
+        self._reset()
+        params = vars.params
+        update = vars.get_update()
+
+        try:
+            step_size = self.search(update=update, vars=vars)
+        except MaxLineSearchItersReached:
+            step_size = self._best_step_size
+
+        # set loss_approx
+        if vars.loss_approx is None: vars.loss_approx = self._lowest_loss
+
+        # this is last module - set step size to found step_size times lr
+        if vars.is_last:
+
+            if vars.last_module_lrs is None:
+                self.set_step_size_(step_size, params=params, update=update)
+
+            else:
+                self._set_per_parameter_step_size_([step_size*lr for lr in vars.last_module_lrs], params=params, update=update)
+
+            vars.stop = True; vars.skip_update = True
+            return vars
+
+        # revert parameters and multiply update by step size
+        self.set_step_size_(0, params=params, update=update)
+        torch._foreach_mul_(vars.update, step_size)
+        return vars
+
+
+class GridLineSearch(LineSearch):
+    """Mostly for testing, this is not practical"""
+    def __init__(self, start, end, num):
+        defaults = dict(start=start,end=end,num=num)
+        super().__init__(defaults)
+
+    @torch.no_grad
+    def search(self, update, vars):
+        start,end,num=itemgetter('start','end','num')(self.settings[vars.params[0]])
+
+        for lr in torch.linspace(start,end,num):
+            self.evaluate_step_size(lr.item(), vars=vars, backward=False)
+
+        return self._best_step_size

torchzero/modules/line_search/scipy.py

@@ -0,0 +1,37 @@
+from collections.abc import Mapping
+from operator import itemgetter
+
+import torch
+
+from .line_search import LineSearch
+
+
+class ScipyMinimizeScalar(LineSearch):
+    def __init__(
+        self,
+        method: str | None = None,
+        maxiter: int | None = None,
+        bracket=None,
+        bounds=None,
+        tol: float | None = None,
+        options=None,
+    ):
+        defaults = dict(method=method,bracket=bracket,bounds=bounds,tol=tol,options=options,maxiter=maxiter)
+        super().__init__(defaults)
+
+        import scipy.optimize
+        self.scopt = scipy.optimize
+
+
+    @torch.no_grad
+    def search(self, update, vars):
+        objective = self.make_objective(vars=vars)
+        method, bracket, bounds, tol, options, maxiter = itemgetter(
+            'method', 'bracket', 'bounds', 'tol', 'options', 'maxiter')(self.settings[vars.params[0]])
+
+        if maxiter is not None:
+            options = dict(options) if isinstance(options, Mapping) else {}
+            options['maxiter'] = maxiter
+
+        res = self.scopt.minimize_scalar(objective, method=method, bracket=bracket, bounds=bounds, tol=tol, options=options)
+        return res.x

torchzero/modules/line_search/strong_wolfe.py

@@ -0,0 +1,260 @@
+import math
+import warnings
+from operator import itemgetter
+
+import torch
+from torch.optim.lbfgs import _cubic_interpolate
+
+from .line_search import LineSearch
+from .backtracking import backtracking_line_search
+from ...utils import totensor
+
+
+def _zoom(f,
+          a_l, a_h,
+          f_l, g_l,
+          f_h, g_h,
+          f_0, g_0,
+          c1, c2,
+          maxzoom):
+
+    for i in range(maxzoom):
+        a_j = _cubic_interpolate(
+            *(totensor(i) for i in (a_l, f_l, g_l, a_h, f_h, g_h))
+
+        )
+
+        # if interpolation fails or produces endpoint, bisect
+        delta = abs(a_h - a_l)
+        if a_j is None or a_j == a_l or a_j == a_h:
+            a_j = a_l + 0.5 * delta
+
+
+        f_j, g_j = f(a_j)
+
+        # check armijo
+        armijo = f_j <= f_0 + c1 * a_j * g_0
+
+        # check strong wolfe
+        wolfe = abs(g_j) <= c2 * abs(g_0)
+
+
+        # minimum between alpha_low and alpha_j
+        if not armijo or f_j >= f_l:
+            a_h = a_j
+            f_h = f_j
+            g_h = g_j
+        else:
+            # alpha_j satisfies armijo
+            if wolfe:
+                return a_j, f_j
+
+            # minimum between alpha_j and alpha_high
+            if g_j * (a_h - a_l) >= 0:
+                # between alpha_low and alpha_j
+                # a_h = a_l
+                # f_h = f_l
+                # g_h = g_l
+                a_h = a_j
+                f_h = f_j
+                g_h = g_j
+
+            # is this messing it up?
+            else:
+                a_l = a_j
+                f_l = f_j
+                g_l = g_j
+
+
+
+
+        # check if interval too small
+        delta = abs(a_h - a_l)
+        if delta <= 1e-9 or delta <= 1e-6 * max(abs(a_l), abs(a_h)):
+            l_satisfies_wolfe = (f_l <= f_0 + c1 * a_l * g_0) and (abs(g_l) <= c2 * abs(g_0))
+            h_satisfies_wolfe = (f_h <= f_0 + c1 * a_h * g_0) and (abs(g_h) <= c2 * abs(g_0))
+
+            if l_satisfies_wolfe and h_satisfies_wolfe: return a_l if f_l <= f_h else a_h, f_h
+            if l_satisfies_wolfe: return a_l, f_l
+            if h_satisfies_wolfe: return a_h, f_h
+            if f_l <= f_0 + c1 * a_l * g_0: return a_l, f_l
+            return None,None
+
+        if a_j is None or a_j == a_l or a_j == a_h:
+            a_j = a_l + 0.5 * delta
+
+
+    return None,None
+
+
+def strong_wolfe(
+    f,
+    f_0,
+    g_0,
+    init: float = 1.0,
+    c1: float = 1e-4,
+    c2: float = 0.9,
+    maxiter: int = 25,
+    maxzoom: int = 15,
+    # a_max: float = 1e30,
+    expand: float = 2.0,  # Factor to increase alpha in bracketing
+    plus_minus: bool = False,
+) -> tuple[float,float] | tuple[None,None]:
+    a_prev = 0.0
+
+    if g_0 == 0: return None,None
+    if g_0 > 0:
+        # if direction is not a descent direction, perform line search in opposite direction
+        if plus_minus:
+            def inverted_objective(alpha):
+                l, g = f(-alpha)
+                return l, -g
+            a, v = strong_wolfe(
+                inverted_objective,
+                init=init,
+                f_0=f_0,
+                g_0=-g_0,
+                c1=c1,
+                c2=c2,
+                maxiter=maxiter,
+                # a_max=a_max,
+                expand=expand,
+                plus_minus=False,
+            )
+            if a is not None and v is not None: return -a, v
+        return None, None
+
+    f_prev = f_0
+    g_prev = g_0
+    a_cur = init
+
+    # bracket
+    for i in range(maxiter):
+
+        f_cur, g_cur = f(a_cur)
+
+        # check armijo
+        armijo_violated = f_cur > f_0 + c1 * a_cur * g_0
+        func_increased = f_cur >= f_prev and i > 0
+
+        if armijo_violated or func_increased:
+            return _zoom(f,
+                         a_prev, a_cur,
+                         f_prev, g_prev,
+                         f_cur, g_cur,
+                         f_0, g_0,
+                         c1, c2,
+                         maxzoom=maxzoom,
+                         )
+
+
+
+        # check strong wolfe
+        if abs(g_cur) <= c2 * abs(g_0):
+            return a_cur, f_cur
+
+        # minimum is bracketed
+        if g_cur >= 0:
+            return _zoom(f,
+                        #alpha_curr, alpha_prev,
+                        a_prev, a_cur,
+                        #phi_curr, phi_prime_curr,
+                        f_prev, g_prev,
+                        f_cur, g_cur,
+                        f_0, g_0,
+                        c1, c2,
+                        maxzoom=maxzoom,)
+
+        # otherwise continue bracketing
+        a_next = a_cur * expand
+
+        # update previous point and continue loop with increased step size
+        a_prev = a_cur
+        f_prev = f_cur
+        g_prev = g_cur
+        a_cur = a_next
+
+
+    # max iters reached
+    return None, None
+
+def _notfinite(x):
+    if isinstance(x, torch.Tensor): return not torch.isfinite(x).all()
+    return not math.isfinite(x)
+
+class StrongWolfe(LineSearch):
+    def __init__(
+        self,
+        init: float = 1.0,
+        c1: float = 1e-4,
+        c2: float = 0.9,
+        maxiter: int = 25,
+        maxzoom: int = 10,
+        # a_max: float = 1e10,
+        expand: float = 2.0,
+        adaptive = True,
+        fallback = False,
+        plus_minus = False,
+    ):
+        defaults=dict(init=init,c1=c1,c2=c2,maxiter=maxiter,maxzoom=maxzoom,
+                      expand=expand, adaptive=adaptive, fallback=fallback, plus_minus=plus_minus)
+        super().__init__(defaults=defaults)
+
+        self.global_state['initial_scale'] = 1.0
+        self.global_state['beta_scale'] = 1.0
+
+    @torch.no_grad
+    def search(self, update, vars):
+        objective = self.make_objective_with_derivative(vars=vars)
+
+        init, c1, c2, maxiter, maxzoom, expand, adaptive, fallback, plus_minus = itemgetter(
+            'init', 'c1', 'c2', 'maxiter', 'maxzoom',
+            'expand', 'adaptive', 'fallback', 'plus_minus')(self.settings[vars.params[0]])
+
+        f_0, g_0 = objective(0)
+
+        step_size,f_a = strong_wolfe(
+            objective,
+            f_0=f_0, g_0=g_0,
+            init=init * self.global_state.setdefault("initial_scale", 1),
+            c1=c1,
+            c2=c2,
+            maxiter=maxiter,
+            maxzoom=maxzoom,
+            expand=expand,
+            plus_minus=plus_minus,
+        )
+
+        if f_a is not None and (f_a > f_0 or _notfinite(f_a)): step_size = None
+        if step_size is not None and step_size != 0 and not _notfinite(step_size):
+            self.global_state['initial_scale'] = min(1.0, self.global_state['initial_scale'] * math.sqrt(2))
+            return step_size
+
+        # fallback to backtracking on fail
+        if adaptive: self.global_state['initial_scale'] *= 0.5
+        if not fallback: return 0
+
+        objective = self.make_objective(vars=vars)
+
+        # # directional derivative
+        g_0 = -sum(t.sum() for t in torch._foreach_mul(vars.get_grad(), vars.get_update()))
+
+        step_size = backtracking_line_search(
+            objective,
+            g_0,
+            init=init * self.global_state["initial_scale"],
+            beta=0.5 * self.global_state["beta_scale"],
+            c=c1,
+            maxiter=maxiter * 2,
+            a_min=None,
+            try_negative=plus_minus,
+        )
+
+        # found an alpha that reduces loss
+        if step_size is not None:
+            self.global_state['beta_scale'] = min(1.0, self.global_state.get('beta_scale', 1) * math.sqrt(1.5))
+            return step_size
+
+        # on fail reduce beta scale value
+        self.global_state['beta_scale'] /= 1.5
+        return 0

torchzero/modules/line_search/trust_region.py

@@ -0,0 +1,61 @@
+from operator import itemgetter
+
+import torch
+
+from .line_search import LineSearch
+
+
+class TrustRegion(LineSearch):
+    """Basic first order trust region, re-evaluates closure with updated parameters and scales step size based on function value change"""
+    def __init__(self, nplus: float=1.5, nminus: float=0.75, c: float=1e-4, init: float = 1, backtrack: bool = True, adaptive: bool = True):
+        defaults = dict(nplus=nplus, nminus=nminus, c=c, init=init, backtrack=backtrack, adaptive=adaptive)
+        super().__init__(defaults)
+
+    @torch.no_grad
+    def search(self, update, vars):
+
+        nplus, nminus, c, init, backtrack, adaptive = itemgetter('nplus','nminus','c','init','backtrack', 'adaptive')(self.settings[vars.params[0]])
+        step_size = self.global_state.setdefault('step_size', init)
+        previous_success = self.global_state.setdefault('previous_success', False)
+        nplus_mul =  self.global_state.setdefault('nplus_mul', 1)
+        nminus_mul = self.global_state.setdefault('nminus_mul', 1)
+
+
+        f_0 = self.evaluate_step_size(0, vars, backward=False)
+
+        # directional derivative (0 if c = 0 because it is not needed)
+        if c == 0: d = 0
+        else: d = -sum(t.sum() for t in torch._foreach_mul(vars.get_grad(), update))
+
+        # test step size
+        sufficient_f = f_0 + c * step_size * min(d, 0) # pyright:ignore[reportArgumentType]
+
+        f_1 = self.evaluate_step_size(step_size, vars, backward=False)
+
+        proposed = step_size
+
+        # very good step
+        if f_1 < sufficient_f:
+            self.global_state['step_size'] *= nplus * nplus_mul
+
+            # two very good steps in a row - increase nplus_mul
+            if adaptive:
+                if previous_success: self.global_state['nplus_mul'] *= nplus
+                else: self.global_state['nplus_mul'] = 1
+
+        # acceptable step step
+        #elif f_1 <= f_0: pass
+
+        # bad step
+        if f_1 >= f_0:
+            self.global_state['step_size'] *= nminus * nminus_mul
+
+            # two bad steps in a row - decrease nminus_mul
+            if adaptive:
+                if previous_success: self.global_state['nminus_mul'] *= nminus
+                else: self.global_state['nminus_mul'] = 1
+
+            if backtrack: proposed = 0
+            else: proposed *= nminus * nminus_mul
+
+        return proposed

torchzero/modules/lr/__init__.py

@@ -0,0 +1,2 @@
+from .lr import LR, StepSize, Warmup
+from .step_size import PolyakStepSize, RandomStepSize

torchzero/modules/lr/lr.py

@@ -0,0 +1,59 @@
+import torch
+
+from ...core import Transform
+from ...utils import NumberList, TensorList, generic_eq
+
+
+def lazy_lr(tensors: TensorList, lr: float | list, inplace:bool):
+    """multiplies by lr if lr is not 1"""
+    if generic_eq(lr, 1): return tensors
+    if inplace: return tensors.mul_(lr)
+    return tensors * lr
+
+class LR(Transform):
+    def __init__(self, lr: float):
+        defaults=dict(lr=lr)
+        super().__init__(defaults, uses_grad=False)
+
+    @torch.no_grad
+    def transform(self, tensors, params, grads, vars):
+        return lazy_lr(TensorList(tensors), lr=self.get_settings('lr', params=params), inplace=True)
+
+class StepSize(Transform):
+    """this is exactly the same as LR, except the `lr` parameter can be renamed to any other name"""
+    def __init__(self, step_size: float, key = 'step_size'):
+        defaults={"key": key, key: step_size}
+        super().__init__(defaults, uses_grad=False)
+
+    @torch.no_grad
+    def transform(self, tensors, params, grads, vars):
+        lrs = []
+        for p in params:
+            settings = self.settings[p]
+            lrs.append(settings[settings['key']])
+        return lazy_lr(TensorList(tensors), lr=lrs, inplace=True)
+
+
+def warmup(step: int, start_lr: float | NumberList, end_lr: float | NumberList, steps: float):
+    """returns warm up lr scalar"""
+    if step > steps: return end_lr
+    return start_lr + (end_lr - start_lr) * (step / steps)
+
+class Warmup(Transform):
+    def __init__(self, start_lr = 1e-5, end_lr:float = 1, steps = 100):
+        defaults = dict(start_lr=start_lr,end_lr=end_lr, steps=steps)
+        super().__init__(defaults, uses_grad=False)
+
+    @torch.no_grad
+    def transform(self, tensors, params, grads, vars):
+        start_lr, end_lr = self.get_settings('start_lr', 'end_lr', params=params, cls = NumberList)
+        num_steps = self.settings[params[0]]['steps']
+        step = self.global_state.get('step', 0)
+
+        target = lazy_lr(
+            TensorList(tensors),
+            lr=warmup(step=step, start_lr=start_lr, end_lr=end_lr, steps=num_steps),
+            inplace=True
+        )
+        self.global_state['step'] = step + 1
+        return target

torchzero/modules/lr/step_size.py

@@ -0,0 +1,97 @@
+import random
+from typing import Any
+
+import torch
+
+from ...core import Transform
+from ...utils import TensorList, NumberList
+
+
+class PolyakStepSize(Transform):
+    """Polyak step-size.
+
+    Args:
+        max (float | None, optional): maximum possible step size. Defaults to None.
+        min_obj_value (int, optional): (estimated) minimal possible value of the objective function (lowest possible loss). Defaults to 0.
+        use_grad (bool, optional):
+            if True, uses dot product of update and gradient to compute the step size.
+            Otherwise, dot product of update with itself is used, which has no geometric meaning so it probably won't work well.
+            Defaults to True.
+        parameterwise (bool, optional):
+            if True, calculate Polyak step-size for each parameter separately,
+            if False calculate one global step size for all parameters. Defaults to False.
+        alpha (float, optional): multiplier to Polyak step-size. Defaults to 1.
+    """
+    def __init__(self, max: float | None = None, min_obj_value: float = 0, use_grad=True, parameterwise=False, alpha: float = 1):
+
+        defaults = dict(alpha=alpha, max=max, min_obj_value=min_obj_value, use_grad=use_grad, parameterwise=parameterwise)
+        super().__init__(defaults, uses_grad=use_grad)
+
+    @torch.no_grad
+    def transform(self, tensors, params, grads, vars):
+        loss = vars.get_loss(False)
+        assert grads is not None
+        tensors = TensorList(tensors)
+        grads = TensorList(grads)
+        alpha = self.get_settings('alpha', params=params, cls=NumberList)
+        settings = self.settings[params[0]]
+        parameterwise = settings['parameterwise']
+        use_grad = settings['use_grad']
+        max = settings['max']
+        min_obj_value = settings['min_obj_value']
+
+        if parameterwise:
+            if use_grad: denom = (tensors * grads).sum()
+            else: denom = tensors.pow(2).sum()
+            polyak_step_size: TensorList | Any = (loss - min_obj_value) / denom.where(denom!=0, 1)
+            polyak_step_size = polyak_step_size.where(denom != 0, 0)
+            if max is not None: polyak_step_size = polyak_step_size.clamp_max(max)
+
+        else:
+            if use_grad: denom = tensors.dot(grads)
+            else: denom = tensors.dot(tensors)
+            if denom == 0: polyak_step_size = 0 # we converged
+            else: polyak_step_size = (loss - min_obj_value) / denom
+
+            if max is not None:
+                if polyak_step_size > max: polyak_step_size = max
+
+        tensors.mul_(alpha * polyak_step_size)
+        return tensors
+
+
+
+class RandomStepSize(Transform):
+    """Uses random global step size from `low` to `high`.
+
+    Args:
+        low (float, optional): minimum learning rate. Defaults to 0.
+        high (float, optional): maximum learning rate. Defaults to 1.
+        parameterwise (bool, optional):
+            if True, generate random step size for each parameter separately,
+            if False generate one global random step size. Defaults to False.
+    """
+    def __init__(self, low: float = 0, high: float = 1, parameterwise=False, seed:int|None=None):
+        defaults = dict(low=low, high=high, parameterwise=parameterwise,seed=seed)
+        super().__init__(defaults, uses_grad=False)
+
+    @torch.no_grad
+    def transform(self, tensors, params, grads, vars):
+        settings = self.settings[params[0]]
+        parameterwise = settings['parameterwise']
+
+        seed = settings['seed']
+        if 'generator' not in self.global_state:
+            self.global_state['generator'] = random.Random(seed)
+        generator: random.Random = self.global_state['generator']
+
+        if parameterwise:
+            low, high = self.get_settings('low', 'high', params=params)
+            lr = [generator.uniform(l, h) for l, h in zip(low, high)]
+        else:
+            low = settings['low']
+            high = settings['high']
+            lr = generator.uniform(low, high)
+
+        torch._foreach_mul_(tensors, lr)
+        return tensors