torchzero 0.1.8__py3-none-any.whl → 0.3.2__py3-none-any.whl

torchzero/optim/zeroth_order/rfdm.py

@@ -1,217 +0,0 @@
-import torch
-
-from ...modules import SGD, WrapClosure, LR
-from ...modules import RandomizedFDM as _RandomizedFDM
-from ...modules import WeightDecay
-from ...modules.gradient_approximation._fd_formulas import _FD_Formulas
-from ...tensorlist import Distributions
-from ..modular import Modular
-
-class RandomizedFDM(Modular):
-    """Randomized finite difference gradient approximation (e.g. SPSA, RDSA, Nesterov random search).
-
-    With `forward` and `backward` formulas performs `1 + n_samples` evaluations per step;
-    with `central` formula performs `2 * n_samples` evaluations per step.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        lr (float, optional): learning rate. Defaults to 1e-3.
-        eps (float, optional): finite difference epsilon. Defaults to 1e-3.
-        formula (_FD_Formulas, optional): finite difference formula. Defaults to "forward".
-        n_samples (int, optional): number of random gradient approximations that will be averaged. Defaults to 1.
-        distribution (Distributions, optional): distribution for random perturbations. Defaults to "normal".
-        randomize_every (int, optional): number of steps between randomizing perturbations. Defaults to 1.
-        momentum (float, optional): momentum. Defaults to 0.
-        dampening (float, optional): momentum dampening. Defaults to 0.
-        nesterov (bool, optional):
-            enables nesterov momentum, otherwise uses heavyball momentum. Defaults to False.
-        weight_decay (float, optional): weight decay (L2 regularization). Defaults to 0.
-        decoupled (bool, optional):
-            decouples weight decay from gradient. If True, weight decay doesn't depend on learning rate.
-    """
-    def __init__(
-        self,
-        params,
-        lr: float = 1e-3,
-        eps: float = 1e-3,
-        formula: _FD_Formulas = "forward",
-        n_samples: int = 1,
-        distribution: Distributions = "normal",
-        momentum: float = 0,
-        dampening: float = 0,
-        nesterov: bool = False,
-        weight_decay: float = 0,
-        decoupled=False,
-    ):
-        modules: list = [
-            _RandomizedFDM(
-                eps=eps,
-                formula=formula,
-                n_samples=n_samples,
-                distribution=distribution,
-            ),
-            SGD(momentum = momentum, dampening = dampening, weight_decay = weight_decay if not decoupled else 0, nesterov = nesterov),
-            LR(lr),
-        ]
-        if decoupled: modules.append(WeightDecay(weight_decay))
-        super().__init__(params, modules)
-
-
-class SPSA(RandomizedFDM):
-    """Simultaneous perturbation stochastic approximation method.
-    This is the same as a randomized finite difference method with central formula
-    and perturbations taken from rademacher distibution.
-    Due to rademacher having values -1 or 1, the original formula divides by the perturbation,
-    but that is equivalent to multiplying by it, which is the same as central difference formula.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        lr (float, optional): learning rate. Defaults to 1e-3.
-        eps (float, optional): finite difference epsilon. Defaults to 1e-3.
-        momentum (float, optional): momentum factor. Defaults to 0.
-        weight_decay (float, optional): weight decay (L2 penalty). Defaults to 0.
-        dampening (float, optional): dampening for momentum. Defaults to 0.
-        nesterov (bool, optional): enables Nesterov momentum (supports dampening). Defaults to False.
-        formula (_FD_Formulas, optional): finite difference formula. Defaults to "central".
-        n_samples (int, optional): number of random gradient approximations that will be averaged. Defaults to 1.
-        distribution (Distributions, optional): distribution for random perturbations. Defaults to "rademacher".
-        randomize_every (int, optional): number of steps between randomizing perturbations. Defaults to 1.
-        momentum (float, optional): momentum. Defaults to 0.
-        dampening (float, optional): momentum dampening. Defaults to 0.
-        nesterov (bool, optional):
-            enables nesterov momentum, otherwise uses heavyball momentum. Defaults to False.
-        weight_decay (float, optional): weight decay (L2 regularization). Defaults to 0.
-        decoupled (bool, optional):
-            decouples weight decay from gradient. If True, weight decay doesn't depend on learning rate.
-
-    reference
-        *Spall, J. C. (1992), “Multivariate Stochastic Approximation Using a Simultaneous Perturbation
-        Gradient Approximation,” IEEE Transactions on Automatic Control, vol. 37(3), pp. 332–341.*
-    """
-    def __init__(
-        self,
-        params,
-        lr: float = 1e-3,
-        eps: float = 1e-3,
-        formula: _FD_Formulas = "central",
-        n_samples: int = 1,
-        distribution: Distributions = 'rademacher',
-        momentum: float = 0,
-        dampening: float = 0,
-        nesterov: bool = False,
-        weight_decay: float = 0,
-        decoupled=False,    ):
-        super().__init__(
-            params = params,
-            lr = lr,
-            eps = eps,
-            formula = formula,
-            n_samples = n_samples,
-            distribution = distribution,
-            momentum = momentum,
-            dampening = dampening,
-            nesterov = nesterov,
-            weight_decay = weight_decay,
-            decoupled = decoupled,
-        )
-
-
-class RandomGaussianSmoothing(RandomizedFDM):
-    """Random search with gaussian smoothing.
-    This is similar to forward randomized finite difference method, and it
-    approximates and averages the gradient with multiple random perturbations taken from normal distribution,
-    which is an approximation for the gradient of a gaussian smoothed version of the objective function.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        lr (float, optional): learning rate. Defaults to 1e-2.
-        eps (float, optional): finite difference epsilon. Defaults to 1e-2.
-        momentum (float, optional): momentum factor. Defaults to 0.
-        weight_decay (float, optional): weight decay (L2 penalty). Defaults to 0.
-        dampening (float, optional): dampening for momentum. Defaults to 0.
-        nesterov (bool, optional): enables Nesterov momentum (supports dampening). Defaults to False.
-        formula (_FD_Formulas, optional): finite difference formula. Defaults to "forward".
-        n_samples (int, optional): number of random gradient approximations that will be averaged. Defaults to 1.
-        distribution (Distributions, optional): distribution for random perturbations. Defaults to "normal".
-        randomize_every (int, optional): number of steps between randomizing perturbations. Defaults to 1.
-        momentum (float, optional): momentum. Defaults to 0.
-        dampening (float, optional): momentum dampening. Defaults to 0.
-        nesterov (bool, optional):
-            enables nesterov momentum, otherwise uses heavyball momentum. Defaults to False.
-        weight_decay (float, optional): weight decay (L2 regularization). Defaults to 0.
-        decoupled (bool, optional):
-            decouples weight decay from gradient. If True, weight decay doesn't depend on learning rate.
-
-    reference
-        *Nesterov, Y., & Spokoiny, V. (2017).
-        Random gradient-free minimization of convex functions.
-        Foundations of Computational Mathematics, 17(2), 527-566.*
-
-    """
-    def __init__(
-        self,
-        params,
-        lr: float = 1e-2,
-        eps: float = 1e-2,
-        formula: _FD_Formulas = "forward",
-        n_samples: int = 10,
-        distribution: Distributions = 'normal',
-        momentum: float = 0,
-        dampening: float = 0,
-        nesterov: bool = False,
-        weight_decay: float = 0,
-        decoupled=False
-    ):
-        super().__init__(
-            params = params,
-            lr = lr,
-            eps = eps,
-            formula = formula,
-            n_samples = n_samples,
-            distribution = distribution,
-            momentum = momentum,
-            dampening = dampening,
-            nesterov = nesterov,
-            weight_decay = weight_decay,
-            decoupled = decoupled,
-        )
-
-class RandomizedFDMWrapper(Modular):
-    """Randomized finite difference gradient approximation (e.g. SPSA, RDSA, Nesterov random search).
-
-    With `forward` and `backward` formulas performs `1 + n_samples` evaluations per step;
-    with `central` formula performs `2 * n_samples` evaluations per step.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        optimizer (torch.optim.Optimizer): optimizer that will perform optimization using RFDM-approximated gradients.
-        eps (float, optional): finite difference epsilon. Defaults to 1e-3.
-        formula (_FD_Formulas, optional): finite difference formula. Defaults to "forward".
-        n_samples (int, optional): number of random gradient approximations that will be averaged. Defaults to 1.
-        distribution (Distributions, optional): distribution for random perturbations. Defaults to "normal".
-        randomize_every (int, optional): number of steps between randomizing perturbations. Defaults to 1.
-        randomize_closure (bool, optional): whether to generate a new random perturbation each time closure
-            is evaluated with `backward=True` (this ignores `randomize_every`). Defaults to False. Defaults to False.
-    """
-    def __init__(
-        self,
-        optimizer: torch.optim.Optimizer,
-        eps: float = 1e-3,
-        formula: _FD_Formulas = "forward",
-        n_samples: int = 1,
-        distribution: Distributions = "normal",
-    ):
-        modules = [
-            _RandomizedFDM(
-                eps=eps,
-                formula=formula,
-                n_samples=n_samples,
-                distribution=distribution,
-                target = 'closure',
-            ),
-            WrapClosure(optimizer)
-        ]
-
-        # some optimizers have `eps` setting in param groups too.
-        # it should not be passed to FDM
-        super().__init__([p for g in optimizer.param_groups.copy() for p in g['params']], modules)

torchzero/optim/zeroth_order/rs.py

@@ -1,85 +0,0 @@
-import torch
-
-from ...core import TensorListOptimizer, _ClosureType
-
-
-class RandomSearch(TensorListOptimizer):
-    """Pure random search.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        min (float, optional): lower bound of the search space. Defaults to -10.
-        max (float, optional): upper bound of the search space. Defaults to 10.
-        stochastic (bool, optional):
-            evaluate function twice per step,
-            and only accept new params if they decreased the loss.
-            Defaults to False.
-    """
-    def __init__(self, params, min:float = -10, max:float = 10, stochastic = False):
-        defaults = dict(min=min, max = max)
-        super().__init__(params, defaults)
-        self.lowest_loss = float('inf')
-        self.stochastic = stochastic
-
-    @torch.no_grad
-    def step(self, closure: _ClosureType): # type:ignore # pylint:disable=W0222
-        if self.stochastic: self.lowest_loss = closure()
-
-        settings = self.get_all_group_keys()
-        params = self.get_params()
-
-        old_params = params.clone()
-        new_params = params.uniform_like(settings['min'], settings['max'])
-        params.set_(new_params)
-        loss = closure(False)
-
-        if loss < self.lowest_loss: self.lowest_loss = loss
-        else: params.set_(old_params)
-        return loss
-
-class CyclicRS(TensorListOptimizer):
-    """Performs random search cycling through each coordinate.
-    Works surprisingly well on up to ~100 dimensional problems.
-
-    Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
-        min (float, optional): lower bound of the search space. Defaults to -10.
-        max (float, optional): upper bound of the search space. Defaults to 10.
-        stochastic (bool, optional):
-            evaluate function twice per step,
-            and only accept new params if they decreased the loss.
-            Defaults to False.
-    """
-    def __init__(self, params, min:float = -10, max:float = 10, stochastic = False):
-        defaults = dict(min=min, max = max)
-        super().__init__(params, defaults)
-        self.lowest_loss = float('inf')
-        self.cur_param = 0
-        self.cur_value = 0
-        self.stochastic = stochastic
-
-    @torch.no_grad
-    def step(self, closure: _ClosureType): # type:ignore # pylint:disable=W0222
-        if self.stochastic: self.lowest_loss = closure()
-        settings = self.get_all_group_keys()
-        params = self.get_params()
-
-        if self.cur_param >= len(params): self.cur_param = 0
-        param = params[self.cur_param]
-        if self.cur_value >= param.numel():
-            self.cur_value = 0
-            self.cur_param += 1
-            if self.cur_param >= len(params): self.cur_param = 0
-            param = params[self.cur_param]
-
-        flat = param.view(-1)
-        old_value = flat[self.cur_value].clone()
-        flat[self.cur_value] = torch.rand(1).uniform_(settings['min'][self.cur_param], settings['max'][self.cur_param]) # type:ignore
-
-        loss = closure(False)
-        if loss < self.lowest_loss: self.lowest_loss = loss
-        else:
-            flat[self.cur_value] = old_value
-
-        self.cur_value += 1
-        return loss

torchzero/random/__init__.py

@@ -1 +0,0 @@
-from .random import randmask, rademacher, uniform, sphere, sample, sample_like, Distributions

torchzero/random/random.py

@@ -1,46 +0,0 @@
-from typing import Literal
-
-import torch
-
-Distributions = Literal['normal', 'uniform', 'sphere', 'rademacher']
-
-def rademacher(shape, p: float=0.5, device=None, requires_grad = False, dtype=None, generator=None):
-    """Returns a tensor filled with random numbers from Rademacher distribution.
-
-    *p* chance to draw a -1 and 1-*p* chance to draw a 1. Looks like this:
-
-    ```
-    [-1,  1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1,  1, -1,  1, -1, -1,  1, -1,  1]
-    ```
-    """
-    if isinstance(shape, int): shape = (shape, )
-    return torch.bernoulli(torch.full(shape, p, dtype=dtype, device=device, requires_grad=requires_grad), generator=generator) * 2 - 1
-
-def randmask(shape, p: float=0.5, device=None, requires_grad = False, generator=None):
-    """Returns a tensor randomly filled with True and False.
-
-    *p* chance to draw `True` and 1-*p* to draw `False`."""
-    return torch.rand(shape, device=device, requires_grad=requires_grad, generator=generator) < p
-
-def uniform(shape, low: float, high: float, device=None, requires_grad=None, dtype=None):
-    """Returns a tensor filled with random numbers from a uniform distribution between `low` and `high`."""
-    return torch.empty(shape, device=device, dtype=dtype, requires_grad=requires_grad).uniform_(low, high)
-
-def sphere(shape, radius: float, device=None, requires_grad=None, dtype=None, generator = None):
-    """Returns a tensor filled with random numbers sampled on a unit sphere with center at 0."""
-    r = torch.randn(shape, device=device, dtype=dtype, requires_grad=requires_grad, generator=generator)
-    return (r / torch.linalg.vector_norm(r)) * radius # pylint:disable=not-callable
-
-def sample(shape, eps: float = 1, distribution: Distributions = 'normal', generator=None, device=None, dtype=None, requires_grad=False):
-    """generic random sampling function for different distributions."""
-    if distribution == 'normal': return torch.randn(shape,dtype=dtype,device=device,requires_grad=requires_grad, generator=generator) * eps
-    if distribution == 'uniform':
-        return torch.empty(size=shape,dtype=dtype,device=device,requires_grad=requires_grad).uniform_(-eps/2, eps/2, generator=generator)
-
-    if distribution == 'sphere': return sphere(shape, eps,dtype=dtype,device=device,requires_grad=requires_grad, generator=generator)
-    if distribution == 'rademacher':
-        return rademacher(shape, eps,dtype=dtype,device=device,requires_grad=requires_grad, generator=generator) * eps
-    raise ValueError(f'Unknow distribution {distribution}')
-
-def sample_like(x: torch.Tensor, eps: float = 1, distribution: Distributions = 'normal', generator=None):
-    return sample(x, eps=eps, distribution=distribution, generator=generator, device=x.device, dtype=x.dtype, requires_grad=x.requires_grad)