torchzero 0.0.1__py3-none-any.whl

torchzero/optim/modular.py

@@ -0,0 +1,132 @@
+from collections import abc
+import warnings
+from inspect import cleandoc
+import torch
+
+from ..core import OptimizerModule, TensorListOptimizer, OptimizationState, _Chain, _Chainable
+from ..utils.python_tools import flatten
+
+def _unroll_modules(flat_modules: list[OptimizerModule], nested) -> list[OptimizerModule]:
+    """returns a list of all modules, including all nested ones"""
+    unrolled = []
+    for m in flat_modules:
+        unrolled.append(m)
+        if len(m.children) > 0:
+            unrolled.extend(_unroll_modules(list(m.children.values()), nested=True))
+        if nested:
+            if m.next_module is not None:
+                unrolled.extend(_unroll_modules([m.next_module], nested=True))
+    return unrolled
+
+
+class Modular(TensorListOptimizer):
+    """Creates a modular optimizer by chaining together a sequence of optimizer modules.
+
+    Args:
+        params: iterable of parameters to optimize or dicts defining parameter groups.
+        *modules (Iterable[OptimizerModule] | OptimizerModule):
+            A sequence of optimizer modules to chain together. This argument will be flattened."""
+    def __init__(self, params, *modules: _Chainable):
+        flat_modules = flatten(modules)
+        self.modules: list[OptimizerModule] = flat_modules
+        self.chain = _Chain(flat_modules)
+
+        # save unrolled modules and make sure there is only 1 LR module.
+        self.unrolled_modules = _unroll_modules(flat_modules, nested=False)
+        num_lr_modules = len([m for m in self.unrolled_modules if m.IS_LR_MODULE])
+        if num_lr_modules > 1:
+            warnings.warn(cleandoc(
+                f"""More then 1 lr modules have been added.
+                This may lead to incorrect behaviour with learning rate scheduling and per-parameter learning rates.
+                Make sure there is a single `LR` module, use `Alpha` module instead of it where needed.
+                \nList of modules: {self.unrolled_modules}; \nlist of lr modules: {[m for m in self.unrolled_modules if m.IS_LR_MODULE]}"""
+            ))
+
+        if isinstance(params, torch.nn.Module):
+            self.model = params
+            params = list(params.parameters())
+        else:
+            self.model = None
+            params = list(params)
+
+        # if there is an `lr` setting, make sure there is an LR module that can use it
+        for p in params:
+            if isinstance(p, dict):
+                if 'lr' in p:
+                    if num_lr_modules == 0:
+                        warnings.warn(cleandoc(
+                            """Passed "lr" setting in a parameter group, but there is no LR module that can use that setting.
+                            Add an `LR` module to make per-layer "lr" setting work."""
+                        ))
+
+        super().__init__(params, {})
+        self.chain._initialize_(params, set_passed_params=True)
+
+        # run post-init hooks
+        for module in self.unrolled_modules:
+            for hook in module.post_init_hooks:
+                hook(self, module)
+
+    def get_lr_module(self, last=True) -> OptimizerModule:
+        """
+        Retrieves the module in the chain that controls the learning rate.
+
+        This method is useful for setting up a learning rate scheduler. By default, it retrieves the last module in the chain
+        that has an `lr` group parameter.
+
+        Args:
+            last (bool, optional):
+                If multiple modules have an `lr` parameter, this argument controls which one is returned.
+                - If `True` (default), the last module is returned.
+                - If `False`, the first module is returned.
+
+        Returns:
+            OptimizerModule: The module that controls the learning rate.
+
+        Raises:
+            ValueError: If no modules in the chain have an `lr` parameter. To fix this, add an `LR` module.
+
+        Example:
+
+        .. code:: py
+            from torch.optim.lr_scheduler import OneCycleLR
+            import torchzero as tz
+
+            opt = tz.Modular(model.parameters(), [tz.m.RMSProp(), tz.m.LR(1e-2), tz.m.DirectionalNewton()])
+            lr_scheduler = OneCycleLR(opt.get_lr_module(), max_lr = 1e-1, total_steps = 1000, cycle_momentum=False)
+
+        """
+        modules = list(reversed(self.unrolled_modules)) if last else self.unrolled_modules
+        for m in modules:
+            if 'lr' in m.param_groups[0]: return m
+
+        raise ValueError(f'No modules out of {", ".join(m.__class__.__name__ for m in modules)} support and `lr` parameter. The easiest way to fix is is to add an `LR(1)` module at the end.')
+
+    def get_module_by_name(self, name: str | type, last=True) -> OptimizerModule:
+        """Returns the first or last module in the chain that matches the provided name or type.
+
+        Args:
+            name (str | type): the name (as a string) or the type of the module to search for.
+            last (bool, optional):
+                If multiple modules match, this argument controls which one is returned.
+                - If `True` (default), the last matching module is returned.
+                - If `False`, the first matching module is returned.
+
+        Returns:
+            OptimizerModule: The matching optimizer module.
+
+        Raises:
+            ValueError: If no modules in the chain match the provided name or type.
+        """
+        modules = list(reversed(self.unrolled_modules)) if last else self.unrolled_modules
+        for m in modules:
+            if isinstance(name, str) and m.__class__.__name__ == name: return m
+            if isinstance(name, type) and isinstance(m, name): return m
+
+        raise ValueError(f'No modules out of {", ".join(m.__class__.__name__ for m in modules)} match "{name}".')
+
+    def step(self, closure=None): # type:ignore
+        state = OptimizationState(closure, self.model)
+        res = self.chain.step(state)
+        for hook in state.post_step_hooks: hook(self, state)
+        return res

torchzero/optim/quasi_newton/__init__.py

@@ -0,0 +1 @@
+from .directional_newton import DirectionalNewton

torchzero/optim/quasi_newton/directional_newton.py

@@ -0,0 +1,58 @@
+from ...modules import (
+    SGD,
+)
+from ...modules import DirectionalNewton as _DirectionalNewton, LR
+from ..modular import Modular
+
+
+class DirectionalNewton(Modular):
+    """Minimizes a parabola in the direction of the gradient (or update if momentum or weight decay is enabled)
+    via one additional forward pass, and uses another forward pass to make sure it didn't overstep.
+    So in total this performs three forward passes and one backward.
+
+    First forward and backward pass is used to calculate the value and gradient at initial parameters.
+    Then a gradient descent step is performed with `lr` learning rate, and loss is recalculated
+    with new parameters. A quadratic is fitted to two points and gradient,
+    if it has positive curvature, this makes a step towards the minimum, and checks if lr decreased
+    with an additional forward pass.
+
+    Args:
+        params: iterable of parameters to optimize or dicts defining parameter groups.
+        lr (float, optional):
+            learning rate. Since you shouldn't put this module after LR(), you have to specify
+            the learning rate in this argument. Defaults to 1e-2.
+        max_dist (float | None, optional):
+            maximum distance to step when minimizing quadratic.
+            If minimum is further than this distance, minimization is not performed. Defaults to 1e4.
+        validate_step (bool, optional):
+            uses an additional forward pass to check
+            if step towards the minimum actually decreased the loss. Defaults to True.
+        momentum (float, optional): momentum. Defaults to 0.
+        dampening (float, optional): momentum dampening. Defaults to 0.
+        weight_decay (float, optional): weight decay (L2 regularization). Defaults to 0.
+        nesterov (bool, optional):
+            enables nesterov momentum, otherwise uses heavyball momentum. Defaults to False.
+
+    Note:
+        While lr scheduling is supported, this uses lr of the first parameter for all parameters.
+    """
+    def __init__(
+        self,
+        params,
+        lr: float = 1e-4,
+        max_dist: float | None = 1e5,
+        validate_step: bool = True,
+        momentum: float = 0,
+        dampening: float = 0,
+        weight_decay: float = 0,
+        nesterov: bool = False,
+
+    ):
+
+        modules = [
+            SGD(momentum=momentum,dampening=dampening,weight_decay=weight_decay,nesterov=nesterov),
+            LR(lr),
+            _DirectionalNewton(max_dist, validate_step)
+        ]
+        super().__init__(params, modules)
+

torchzero/optim/second_order/__init__.py

@@ -0,0 +1 @@
+from .newton import ExactNewton

torchzero/optim/second_order/newton.py

@@ -0,0 +1,94 @@
+from typing import Any, Literal
+
+import torch
+
+from ...modules import (
+    LR,
+    ClipNorm,
+    FallbackLinearSystemSolvers,
+    LinearSystemSolvers,
+    LineSearches,
+    get_line_search,
+)
+from ...modules import ExactNewton as _ExactNewton
+from ..modular import Modular
+
+
+class ExactNewton(Modular):
+    """Peforms an exact Newton step using batched autograd. Note that torch.func would be way more efficient
+    but much more restrictive to what operations are allowed (I will add it at some point).
+
+    Args:
+        params: iterable of parameters to optimize or dicts defining parameter groups.
+        lr (float, optional): learning rate. Defaults to 1.
+        tikhonov (float, optional):
+            tikhonov regularization (constant value added to the diagonal of the hessian). Defaults to 0.
+        solver (LinearSystemSolvers, optional):
+            solver for Hx = g. Defaults to "cholesky_lu" (cholesky or LU if it fails).
+        fallback (FallbackLinearSystemSolvers, optional):
+            what to do if solver fails. Defaults to "safe_diag"
+            (takes nonzero diagonal elements, or fallbacks to gradient descent if all elements are 0).
+        max_norm (float, optional):
+            clips the newton step to L2 norm to avoid instability by giant steps.
+            A mauch better way is to use trust region methods. I haven't implemented any
+            but you can use `tz.optim.wrappers.scipy.ScipyMinimize` with one of the trust region methods.
+            Defaults to None.
+        validate (bool, optional):
+            validate if the step didn't increase the loss by `loss * tol` with an additional forward pass.
+            If not, undo the step and perform a gradient descent step.
+        tol (float, optional):
+            only has effect if `validate` is enabled.
+            If loss increased by `loss * tol`, perform gradient descent step.
+            Set this to 0 to guarantee that loss always decreases. Defaults to 1.
+        gd_lr (float, optional):
+            only has effect if `validate` is enabled.
+            Gradient descent step learning rate. Defaults to 1e-2.
+        line_search (OptimizerModule | None, optional): line search module, can be None. Defaults to None.
+        batched_hessian (bool, optional):
+            whether to use experimental pytorch vmap-vectorized hessian calculation. As per pytorch docs,
+            should be faster, but this feature being experimental, there may be performance cliffs.
+            Defaults to True.
+        diag (False, optional):
+            only use the diagonal of the hessian. This will still calculate the full hessian!
+            This is mainly useful for benchmarking.
+    """
+    def __init__(
+        self,
+        params,
+        lr: float = 1,
+        tikhonov: float | Literal['eig'] = 0.0,
+        solver: LinearSystemSolvers = "cholesky_lu",
+        fallback: FallbackLinearSystemSolvers = "safe_diag",
+        max_norm: float | None = None,
+        validate=False,
+        tol: float = 1,
+        gd_lr = 1e-2,
+        line_search: LineSearches | None = None,
+        batched_hessian = True,
+
+        diag: bool = False,
+    ):
+        modules: list[Any] = [
+            _ExactNewton(
+                tikhonov=tikhonov,
+                batched_hessian=batched_hessian,
+                solver=solver,
+                fallback=fallback,
+                validate=validate,
+                tol = tol,
+                gd_lr=gd_lr,
+                diag = diag,
+            ),
+        ]
+
+        if max_norm is not None:
+            modules.append(ClipNorm(max_norm))
+
+        modules.append(LR(lr))
+
+        if line_search is not None:
+            modules.append(get_line_search(line_search))
+
+        super().__init__(params, modules)
+
+

torchzero/optim/wrappers/nevergrad.py

@@ -0,0 +1,113 @@
+import typing
+from collections import abc
+
+import numpy as np
+import torch
+
+import nevergrad as ng
+
+from ...core import TensorListOptimizer
+
+
+def _ensure_float(x):
+    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
+    if isinstance(x, np.ndarray): return x.item()
+    return float(x)
+
+class NevergradOptimizer(TensorListOptimizer):
+    """Use nevergrad optimizer as pytorch optimizer.
+    Note that it is recommended to specify `budget` to the number of iterations you expect to run,
+    as some nevergrad optimizers will error without it.
+
+    Args:
+        params: iterable of parameters to optimize or dicts defining parameter groups.
+        opt_cls (type[ng.optimizers.base.Optimizer]):
+            nevergrad optimizer class. For example, `ng.optimizers.NGOpt`.
+        budget (int | None, optional):
+            nevergrad parameter which sets allowed number of function evaluations (forward passes).
+            This only affects the behaviour of many nevergrad optimizers, for example some
+            use certain rule for first 50% of the steps, and then switch to another rule.
+            This parameter doesn't actually limit the maximum number of steps!
+            But it doesn't have to be exact. Defaults to None.
+        mutable_sigma (bool, optional):
+            nevergrad parameter, sets whether the mutation standard deviation must mutate as well
+            (for mutation based algorithms). Defaults to False.
+        use_init (bool, optional):
+            whether to use initial model parameters as initial parameters for the nevergrad parametrization.
+            The reason you might want to set this to False is because True seems to break some optimizers
+            (mainly portfolio ones by initalizing them all to same parameters so they all perform exactly the same steps).
+            However if you are fine-tuning something, you have to set this to True, otherwise it will start from
+            new random parameters. Defaults to True.
+    """
+    def __init__(
+        self,
+        params,
+        opt_cls:"type[ng.optimizers.base.Optimizer] | abc.Callable[..., ng.optimizers.base.Optimizer]",
+        budget: int | None = None,
+        mutable_sigma = False,
+        lb: float | None = None,
+        ub: float | None = None,
+        use_init = True,
+    ):
+        defaults = dict(lb=lb, ub=ub, use_init=use_init, mutable_sigma=mutable_sigma)
+        super().__init__(params, defaults)
+        self.opt_cls = opt_cls
+        self.opt = None
+        self.budget = budget
+
+    @torch.no_grad
+    def step(self, closure): # type:ignore # pylint:disable=signature-differs
+        params = self.get_params()
+        if self.opt is None:
+            ng_params = []
+            for group in self.param_groups:
+                params = group['params']
+                mutable_sigma = group['mutable_sigma']
+                use_init = group['use_init']
+                lb = group['lb']
+                ub = group['ub']
+                for p in params:
+                    if p.requires_grad:
+                        if use_init:
+                            ng_params.append(
+                                ng.p.Array(init = p.detach().cpu().numpy(), lower=lb, upper=ub, mutable_sigma=mutable_sigma))
+                        else:
+                            ng_params.append(
+                                ng.p.Array(shape = p.shape, lower=lb, upper=ub, mutable_sigma=mutable_sigma))
+
+            parametrization = ng.p.Tuple(*ng_params)
+            self.opt = self.opt_cls(parametrization, budget=self.budget)
+
+        x: ng.p.Tuple = self.opt.ask() # type:ignore
+        for cur, new in zip(params, x):
+            cur.set_(torch.from_numpy(new.value).to(dtype=cur.dtype, device=cur.device, copy=False).reshape_as(cur)) # type:ignore
+
+        loss = closure(False)
+        self.opt.tell(x, _ensure_float(loss))
+        return loss
+
+
+
+# class NevergradSubspace(ModularOptimizer):
+#     def __init__(
+#         self,
+#         params,
+#         opt_cls:"type[ng.optimizers.base.Optimizer] | abc.Callable[..., ng.optimizers.base.Optimizer]",
+#         budget=None,
+#         mutable_sigma = False,
+#         use_init = True,
+#         projections = Proj2Masks(5),
+#     ):
+
+#         modules = [
+#             Subspace(projections, update_every=100),
+#             UninitializedClosureOptimizerWrapper(
+#                 NevergradOptimizer,
+#                 opt_cls = opt_cls,
+#                 budget = budget,
+#                 mutable_sigma = mutable_sigma,
+#                 use_init = use_init,
+#             ),
+#         ]
+
+#         super().__init__(params, modules)

torchzero/optim/wrappers/nlopt.py

@@ -0,0 +1,165 @@
+from typing import Literal
+from collections.abc import Mapping, Callable
+from functools import partial
+import numpy as np
+import torch
+
+import nlopt
+from ...core import TensorListOptimizer, _ClosureType
+from ...tensorlist import TensorList
+
+_ALGOS_LITERAL = Literal[
+    "GN_DIRECT",  # = _nlopt.GN_DIRECT
+    "GN_DIRECT_L",  # = _nlopt.GN_DIRECT_L
+    "GN_DIRECT_L_RAND",  # = _nlopt.GN_DIRECT_L_RAND
+    "GN_DIRECT_NOSCAL",  # = _nlopt.GN_DIRECT_NOSCAL
+    "GN_DIRECT_L_NOSCAL",  # = _nlopt.GN_DIRECT_L_NOSCAL
+    "GN_DIRECT_L_RAND_NOSCAL",  # = _nlopt.GN_DIRECT_L_RAND_NOSCAL
+    "GN_ORIG_DIRECT",  # = _nlopt.GN_ORIG_DIRECT
+    "GN_ORIG_DIRECT_L",  # = _nlopt.GN_ORIG_DIRECT_L
+    "GD_STOGO",  # = _nlopt.GD_STOGO
+    "GD_STOGO_RAND",  # = _nlopt.GD_STOGO_RAND
+    "LD_LBFGS_NOCEDAL",  # = _nlopt.LD_LBFGS_NOCEDAL
+    "LD_LBFGS",  # = _nlopt.LD_LBFGS
+    "LN_PRAXIS",  # = _nlopt.LN_PRAXIS
+    "LD_VAR1",  # = _nlopt.LD_VAR1
+    "LD_VAR2",  # = _nlopt.LD_VAR2
+    "LD_TNEWTON",  # = _nlopt.LD_TNEWTON
+    "LD_TNEWTON_RESTART",  # = _nlopt.LD_TNEWTON_RESTART
+    "LD_TNEWTON_PRECOND",  # = _nlopt.LD_TNEWTON_PRECOND
+    "LD_TNEWTON_PRECOND_RESTART",  # = _nlopt.LD_TNEWTON_PRECOND_RESTART
+    "GN_CRS2_LM",  # = _nlopt.GN_CRS2_LM
+    "GN_MLSL",  # = _nlopt.GN_MLSL
+    "GD_MLSL",  # = _nlopt.GD_MLSL
+    "GN_MLSL_LDS",  # = _nlopt.GN_MLSL_LDS
+    "GD_MLSL_LDS",  # = _nlopt.GD_MLSL_LDS
+    "LD_MMA",  # = _nlopt.LD_MMA
+    "LN_COBYLA",  # = _nlopt.LN_COBYLA
+    "LN_NEWUOA",  # = _nlopt.LN_NEWUOA
+    "LN_NEWUOA_BOUND",  # = _nlopt.LN_NEWUOA_BOUND
+    "LN_NELDERMEAD",  # = _nlopt.LN_NELDERMEAD
+    "LN_SBPLX",  # = _nlopt.LN_SBPLX
+    "LN_AUGLAG",  # = _nlopt.LN_AUGLAG
+    "LD_AUGLAG",  # = _nlopt.LD_AUGLAG
+    "LN_AUGLAG_EQ",  # = _nlopt.LN_AUGLAG_EQ
+    "LD_AUGLAG_EQ",  # = _nlopt.LD_AUGLAG_EQ
+    "LN_BOBYQA",  # = _nlopt.LN_BOBYQA
+    "GN_ISRES",  # = _nlopt.GN_ISRES
+    "AUGLAG",  # = _nlopt.AUGLAG
+    "AUGLAG_EQ",  # = _nlopt.AUGLAG_EQ
+    "G_MLSL",  # = _nlopt.G_MLSL
+    "G_MLSL_LDS",  # = _nlopt.G_MLSL_LDS
+    "LD_SLSQP",  # = _nlopt.LD_SLSQP
+    "LD_CCSAQ",  # = _nlopt.LD_CCSAQ
+    "GN_ESCH",  # = _nlopt.GN_ESCH
+    "GN_AGS",  # = _nlopt.GN_AGS
+]
+
+def _ensure_float(x):
+    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
+    if isinstance(x, np.ndarray): return x.item()
+    return float(x)
+
+def _ensure_tensor(x):
+    if isinstance(x, np.ndarray):
+        x.setflags(write=True)
+        return torch.from_numpy(x)
+    return torch.tensor(x, dtype=torch.float32)
+
+inf = float('inf')
+class NLOptOptimizer(TensorListOptimizer):
+    """Use nlopt as pytorch optimizer, with gradient supplied by pytorch autograd.
+    Note that this performs full minimization on each step,
+    so usually you would want to perform a single step, although performing multiple steps will refine the
+    solution.
+
+    Some algorithms are buggy with numpy>=2.
+
+    Args:
+        params: iterable of parameters to optimize or dicts defining parameter groups.
+        algorithm (int | _ALGOS_LITERAL): optimization algorithm from https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/
+        maxeval (int | None):
+            maximum allowed function evaluations, set to None to disable. But some stopping criterion
+            must be set otherwise nlopt will run forever.
+        lb (float | None, optional): optional lower bounds, some algorithms require this. Defaults to None.
+        ub (float | None, optional): optional upper bounds, some algorithms require this. Defaults to None.
+        stopval (float | None, optional): stop minimizing when an objective value ≤ stopval is found. Defaults to None.
+        ftol_rel (float | None, optional): set relative tolerance on function value. Defaults to None.
+        ftol_abs (float | None, optional): set absolute tolerance on function value. Defaults to None.
+        xtol_rel (float | None, optional): set relative tolerance on optimization parameters. Defaults to None.
+        xtol_abs (float | None, optional): set absolute tolerances on optimization parameters. Defaults to None.
+        maxtime (float | None, optional): stop when the optimization time (in seconds) exceeds maxtime. Defaults to None.
+    """
+    def __init__(
+        self,
+        params,
+        algorithm: int | _ALGOS_LITERAL,
+        maxeval: int | None,
+        lb: float | None = None,
+        ub: float | None = None,
+        stopval: float | None = None,
+        ftol_rel: float | None = None,
+        ftol_abs: float | None = None,
+        xtol_rel: float | None = None,
+        xtol_abs: float | None = None,
+        maxtime: float | None = None,
+    ):
+        defaults = dict(lb=lb, ub=ub)
+        super().__init__(params, defaults)
+
+        self.opt: nlopt.opt | None = None
+        if isinstance(algorithm, str): algorithm = getattr(nlopt, algorithm.upper())
+        self.algorithm: int = algorithm # type:ignore
+        self.algorithm_name: str | None = None
+
+        self.maxeval = maxeval; self.stopval = stopval
+        self.ftol_rel = ftol_rel; self.ftol_abs = ftol_abs
+        self.xtol_rel = xtol_rel; self.xtol_abs = xtol_abs
+        self.maxtime = maxtime
+
+        self._last_loss = None
+
+    def _f(self, x: np.ndarray, grad: np.ndarray, closure: _ClosureType, params: TensorList):
+        params.from_vec_(_ensure_tensor(x).to(params[0], copy=False))
+        if grad.size > 0:
+            with torch.enable_grad(): loss = closure()
+            self._last_loss = _ensure_float(loss)
+            grad[:] = params.ensure_grad_().grad.to_vec().reshape(grad.shape).detach().cpu().numpy()
+            return self._last_loss
+
+        self._last_loss = _ensure_float(closure(False))
+        return self._last_loss
+
+    @torch.no_grad
+    def step(self, closure: _ClosureType): # pylint: disable = signature-differs
+
+        params = self.get_params()
+
+        # make bounds
+        lb, ub = self.get_group_keys('lb', 'ub', cls=list)
+        lower = []
+        upper = []
+        for p, l, u in zip(params, lb, ub):
+            if l is None: l = -inf
+            if u is None: u = inf
+            lower.extend([l] * p.numel())
+            upper.extend([u] * p.numel())
+
+        x0 = params.to_vec().detach().cpu().numpy()
+
+        self.opt = nlopt.opt(self.algorithm, x0.size)
+        self.opt.set_min_objective(partial(self._f, closure = closure, params = params))
+        self.opt.set_lower_bounds(lower)
+        self.opt.set_upper_bounds(upper)
+
+        if self.maxeval is not None: self.opt.set_maxeval(self.maxeval)
+        if self.stopval is not None: self.opt.set_stopval(self.stopval)
+        if self.ftol_rel is not None: self.opt.set_ftol_rel(self.ftol_rel)
+        if self.ftol_abs is not None: self.opt.set_ftol_abs(self.ftol_abs)
+        if self.xtol_rel is not None: self.opt.set_xtol_rel(self.xtol_rel)
+        if self.xtol_abs is not None: self.opt.set_xtol_abs(self.xtol_abs)
+        if self.maxtime is not None: self.opt.set_maxtime(self.maxtime)
+
+        x = self.opt.optimize(x0)
+        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        return self._last_loss