torchzero 0.3.14__py3-none-any.whl → 0.4.0__py3-none-any.whl

torchzero/optim/wrappers/directsearch.py

@@ -7,24 +7,13 @@ import numpy as np
 import torch
 from directsearch.ds import DEFAULT_PARAMS
 
-from ...utils import Optimizer, TensorList
-
-
-def _ensure_float(x):
-    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
-    if isinstance(x, np.ndarray): return x.item()
-    return float(x)
-
-def _ensure_numpy(x):
-    if isinstance(x, torch.Tensor): return x.detach().cpu()
-    if isinstance(x, np.ndarray): return x
-    return np.array(x)
-
+from ...utils import TensorList
+from .wrapper import WrapperBase
 
 Closure = Callable[[bool], Any]
 
 
-class DirectSearch(Optimizer):
+class DirectSearch(WrapperBase):
     """Use directsearch as pytorch optimizer.
 
     Note that this performs full minimization on each step,
@@ -96,28 +85,23 @@ class DirectSearch(Optimizer):
         del kwargs['self'], kwargs['params'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
-
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
         res = directsearch.solve(
-            partial(self._objective, params = params, closure = closure),
+            partial(self._f, params=params, closure=closure),
             x0 = x0,
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.f
 
 
 
-class DirectSearchDS(Optimizer):
+class DirectSearchDS(WrapperBase):
     def __init__(
         self,
         params,
@@ -139,26 +123,21 @@ class DirectSearchDS(Optimizer):
         del kwargs['self'], kwargs['params'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
-
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
         res = directsearch.solve_directsearch(
-            partial(self._objective, params = params, closure = closure),
+            partial(self._f, params = params, closure = closure),
             x0 = x0,
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.f
 
-class DirectSearchProbabilistic(Optimizer):
+class DirectSearchProbabilistic(WrapperBase):
     def __init__(
         self,
         params,
@@ -179,27 +158,22 @@ class DirectSearchProbabilistic(Optimizer):
         del kwargs['self'], kwargs['params'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
-
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
         res = directsearch.solve_probabilistic_directsearch(
-            partial(self._objective, params = params, closure = closure),
+            partial(self._f, params = params, closure = closure),
             x0 = x0,
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.f
 
 
-class DirectSearchSubspace(Optimizer):
+class DirectSearchSubspace(WrapperBase):
     def __init__(
         self,
         params,
@@ -223,28 +197,23 @@ class DirectSearchSubspace(Optimizer):
         del kwargs['self'], kwargs['params'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
-
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
         res = directsearch.solve_subspace_directsearch(
-            partial(self._objective, params = params, closure = closure),
+            partial(self._f, params = params, closure = closure),
             x0 = x0,
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.f
 
 
 
-class DirectSearchSTP(Optimizer):
+class DirectSearchSTP(WrapperBase):
     def __init__(
         self,
         params,
@@ -260,21 +229,16 @@ class DirectSearchSTP(Optimizer):
         del kwargs['self'], kwargs['params'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
-
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
         res = directsearch.solve_stp(
-            partial(self._objective, params = params, closure = closure),
+            partial(self._f, params = params, closure = closure),
             x0 = x0,
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.f

torchzero/optim/wrappers/fcmaes.py

@@ -9,20 +9,15 @@ import fcmaes
 import fcmaes.optimizer
 import fcmaes.retry
 
-from ...utils import Optimizer, TensorList
+from ...utils import TensorList
+from .wrapper import WrapperBase
 
 Closure = Callable[[bool], Any]
 
-
-def _ensure_float(x) -> float:
-    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
-    if isinstance(x, np.ndarray): return float(x.item())
-    return float(x)
-
 def silence_fcmaes():
     fcmaes.retry.logger.disable('fcmaes')
 
-class FcmaesWrapper(Optimizer):
+class FcmaesWrapper(WrapperBase):
     """Use fcmaes as pytorch optimizer. Particularly fcmaes has BITEOPT which appears to win in many benchmarks.
 
     Note that this performs full minimization on each step, so only perform one step with this.
@@ -42,7 +37,7 @@ class FcmaesWrapper(Optimizer):
             CMA-ES population size used for all CMA-ES runs.
             Not used for differential evolution.
             Ignored if parameter optimizer is defined. Defaults to 31.
-        capacity (int | None, optional): capacity of the evaluation store.. Defaults to 500.
+        capacity (int | None, optional): capacity of the evaluation store. Defaults to 500.
         stop_fitness (float | None, optional):
             Limit for fitness value. optimization runs terminate if this value is reached. Defaults to -np.inf.
         statistic_num (int | None, optional):
@@ -61,46 +56,30 @@ class FcmaesWrapper(Optimizer):
         popsize: int | None = 31,
         capacity: int | None = 500,
         stop_fitness: float | None = -np.inf,
-        statistic_num: int | None = 0
+        statistic_num: int | None = 0,
+        silence: bool = True,
     ):
-        super().__init__(params, lb=lb, ub=ub)
-        silence_fcmaes()
+        super().__init__(params, dict(lb=lb,ub=ub))
+        if silence:
+            silence_fcmaes()
         kwargs = locals().copy()
-        del kwargs['self'], kwargs['params'], kwargs['lb'], kwargs['ub'], kwargs['__class__']
+        del kwargs['self'], kwargs['params'], kwargs['lb'], kwargs['ub'], kwargs['__class__'], kwargs["silence"]
         self._kwargs = kwargs
         self._kwargs['workers'] = 1
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure) -> float:
-        if self.raised: return np.inf
-        try:
-            params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-            return _ensure_float(closure(False))
-        except Exception as e:
-            # ha ha, I found a way to make exceptions work in fcmaes and scipy direct
-            self.e = e
-            self.raised = True
-            return np.inf
 
     @torch.no_grad
     def step(self, closure: Closure):
-        self.raised = False
-        self.e = None
 
-        params = self.get_params()
-
-        lb, ub = self.group_vals('lb', 'ub', cls=list)
-        bounds = []
-        for p, l, u in zip(params, lb, ub):
-            bounds.extend([[l, u]] * p.numel())
+        params = TensorList(self._get_params())
+        bounds = self._get_bounds()
 
         res = fcmaes.retry.minimize(
-            partial(self._objective, params=params, closure=closure), # pyright:ignore[reportArgumentType]
+            partial(self._f, params=params, closure=closure), # pyright:ignore[reportArgumentType]
             bounds=bounds, # pyright:ignore[reportArgumentType]
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(res.x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-
-        if self.e is not None: raise self.e from None
+        params.from_vec_(torch.as_tensor(res.x, device = params[0].device, dtype=params[0].dtype))
         return res.fun
 

torchzero/optim/wrappers/mads.py

@@ -6,24 +6,13 @@ import numpy as np
 import torch
 from mads.mads import orthomads
 
-from ...utils import Optimizer, TensorList
-
-
-def _ensure_float(x):
-    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
-    if isinstance(x, np.ndarray): return x.item()
-    return float(x)
-
-def _ensure_numpy(x):
-    if isinstance(x, torch.Tensor): return x.detach().cpu()
-    if isinstance(x, np.ndarray): return x
-    return np.array(x)
-
+from ...utils import TensorList
+from .wrapper import WrapperBase
 
 Closure = Callable[[bool], Any]
 
 
-class MADS(Optimizer):
+class MADS(WrapperBase):
     """Use mads.orthomads as pytorch optimizer.
 
     Note that this performs full minimization on each step,
@@ -53,37 +42,28 @@ class MADS(Optimizer):
         displog = False,
         savelog = False,
     ):
-        super().__init__(params, lb=lb, ub=ub)
+        super().__init__(params, dict(lb=lb, ub=ub))
 
         kwargs = locals().copy()
         del kwargs['self'], kwargs['params'], kwargs['lb'], kwargs['ub'], kwargs['__class__']
         self._kwargs = kwargs
 
-    def _objective(self, x: np.ndarray, params: TensorList, closure):
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
-        return _ensure_float(closure(False))
 
     @torch.no_grad
     def step(self, closure: Closure):
-        params = self.get_params()
-
-        x0 = params.to_vec().detach().cpu().numpy()
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
+        lb, ub = self._get_lb_ub()
 
-        lb, ub = self.group_vals('lb', 'ub', cls=list)
-        bounds_lower = []
-        bounds_upper = []
-        for p, l, u in zip(params, lb, ub):
-            bounds_lower.extend([l] * p.numel())
-            bounds_upper.extend([u] * p.numel())
 
         f, x = orthomads(
             design_variables=x0,
-            bounds_upper=np.asarray(bounds_upper),
-            bounds_lower=np.asarray(bounds_lower),
-            objective_function=partial(self._objective, params = params, closure = closure),
+            bounds_upper=np.asarray(ub),
+            bounds_lower=np.asarray(lb),
+            objective_function=partial(self._f, params=params, closure=closure),
             **self._kwargs
         )
 
-        params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        params.from_vec_(torch.as_tensor(x, device = params[0].device, dtype=params[0].dtype,))
         return f
 

torchzero/optim/wrappers/moors.py

@@ -0,0 +1,66 @@
+from collections.abc import Callable
+from functools import partial
+from typing import Any, Literal
+
+import numpy as np
+import torch
+
+from ...utils import TensorList
+from .wrapper import WrapperBase
+
+Closure = Callable[[bool], Any]
+
+class MoorsWrapper(WrapperBase):
+    """Use moo-rs (pymoors) is PyTorch optimizer.
+
+    Note that this performs full minimization on each step,
+    so usually you would want to perform a single step.
+
+    To use this, define a function that accepts fitness function and number of variables and returns a pymoors algorithm:
+
+    ```python
+    alg_fn = lambda fitness_fn, num_vars: pymoors.Nsga2(
+        fitness_fn=fitness_fn,
+        num_vars=num_vars,
+        num_iterations=100,
+        sampler = pymoors.RandomSamplingFloat(min=-3, max=3),
+        crossover = pymoors.SinglePointBinaryCrossover(),
+        mutation = pymoors.GaussianMutation(gene_mutation_rate=1e-2, sigma=0.1),
+        population_size = 32,
+        num_offsprings = 32,
+    )
+
+    optimizer = MoorsWrapper(model.parameters(), alg_fn)
+    ```
+
+    All algorithms in pymoors have slightly different APIs, refer to their docs.
+
+    """
+    def __init__(
+        self,
+        params,
+        algorithm_fn: Callable[[Callable[[np.ndarray], np.ndarray], int], Any]
+    ):
+        super().__init__(params, {})
+        self._algorithm_fn = algorithm_fn
+
+    def _objective(self, x: np.ndarray, params, closure):
+        fs = []
+        for x_i in x:
+            f_i = self._fs(x_i, params=params, closure=closure)
+            fs.append(f_i)
+        return np.stack(fs, dtype=np.float64) # pymoors needs float64
+
+    @torch.no_grad
+    def step(self, closure: Closure):
+        params = TensorList(self._get_params())
+        objective = partial(self._objective, params=params, closure=closure)
+
+        algorithm = self._algorithm_fn(objective, params.global_numel())
+
+        algorithm.run()
+        pop = algorithm.population
+
+        params.from_vec_(torch.as_tensor(pop.best[0].genes, device = params[0].device, dtype=params[0].dtype,))
+        return pop.best[0].fitness
+

torchzero/optim/wrappers/nevergrad.py

@@ -6,7 +6,7 @@ import torch
 
 import nevergrad as ng
 
-from ...utils import Optimizer
+from .wrapper import WrapperBase
 
 
 def _ensure_float(x) -> float:
@@ -14,7 +14,7 @@ def _ensure_float(x) -> float:
     if isinstance(x, np.ndarray): return float(x.item())
     return float(x)
 
-class NevergradWrapper(Optimizer):
+class NevergradWrapper(WrapperBase):
     """Use nevergrad optimizer as pytorch optimizer.
     Note that it is recommended to specify `budget` to the number of iterations you expect to run,
     as some nevergrad optimizers will error without it.
@@ -72,7 +72,7 @@ class NevergradWrapper(Optimizer):
 
     @torch.no_grad
     def step(self, closure): # pylint:disable=signature-differs # pyright:ignore[reportIncompatibleMethodOverride]
-        params = self.get_params()
+        params = self._get_params()
         if self.opt is None:
             ng_params = []
             for group in self.param_groups:
@@ -95,7 +95,7 @@ class NevergradWrapper(Optimizer):
 
         x: ng.p.Tuple = self.opt.ask() # type:ignore
         for cur, new in zip(params, x):
-            cur.set_(torch.from_numpy(new.value).to(dtype=cur.dtype, device=cur.device, copy=False).reshape_as(cur)) # type:ignore
+            cur.set_(torch.as_tensor(new.value, dtype=cur.dtype, device=cur.device).reshape_as(cur)) # type:ignore
 
         loss = closure(False)
         self.opt.tell(x, _ensure_float(loss))

torchzero/optim/wrappers/nlopt.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Literal, Any
 from collections.abc import Mapping, Callable
 from functools import partial
@@ -5,7 +6,8 @@ import numpy as np
 import torch
 
 import nlopt
-from ...utils import Optimizer, TensorList
+from ...utils import TensorList
+from .wrapper import WrapperBase
 
 _ALGOS_LITERAL = Literal[
     "GN_DIRECT",  # = _nlopt.GN_DIRECT
@@ -69,14 +71,14 @@ def _ensure_tensor(x):
 inf = float('inf')
 Closure = Callable[[bool], Any]
 
-class NLOptWrapper(Optimizer):
+class NLOptWrapper(WrapperBase):
     """Use nlopt as pytorch optimizer, with gradient supplied by pytorch autograd.
     Note that this performs full minimization on each step,
     so usually you would want to perform a single step, although performing multiple steps will refine the
     solution.
 
     Args:
-        params: iterable of parameters to optimize or dicts defining parameter groups.
+        params (Iterable): iterable of parameters to optimize or dicts defining parameter groups.
         algorithm (int | _ALGOS_LITERAL): optimization algorithm from https://nlopt.readthedocs.io/en/latest/NLopt_Algorithms/
         maxeval (int | None):
             maximum allowed function evaluations, set to None to disable. But some stopping criterion
@@ -96,21 +98,30 @@ class NLOptWrapper(Optimizer):
         algorithm: int | _ALGOS_LITERAL,
         lb: float | None = None,
         ub: float | None = None,
-        maxeval: int | None = 10000, # None can stall on some algos and because they are threaded C you can't even interrupt them
+        maxeval: int | None = None, # None can stall on some algos and because they are threaded C you can't even interrupt them
         stopval: float | None = None,
         ftol_rel: float | None = None,
         ftol_abs: float | None = None,
         xtol_rel: float | None = None,
         xtol_abs: float | None = None,
         maxtime: float | None = None,
+        require_criterion: bool = True,
     ):
+        if require_criterion:
+            if all(i is None for i in (maxeval, stopval, ftol_abs, ftol_rel, xtol_abs, xtol_rel)):
+                raise RuntimeError(
+                    "Specify at least one stopping criterion out of "
+                    "(maxeval, stopval, ftol_rel, ftol_abs, xtol_rel, xtol_abs, maxtime). "
+                    "Pass `require_criterion=False` to suppress this error."
+                )
+
         defaults = dict(lb=lb, ub=ub)
         super().__init__(params, defaults)
 
         self.opt: nlopt.opt | None = None
+        self.algorithm_name: str | int = algorithm
         if isinstance(algorithm, str): algorithm = getattr(nlopt, algorithm.upper())
         self.algorithm: int = algorithm # type:ignore
-        self.algorithm_name: str | None = None
 
         self.maxeval = maxeval; self.stopval = stopval
         self.ftol_rel = ftol_rel; self.ftol_abs = ftol_abs
@@ -119,7 +130,7 @@ class NLOptWrapper(Optimizer):
 
         self._last_loss = None
 
-    def _f(self, x: np.ndarray, grad: np.ndarray, closure, params: TensorList):
+    def _objective(self, x: np.ndarray, grad: np.ndarray, closure, params: TensorList):
         if self.raised:
             if self.opt is not None: self.opt.force_stop()
             return np.inf
@@ -132,7 +143,7 @@ class NLOptWrapper(Optimizer):
             if grad.size > 0:
                 with torch.enable_grad(): loss = closure()
                 self._last_loss = _ensure_float(loss)
-                grad[:] = params.ensure_grad_().grad.to_vec().reshape(grad.shape).detach().cpu().numpy()
+                grad[:] = params.grad.fill_none_(reference=params).to_vec().reshape(grad.shape).numpy(force=True)
                 return self._last_loss
 
             self._last_loss = _ensure_float(closure(False))
@@ -147,25 +158,20 @@ class NLOptWrapper(Optimizer):
     def step(self, closure: Closure): # pylint: disable = signature-differs # pyright:ignore[reportIncompatibleMethodOverride]
         self.e = None
         self.raised = False
-        params = self.get_params()
-
-        # make bounds
-        lb, ub = self.group_vals('lb', 'ub', cls=list)
-        lower = []
-        upper = []
-        for p, l, u in zip(params, lb, ub):
-            if l is None: l = -inf
-            if u is None: u = inf
-            lower.extend([l] * p.numel())
-            upper.extend([u] * p.numel())
+        params = TensorList(self._get_params())
+        x0 = params.to_vec().numpy(force=True)
 
-        x0 = params.to_vec().detach().cpu().numpy().astype(np.float64)
+        plb, pub = self._get_per_parameter_lb_ub()
+        if all(i is None for i in plb) and all(i is None for i in pub):
+            lb = ub = None
+        else:
+            lb, ub = self._get_lb_ub(ld = {None: -np.inf}, ud = {None: np.inf})
 
         self.opt = nlopt.opt(self.algorithm, x0.size)
         self.opt.set_exceptions_enabled(False) # required
-        self.opt.set_min_objective(partial(self._f, closure = closure, params = params))
-        self.opt.set_lower_bounds(lower)
-        self.opt.set_upper_bounds(upper)
+        self.opt.set_min_objective(partial(self._objective, closure = closure, params = params))
+        if lb is not None: self.opt.set_lower_bounds(np.asarray(lb, dtype=x0.dtype))
+        if ub is not None: self.opt.set_upper_bounds(np.asarray(ub, dtype=x0.dtype))
 
         if self.maxeval is not None: self.opt.set_maxeval(self.maxeval)
         if self.stopval is not None: self.opt.set_stopval(self.stopval)
@@ -179,12 +185,12 @@ class NLOptWrapper(Optimizer):
         x = None
         try:
             x = self.opt.optimize(x0)
-        except SystemError:
-            pass
+        # except SystemError as s:
+        #     warnings.warn(f"{self.algorithm_name} raised {s}")
         except Exception as e:
             raise e from None
 
-        if x is not None: params.from_vec_(torch.from_numpy(x).to(device = params[0].device, dtype=params[0].dtype, copy=False))
+        if x is not None: params.from_vec_(torch.as_tensor(x, device = params[0].device, dtype=params[0].dtype))
         if self.e is not None: raise self.e from None
 
         if self._last_loss is None or x is None: return closure(False)

torchzero/optim/wrappers/optuna.py

@@ -1,23 +1,16 @@
-import typing
-from collections import abc
-
-import numpy as np
+import optuna
 import torch
 
-import optuna
+from ...utils import TensorList, tofloat, totensor
+from .wrapper import WrapperBase
 
-from ...utils import Optimizer, totensor, tofloat
 
 def silence_optuna():
     optuna.logging.set_verbosity(optuna.logging.WARNING)
 
-def _ensure_float(x) -> float:
-    if isinstance(x, torch.Tensor): return x.detach().cpu().item()
-    if isinstance(x, np.ndarray): return float(x.item())
-    return float(x)
 
 
-class OptunaSampler(Optimizer):
+class OptunaSampler(WrapperBase):
     """Optimize your next SOTA model using hyperparameter optimization.
 
     Note - optuna is surprisingly scalable to large number of parameters (up to 10,000), despite literally requiring a for-loop because it only supports scalars. Default TPESampler is good for BBO. Maybe not for NNs...
@@ -38,7 +31,7 @@ class OptunaSampler(Optimizer):
         silence: bool = True,
     ):
         if silence: silence_optuna()
-        super().__init__(params, lb=lb, ub=ub)
+        super().__init__(params, dict(lb=lb, ub=ub))
 
         if isinstance(sampler, type): sampler = sampler()
         self.sampler = sampler
@@ -47,7 +40,7 @@ class OptunaSampler(Optimizer):
     @torch.no_grad
     def step(self, closure):
 
-        params = self.get_params()
+        params = TensorList(self._get_params())
         if self.study is None:
             self.study = optuna.create_study(sampler=self.sampler)