bayesian-optimization 2.0.3__tar.gz → 3.0.0b1__tar.gz

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bayesian-optimization
-Version: 2.0.3
+Version: 3.0.0b1
 Summary: Bayesian Optimization package
 License: MIT
 Author: Fernando Nogueira
@@ -207,3 +207,16 @@ For constrained optimization:
 }
 ```
 
+For optimization over non-float parameters:
+```
+@article{garrido2020dealing,
+  title={Dealing with categorical and integer-valued variables in bayesian optimization with gaussian processes},
+  author={Garrido-Merch{\'a}n, Eduardo C and Hern{\'a}ndez-Lobato, Daniel},
+  journal={Neurocomputing},
+  volume={380},
+  pages={20--35},
+  year={2020},
+  publisher={Elsevier}
+}
+```
+

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/README.md

@@ -185,3 +185,16 @@ For constrained optimization:
     year={2014}
 }
 ```
+
+For optimization over non-float parameters:
+```
+@article{garrido2020dealing,
+  title={Dealing with categorical and integer-valued variables in bayesian optimization with gaussian processes},
+  author={Garrido-Merch{\'a}n, Eduardo C and Hern{\'a}ndez-Lobato, Daniel},
+  journal={Neurocomputing},
+  volume={380},
+  pages={20--35},
+  year={2020},
+  publisher={Elsevier}
+}
+```

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/bayes_opt/acquisition.py

@@ -127,7 +127,7 @@ class AcquisitionFunction(abc.ABC):
             self._fit_gp(gp=gp, target_space=target_space)
 
         acq = self._get_acq(gp=gp, constraint=target_space.constraint)
-        return self._acq_min(acq, target_space.bounds, n_random=n_random, n_l_bfgs_b=n_l_bfgs_b)
+        return self._acq_min(acq, target_space, n_random=n_random, n_l_bfgs_b=n_l_bfgs_b)
 
     def _get_acq(
         self, gp: GaussianProcessRegressor, constraint: ConstraintModel | None = None
@@ -182,7 +182,7 @@ class AcquisitionFunction(abc.ABC):
     def _acq_min(
         self,
         acq: Callable[[NDArray[Float]], NDArray[Float]],
-        bounds: NDArray[Float],
+        space: TargetSpace,
         n_random: int = 10_000,
         n_l_bfgs_b: int = 10,
     ) -> NDArray[Float]:
@@ -197,10 +197,8 @@ class AcquisitionFunction(abc.ABC):
         acq : Callable
             Acquisition function to use. Should accept an array of parameters `x`.
 
-        bounds : np.ndarray
-            Bounds of the search space. For `N` parameters this has shape
-            `(N, 2)` with `[i, 0]` the lower bound of parameter `i` and
-            `[i, 1]` the upper bound.
+        space : TargetSpace
+            The target space over which to optimize.
 
         n_random : int
             Number of random samples to use.
@@ -217,15 +215,22 @@ class AcquisitionFunction(abc.ABC):
         if n_random == 0 and n_l_bfgs_b == 0:
             error_msg = "Either n_random or n_l_bfgs_b needs to be greater than 0."
             raise ValueError(error_msg)
-        x_min_r, min_acq_r = self._random_sample_minimize(acq, bounds, n_random=n_random)
-        x_min_l, min_acq_l = self._l_bfgs_b_minimize(acq, bounds, n_x_seeds=n_l_bfgs_b)
-        # Either n_random or n_l_bfgs_b is not 0 => at least one of x_min_r and x_min_l is not None
-        if min_acq_r < min_acq_l:
-            return x_min_r
-        return x_min_l
+        x_min_r, min_acq_r, x_seeds = self._random_sample_minimize(
+            acq, space, n_random=max(n_random, n_l_bfgs_b), n_x_seeds=n_l_bfgs_b
+        )
+        if n_l_bfgs_b:
+            x_min_l, min_acq_l = self._l_bfgs_b_minimize(acq, space, x_seeds=x_seeds)
+            # Either n_random or n_l_bfgs_b is not 0 => at least one of x_min_r and x_min_l is not None
+            if min_acq_r > min_acq_l:
+                return x_min_l
+        return x_min_r
 
     def _random_sample_minimize(
-        self, acq: Callable[[NDArray[Float]], NDArray[Float]], bounds: NDArray[Float], n_random: int
+        self,
+        acq: Callable[[NDArray[Float]], NDArray[Float]],
+        space: TargetSpace,
+        n_random: int,
+        n_x_seeds: int = 0,
     ) -> tuple[NDArray[Float] | None, float]:
         """Random search to find the minimum of `acq` function.
 
@@ -234,14 +239,14 @@ class AcquisitionFunction(abc.ABC):
         acq : Callable
             Acquisition function to use. Should accept an array of parameters `x`.
 
-        bounds : np.ndarray
-            Bounds of the search space. For `N` parameters this has shape
-            `(N, 2)` with `[i, 0]` the lower bound of parameter `i` and
-            `[i, 1]` the upper bound.
+        space : TargetSpace
+            The target space over which to optimize.
 
         n_random : int
             Number of random samples to use.
 
+        n_x_seeds : int
+            Number of top points to return, for use as starting points for L-BFGS-B.
         Returns
         -------
         x_min : np.ndarray
@@ -252,14 +257,22 @@ class AcquisitionFunction(abc.ABC):
         """
         if n_random == 0:
             return None, np.inf
-        x_tries = self.random_state.uniform(bounds[:, 0], bounds[:, 1], size=(n_random, bounds.shape[0]))
+        x_tries = space.random_sample(n_random, random_state=self.random_state)
         ys = acq(x_tries)
         x_min = x_tries[ys.argmin()]
         min_acq = ys.min()
-        return x_min, min_acq
+        if n_x_seeds != 0:
+            idxs = np.argsort(ys)[-n_x_seeds:]
+            x_seeds = x_tries[idxs]
+        else:
+            x_seeds = []
+        return x_min, min_acq, x_seeds
 
     def _l_bfgs_b_minimize(
-        self, acq: Callable[[NDArray[Float]], NDArray[Float]], bounds: NDArray[Float], n_x_seeds: int = 10
+        self,
+        acq: Callable[[NDArray[Float]], NDArray[Float]],
+        space: TargetSpace,
+        x_seeds: NDArray[Float] | None = None,
     ) -> tuple[NDArray[Float] | None, float]:
         """Random search to find the minimum of `acq` function.
 
@@ -268,13 +281,11 @@ class AcquisitionFunction(abc.ABC):
         acq : Callable
             Acquisition function to use. Should accept an array of parameters `x`.
 
-        bounds : np.ndarray
-            Bounds of the search space. For `N` parameters this has shape
-            `(N, 2)` with `[i, 0]` the lower bound of parameter `i` and
-            `[i, 1]` the upper bound.
+        space : TargetSpace
+            The target space over which to optimize.
 
-        n_x_seeds : int
-            Number of starting points for the L-BFGS-B optimizer.
+        x_seeds : int
+            Starting points for the L-BFGS-B optimizer.
 
         Returns
         -------
@@ -284,33 +295,44 @@ class AcquisitionFunction(abc.ABC):
         min_acq : float
             Acquisition function value at `x_min`
         """
-        if n_x_seeds == 0:
-            return None, np.inf
-        x_seeds = self.random_state.uniform(bounds[:, 0], bounds[:, 1], size=(n_x_seeds, bounds.shape[0]))
+        continuous_dimensions = space.continuous_dimensions
+        continuous_bounds = space.bounds[continuous_dimensions]
+
+        if not continuous_dimensions.any():
+            min_acq = np.inf
+            x_min = np.array([np.nan] * space.bounds.shape[0])
+            return x_min, min_acq
 
         min_acq: float | None = None
         x_try: NDArray[Float]
         x_min: NDArray[Float]
         for x_try in x_seeds:
-            # Find the minimum of minus the acquisition function
-            res: OptimizeResult = minimize(acq, x_try, bounds=bounds, method="L-BFGS-B")
 
+            def continuous_acq(x: NDArray[Float], x_try=x_try) -> NDArray[Float]:
+                x_try[continuous_dimensions] = x
+                return acq(x_try)
+
+            # Find the minimum of minus the acquisition function
+            res: OptimizeResult = minimize(
+                continuous_acq, x_try[continuous_dimensions], bounds=continuous_bounds, method="L-BFGS-B"
+            )
             # See if success
             if not res.success:
                 continue
 
             # Store it if better than previous minimum(maximum).
             if min_acq is None or np.squeeze(res.fun) >= min_acq:
-                x_min = res.x
+                x_try[continuous_dimensions] = res.x
+                x_min = x_try
                 min_acq = np.squeeze(res.fun)
 
         if min_acq is None:
             min_acq = np.inf
-            x_min = np.array([np.nan] * bounds.shape[0])
+            x_min = np.array([np.nan] * space.bounds.shape[0])
 
         # Clip output to make sure it lies within the bounds. Due to floating
         # point technicalities this is not always the case.
-        return np.clip(x_min, bounds[:, 0], bounds[:, 1]), min_acq
+        return np.clip(x_min, space.bounds[:, 0], space.bounds[:, 1]), min_acq
 
 
 class UpperConfidenceBound(AcquisitionFunction):

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/bayes_opt/bayesian_optimization.py

@@ -16,13 +16,15 @@ from sklearn.gaussian_process.kernels import Matern
 
 from bayes_opt import acquisition
 from bayes_opt.constraint import ConstraintModel
+from bayes_opt.domain_reduction import DomainTransformer
 from bayes_opt.event import DEFAULT_EVENTS, Events
 from bayes_opt.logger import _get_default_logger
+from bayes_opt.parameter import wrap_kernel
 from bayes_opt.target_space import TargetSpace
 from bayes_opt.util import ensure_rng
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable, Mapping, Sequence
+    from collections.abc import Callable, Iterable, Mapping
 
     from numpy.random import RandomState
     from numpy.typing import NDArray
@@ -31,6 +33,7 @@ if TYPE_CHECKING:
     from bayes_opt.acquisition import AcquisitionFunction
     from bayes_opt.constraint import ConstraintModel
     from bayes_opt.domain_reduction import DomainTransformer
+    from bayes_opt.parameter import BoundsMapping, ParamsType
 
     Float = np.floating[Any]
 
@@ -114,7 +117,7 @@ class BayesianOptimization(Observable):
     ):
         self._random_state = ensure_rng(random_state)
         self._allow_duplicate_points = allow_duplicate_points
-        self._queue: deque[Mapping[str, float] | Sequence[float] | NDArray[Float]] = deque()
+        self._queue: deque[ParamsType] = deque()
 
         if acquisition_function is None:
             if constraint is None:
@@ -128,15 +131,6 @@ class BayesianOptimization(Observable):
         else:
             self._acquisition_function = acquisition_function
 
-        # Internal GP regressor
-        self._gp = GaussianProcessRegressor(
-            kernel=Matern(nu=2.5),
-            alpha=1e-6,
-            normalize_y=True,
-            n_restarts_optimizer=5,
-            random_state=self._random_state,
-        )
-
         if constraint is None:
             # Data structure containing the function to be optimized, the
             # bounds of its domain, and a record of the evaluations we have
@@ -158,14 +152,22 @@ class BayesianOptimization(Observable):
             )
             self.is_constrained = True
 
+        # Internal GP regressor
+        self._gp = GaussianProcessRegressor(
+            kernel=wrap_kernel(Matern(nu=2.5), transform=self._space.kernel_transform),
+            alpha=1e-6,
+            normalize_y=True,
+            n_restarts_optimizer=5,
+            random_state=self._random_state,
+        )
+
         self._verbose = verbose
         self._bounds_transformer = bounds_transformer
         if self._bounds_transformer:
-            try:
-                self._bounds_transformer.initialize(self._space)
-            except (AttributeError, TypeError) as exc:
-                error_msg = "The transformer must be an instance of DomainTransformer"
-                raise TypeError(error_msg) from exc
+            if not isinstance(self._bounds_transformer, DomainTransformer):
+                msg = "The transformer must be an instance of DomainTransformer"
+                raise TypeError(msg)
+            self._bounds_transformer.initialize(self._space)
 
         self._sorting_warning_already_shown = False  # TODO: remove in future version
         super().__init__(events=DEFAULT_EVENTS)
@@ -204,10 +206,7 @@ class BayesianOptimization(Observable):
         return self._space.res()
 
     def register(
-        self,
-        params: Mapping[str, float] | Sequence[float] | NDArray[Float],
-        target: float,
-        constraint_value: float | NDArray[Float] | None = None,
+        self, params: ParamsType, target: float, constraint_value: float | NDArray[Float] | None = None
     ) -> None:
         """Register an observation with known target.
 
@@ -225,10 +224,10 @@ class BayesianOptimization(Observable):
         # TODO: remove in future version
         if isinstance(params, np.ndarray) and not self._sorting_warning_already_shown:
             msg = (
-                "You're attempting to register an np.ndarray. Currently, the optimizer internally sorts"
-                " parameters by key and expects any registered array to respect this order. In future"
-                " versions this behaviour will change and the order as given by the pbounds dictionary"
-                " will be used. If you wish to retain sorted parameters, please manually sort your pbounds"
+                "You're attempting to register an np.ndarray. In previous versions, the optimizer internally"
+                " sorted parameters by key and expected any registered array to respect this order."
+                " In the current and any future version the order as given by the pbounds dictionary will be"
+                " used. If you wish to retain sorted parameters, please manually sort your pbounds"
                 " dictionary before constructing the optimizer."
             )
             warn(msg, stacklevel=1)
@@ -236,9 +235,7 @@ class BayesianOptimization(Observable):
         self._space.register(params, target, constraint_value)
         self.dispatch(Events.OPTIMIZATION_STEP)
 
-    def probe(
-        self, params: Mapping[str, float] | Sequence[float] | NDArray[Float], lazy: bool = True
-    ) -> None:
+    def probe(self, params: ParamsType, lazy: bool = True) -> None:
         """Evaluate the function at the given points.
 
         Useful to guide the optimizer.
@@ -255,10 +252,10 @@ class BayesianOptimization(Observable):
         # TODO: remove in future version
         if isinstance(params, np.ndarray) and not self._sorting_warning_already_shown:
             msg = (
-                "You're attempting to register an np.ndarray. Currently, the optimizer internally sorts"
-                " parameters by key and expects any registered array to respect this order. In future"
-                " versions this behaviour will change and the order as given by the pbounds dictionary"
-                " will be used. If you wish to retain sorted parameters, please manually sort your pbounds"
+                "You're attempting to register an np.ndarray. In previous versions, the optimizer internally"
+                " sorted parameters by key and expected any registered array to respect this order."
+                " In the current and any future version the order as given by the pbounds dictionary will be"
+                " used. If you wish to retain sorted parameters, please manually sort your pbounds"
                 " dictionary before constructing the optimizer."
             )
             warn(msg, stacklevel=1)
@@ -270,10 +267,10 @@ class BayesianOptimization(Observable):
             self._space.probe(params)
             self.dispatch(Events.OPTIMIZATION_STEP)
 
-    def suggest(self) -> dict[str, float]:
+    def suggest(self) -> dict[str, float | NDArray[Float]]:
         """Suggest a promising point to probe next."""
         if len(self._space) == 0:
-            return self._space.array_to_params(self._space.random_sample())
+            return self._space.array_to_params(self._space.random_sample(random_state=self._random_state))
 
         # Finding argmax of the acquisition function.
         suggestion = self._acquisition_function.suggest(gp=self._gp, target_space=self._space, fit_gp=True)
@@ -292,7 +289,7 @@ class BayesianOptimization(Observable):
             init_points = max(init_points, 1)
 
         for _ in range(init_points):
-            sample = self._space.random_sample()
+            sample = self._space.random_sample(random_state=self._random_state)
             self._queue.append(self._space.array_to_params(sample))
 
     def _prime_subscriptions(self) -> None:
@@ -344,7 +341,7 @@ class BayesianOptimization(Observable):
 
         self.dispatch(Events.OPTIMIZATION_END)
 
-    def set_bounds(self, new_bounds: Mapping[str, NDArray[Float] | Sequence[float]]) -> None:
+    def set_bounds(self, new_bounds: BoundsMapping) -> None:
         """Modify the bounds of the search space.
 
         Parameters
@@ -356,4 +353,6 @@ class BayesianOptimization(Observable):
 
     def set_gp_params(self, **params: Any) -> None:
         """Set parameters of the internal Gaussian Process Regressor."""
+        if "kernel" in params:
+            params["kernel"] = wrap_kernel(kernel=params["kernel"], transform=self._space.kernel_transform)
         self._gp.set_params(**params)

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/bayes_opt/constraint.py

@@ -9,6 +9,8 @@ from scipy.stats import norm
 from sklearn.gaussian_process import GaussianProcessRegressor
 from sklearn.gaussian_process.kernels import Matern
 
+from bayes_opt.parameter import wrap_kernel
+
 if TYPE_CHECKING:
     from collections.abc import Callable
 
@@ -55,6 +57,7 @@ class ConstraintModel:
         fun: Callable[..., float] | Callable[..., NDArray[Float]] | None,
         lb: float | NDArray[Float],
         ub: float | NDArray[Float],
+        transform: Callable[[Any], Any] | None = None,
         random_state: int | RandomState | None = None,
     ) -> None:
         self.fun = fun
@@ -68,7 +71,7 @@ class ConstraintModel:
 
         self._model = [
             GaussianProcessRegressor(
-                kernel=Matern(nu=2.5),
+                kernel=wrap_kernel(Matern(nu=2.5), transform) if transform is not None else Matern(nu=2.5),
                 alpha=1e-6,
                 normalize_y=True,
                 n_restarts_optimizer=5,

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/bayes_opt/domain_reduction.py

@@ -14,6 +14,7 @@ from warnings import warn
 
 import numpy as np
 
+from bayes_opt.parameter import FloatParameter
 from bayes_opt.target_space import TargetSpace
 
 if TYPE_CHECKING:
@@ -62,22 +63,19 @@ class SequentialDomainReductionTransformer(DomainTransformer):
 
     def __init__(
         self,
+        parameters: Iterable[str] | None = None,
         gamma_osc: float = 0.7,
         gamma_pan: float = 1.0,
         eta: float = 0.9,
         minimum_window: NDArray[Float] | Sequence[float] | Mapping[str, float] | float = 0.0,
     ) -> None:
+        # TODO: Ensure that this is only applied to continuous parameters
+        self.parameters = parameters
         self.gamma_osc = gamma_osc
         self.gamma_pan = gamma_pan
         self.eta = eta
 
-        self.minimum_window_value: NDArray[Float] | Sequence[float] | float
-        if isinstance(minimum_window, Mapping):
-            self.minimum_window_value = [
-                item[1] for item in sorted(minimum_window.items(), key=lambda x: x[0])
-            ]
-        else:
-            self.minimum_window_value = minimum_window
+        self.minimum_window_value = minimum_window
 
     def initialize(self, target_space: TargetSpace) -> None:
         """Initialize all of the parameters.
@@ -87,6 +85,15 @@ class SequentialDomainReductionTransformer(DomainTransformer):
         target_space : TargetSpace
             TargetSpace this DomainTransformer operates on.
         """
+        if isinstance(self.minimum_window_value, Mapping):
+            self.minimum_window_value = [self.minimum_window_value[key] for key in target_space.keys]
+        else:
+            self.minimum_window_value = self.minimum_window_value
+
+        any_not_float = any([not isinstance(p, FloatParameter) for p in target_space._params_config.values()])
+        if any_not_float:
+            msg = "Domain reduction is only supported for all-FloatParameter optimization."
+            raise ValueError(msg)
         # Set the original bounds
         self.original_bounds = np.copy(target_space.bounds)
         self.bounds = [self.original_bounds]

{bayesian_optimization-2.0.3 → bayesian_optimization-3.0.0b1}/bayes_opt/logger.py

@@ -127,19 +127,19 @@ class ScreenLogger(_Tracker):
         x_ = ("T" if x else "F") if self._default_cell_size < 5 else str(x)
         return f"{x_:<{self._default_cell_size}}"
 
-    def _format_key(self, key: str) -> str:
-        """Format a key.
+    def _format_str(self, str_: str) -> str:
+        """Format a str.
 
         Parameters
         ----------
-        key : string
+        str_ : str
             Value to format.
 
         Returns
         -------
         A stringified, formatted version of `x`.
         """
-        s = f"{key:^{self._default_cell_size}}"
+        s = f"{str_:^{self._default_cell_size}}"
         if len(s) > self._default_cell_size:
             return s[: self._default_cell_size - 3] + "..."
         return s
@@ -168,7 +168,10 @@ class ScreenLogger(_Tracker):
         if self._is_constrained:
             cells[2] = self._format_bool(res["allowed"])
         params = res.get("params", {})
-        cells[3:] = [self._format_number(params.get(key, float("nan"))) for key in keys]
+        cells[3:] = [
+            instance.space._params_config[key].to_string(val, self._default_cell_size)
+            for key, val in params.items()
+        ]
 
         return "| " + " | ".join(colour + x + self._colour_reset for x in cells if x is not None) + " |"
 
@@ -188,10 +191,10 @@ class ScreenLogger(_Tracker):
         # iter, target, allowed [, *params]
         cells: list[str | None] = [None] * (3 + len(keys))
 
-        cells[:2] = self._format_key("iter"), self._format_key("target")
+        cells[:2] = self._format_str("iter"), self._format_str("target")
         if self._is_constrained:
-            cells[2] = self._format_key("allowed")
-        cells[3:] = [self._format_key(key) for key in keys]
+            cells[2] = self._format_str("allowed")
+        cells[3:] = [self._format_str(key) for key in keys]
 
         line = "| " + " | ".join(x for x in cells if x is not None) + " |"
         self._header_length = len(line)