hone-python 0.9.0__py3-none-any.whl

hone/optimizers/bayesian.py

@@ -0,0 +1,265 @@
+"""Bayesian optimization: model the objective with a Gaussian process, and evaluate where the model expects the most improvement.
+
+Each candidate costs a model fit and a search over the model, a fraction of a second up to a few
+hundred evaluations, so Bayesian optimization suits objectives that take far longer than that:
+simulations, experiments, training runs. It makes the most of a small budget.
+
+Range parameters are scaled to [0, 1], on a log scale where they have one; integers are searched
+as numbers and rounded. Each choice
+parameter becomes one column per choice, holding 1 for the chosen one and 0 for the rest.
+"""
+import math
+from typing import Any, List
+
+import numpy as np
+from scipy.optimize import minimize
+from scipy.special import erfcx, log_ndtr
+from threadpoolctl import threadpool_limits
+
+from hone.config import BayesianConfig
+from hone.optimizers._gp import GaussianProcess
+from hone.optimizers.base import Individual, Optimizer, UnsupportedSearchSpace
+from hone.parameters import ChoiceParameter, Parameter, RangeParameter, SearchSpace
+
+GLOBAL_CANDIDATES = 1000  # random candidates per search of the model
+LOCAL_CANDIDATES = 500  # candidates near the best points found so far
+REFINED = 5  # the best candidates are refined by gradient search
+REFIT_EVERY = 5  # past FIT_EVERY_UNTIL observations, refit the model's hyperparameters only this often
+FIT_EVERY_UNTIL = 50
+
+
+class BayesianOptimizer(Optimizer):
+    """Bayesian optimization with a Gaussian process and expected improvement.
+
+    The first generation is spread over the search space (with any initial points). After that,
+    each candidate is the point where a Gaussian process model of the objective predicts the
+    largest expected improvement on the best value so far. Failed evaluations are modelled as
+    the worst value seen, which steers the search away from them.
+
+    Range parameters, integer or not, and choice parameters are supported, but not choice
+    parameters with choose_n. Fixed parameters are passed through.
+
+    Args:
+        search_space: The parameters to optimize over, as a SearchSpace or a list of parameters.
+        config: A BayesianConfig. Defaults are used if omitted.
+        initial_points: Points to evaluate first: one dict of parameter values, or a list of them.
+    """
+    config_class = BayesianConfig
+    config: BayesianConfig
+
+    def __init__(
+        self,
+        search_space: SearchSpace | List[Parameter],
+        config: BayesianConfig | None = None,
+        *,
+        initial_points: dict[str, Any] | list[dict[str, Any]] | None = None,
+    ) -> None:
+        if not isinstance(search_space, SearchSpace):
+            search_space = SearchSpace(search_space)
+        self._encoding = _Encoding(search_space)
+        super().__init__(search_space, config, initial_points=initial_points)
+
+    def reset(self) -> None:
+        super().reset()
+        self.batch_size = self.config.population_size or self._workers
+        self._X: list[np.ndarray] = []  # every evaluated point, encoded
+        self._y: list[float | None] = []  # its value to minimize, or None if it failed
+        self._theta: np.ndarray | None = None  # the model's hyperparameters, kept between fits
+        self._fitted_at = 0  # observations at the last hyperparameter fit
+
+    def _next_population(self, previous: list[Individual] | None, generation: int) -> list[Individual]:
+        if previous is None:
+            initial = self._initial_individuals(generation)
+            count = self.config.initial_samples or 2 * (self._encoding.variables + 1)
+            taken = {tuple(self._encoding.encode(ind.params)) for ind in initial}
+            spread = []
+            for x in self._encoding.spread(self.rng, max(count - len(initial), 0)):
+                # Rounding integers can land two points on the same values; try others instead.
+                for _ in range(100):
+                    if tuple(x) not in taken:
+                        break
+                    x = self._encoding.spread(self.rng, 1)[0]
+                taken.add(tuple(x))
+                spread.append(x)
+            return initial + [Individual(params=self._encoding.decode(x), generation=generation) for x in spread]
+        for individual in previous:
+            if individual.fitness is not None:
+                self._X.append(self._encoding.encode(individual.params))
+                value = individual.fitness if not self.config.maximize else -individual.fitness
+                self._y.append(None if individual.failed else value)
+        # The model's matrices are small, and splitting their arithmetic across threads makes it many
+        # times slower. One thread also makes results the same on every machine.
+        with threadpool_limits(limits=1, user_api="blas"):
+            proposals = self._propose(self.batch_size)
+        return [Individual(params=self._encoding.decode(x), generation=generation) for x in proposals]
+
+    # --- choosing candidates --------------------------------------------------
+
+    def _propose(self, count: int) -> list[np.ndarray]:
+        succeeded = [y for y in self._y if y is not None]
+        if len(succeeded) < 2:
+            return list(self._encoding.spread(self.rng, count))
+        worst = max(succeeded)
+        X = np.array(self._X)
+        model = GaussianProcess(X, np.array([worst if y is None else y for y in self._y]), self._theta)
+        if self._theta is None or len(X) <= FIT_EVERY_UNTIL or len(X) - self._fitted_at >= REFIT_EVERY:
+            model.fit(restarts=2 if self._theta is None else 0, rng=self.rng)
+            self._theta, self._fitted_at = model.theta, len(X)
+
+        # The best value so far. With noise, the best observation is likely a lucky one, so use the
+        # lowest the model predicts at any evaluated point instead.
+        best = float(model.predict(X)[0].min()) if self.config.noise else float(model.z.min())
+        # Candidates still being evaluated are assumed to come out as predicted, so none is proposed again.
+        for individual in self._outstanding():
+            x = self._encoding.encode(individual.params)
+            model = model.with_observation(x, float(model.predict(x)[0][0]))
+        chosen = []
+        for _ in range(count):
+            x = self._maximize_improvement(model, best)
+            chosen.append(x)
+            # Pretend the model's prediction came true, so the next candidate of the batch goes elsewhere.
+            mean, _ = model.predict(x)
+            model = model.with_observation(x, float(mean[0]))
+        return chosen
+
+    def _maximize_improvement(self, model: GaussianProcess, best: float) -> np.ndarray:
+        encoding = self._encoding
+        leaders = model.X[np.argsort(model.z)[:5]]
+        local = leaders[self.rng.integers(len(leaders), size=LOCAL_CANDIDATES)]
+        scales = self.rng.choice([0.01, 0.05, 0.2], size=(LOCAL_CANDIDATES, 1))
+        local = encoding.perturb(self.rng, local, scales)
+        candidates = encoding.snap(np.vstack([encoding.spread(self.rng, GLOBAL_CANDIDATES), local]))
+        scores = _log_expected_improvement(model, candidates, best)
+
+        columns = encoding.numeric
+        if columns.size:
+            starts = candidates[np.argsort(-scores)[:REFINED]]
+            refined = encoding.snap(_refine(model, starts, columns, best))
+            candidates = np.vstack([candidates, refined])
+            scores = np.append(scores, _log_expected_improvement(model, refined, best))
+
+        seen = model.X
+        for index in np.argsort(-scores):
+            if not np.any(np.all(np.abs(seen - candidates[index]) < 1e-9, axis=1)):
+                return candidates[index]
+        return encoding.spread(self.rng, 1)[0]  # every candidate was already evaluated
+
+
+def _refine(model: GaussianProcess, starts: np.ndarray, columns: np.ndarray, best: float) -> np.ndarray:
+    """Each start moved uphill in expected improvement by gradient search over the numeric columns.
+
+    The gradient comes from finite differences, all taken in one vectorized call to the model.
+    """
+    step = 1e-6
+    refined = starts.copy()
+    for row, start in enumerate(starts):
+
+        def negative_and_gradient(u: np.ndarray, start=start) -> tuple[float, np.ndarray]:
+            X = np.repeat(start[None, :], len(columns) + 1, axis=0)
+            X[:, columns] = u
+            X[np.arange(1, len(columns) + 1), columns] += step
+            values = _log_expected_improvement(model, X, best)
+            return -values[0], -(values[1:] - values[0]) / step
+
+        found = minimize(negative_and_gradient, start[columns], jac=True, method="L-BFGS-B", bounds=[(0, 1 - step)] * len(columns))
+        if found.fun <= -_log_expected_improvement(model, start[None, :], best)[0]:
+            refined[row, columns] = found.x
+    return refined
+
+
+def _log_expected_improvement(model: GaussianProcess, X: np.ndarray, best: float) -> np.ndarray:
+    """log E[max(best - f(x), 0)] for each row of X, computed stably even where it is tiny."""
+    mean, sd = model.predict(X)
+    u = (best - mean) / sd
+    log_h = np.empty_like(u)
+    # h(u) = phi(u) + u Phi(u), the expected improvement of a standard normal past -u.
+    upper = u > -1
+    log_h[upper] = np.log(np.exp(-0.5 * u[upper] ** 2) / math.sqrt(2 * math.pi) + u[upper] * np.exp(log_ndtr(u[upper])))
+    lower = np.maximum(u[~upper], -40.0)
+    # For u <= -1: h(u) = phi(u) (1 + u Phi(u) / phi(u)), where Phi(u) / phi(u) = sqrt(pi/2) erfcx(-u / sqrt 2).
+    log_h[~upper] = -0.5 * lower**2 - 0.5 * math.log(2 * math.pi) + np.log1p(
+        lower * math.sqrt(math.pi / 2) * erfcx(-lower / math.sqrt(2))
+    )
+    return log_h + np.log(sd)
+
+
+class _Encoding:
+    """Between parameter values and points in [0, 1]^columns, where the model works."""
+
+    def __init__(self, search_space: SearchSpace) -> None:
+        self.search_space = search_space
+        self._blocks: list[tuple[RangeParameter | ChoiceParameter, slice]] = []
+        self._constant: dict[str, Any] = {}
+        column = 0
+        numeric = []
+        for p in search_space.parameters:
+            if isinstance(p, ChoiceParameter) and p.choose_n is not None:
+                raise UnsupportedSearchSpace(
+                    f"Bayesian optimization doesn't support choose_n choices ({p.name}); use GeneticOptimizer"
+                )
+            if isinstance(p, RangeParameter) and p.maximum > p.minimum:
+                self._blocks.append((p, slice(column, column + 1)))
+                numeric.append(column)
+                column += 1
+            elif isinstance(p, ChoiceParameter) and len(p.choices) > 1:
+                self._blocks.append((p, slice(column, column + len(p.choices))))
+                column += len(p.choices)
+            else:
+                self._constant[p.name] = p.sample(np.random.default_rng(0))  # fixed, or only one possible value
+        if not self._blocks:
+            raise UnsupportedSearchSpace("Bayesian optimization needs at least one parameter with more than one possible value")
+        self.columns = column
+        self.variables = len(self._blocks)
+        self.numeric = np.array(numeric, dtype=int)
+
+    def encode(self, params: dict) -> np.ndarray:
+        x = np.zeros(self.columns)
+        for p, block in self._blocks:
+            if isinstance(p, RangeParameter):
+                x[block] = p.to_unit(params[p.name])
+            else:
+                x[block.start + p.choices.index(params[p.name])] = 1.0
+        return x
+
+    def decode(self, x: np.ndarray) -> dict:
+        values = dict(self._constant)
+        for p, block in self._blocks:
+            if isinstance(p, RangeParameter):
+                values[p.name] = p.from_unit(float(x[block][0]))
+            else:
+                values[p.name] = p.choices[int(np.argmax(x[block]))]
+        return {name: values[name] for name in self.search_space.names}
+
+    def snap(self, X: np.ndarray) -> np.ndarray:
+        """Each row moved to the nearest point that parameter values can take: whole integers and one choice per block."""
+        X = np.clip(X, 0, 1)
+        for p, block in self._blocks:
+            if isinstance(p, RangeParameter) and p.integer:
+                X[:, block] = p.to_unit(p.from_unit(X[:, block]))
+            elif isinstance(p, ChoiceParameter):
+                chosen = np.argmax(X[:, block], axis=1)
+                X[:, block] = 0.0
+                X[np.arange(len(X)), block.start + chosen] = 1.0
+        return X
+
+    def spread(self, rng: np.random.Generator, count: int) -> np.ndarray:
+        """count points spread over the space: a Latin hypercube for numbers, random choices otherwise."""
+        X = np.zeros((count, self.columns))
+        for p, block in self._blocks:
+            if isinstance(p, RangeParameter):
+                X[:, block.start] = (rng.permutation(count) + rng.uniform(size=count)) / max(count, 1)
+            else:
+                X[np.arange(count), block.start + rng.integers(len(p.choices), size=count)] = 1.0
+        return self.snap(X)
+
+    def perturb(self, rng: np.random.Generator, X: np.ndarray, scales: np.ndarray) -> np.ndarray:
+        """X nudged: numbers by Gaussian steps of the given scale, and each choice switched with probability 0.2."""
+        X = X.copy()
+        if self.numeric.size:
+            X[:, self.numeric] += rng.normal(size=(len(X), self.numeric.size)) * scales
+        for p, block in self._blocks:
+            if isinstance(p, ChoiceParameter):
+                switch = rng.uniform(size=len(X)) < 0.2
+                X[switch, block] = 0.0
+                X[np.flatnonzero(switch), block.start + rng.integers(len(p.choices), size=switch.sum())] = 1.0
+        return X

hone/optimizers/choice.py

@@ -0,0 +1,44 @@
+"""Choosing an optimizer for a problem, by what the benchmark shows works best."""
+from typing import Any
+
+from hone.config import BayesianConfig, CMAESConfig, GeneticOptimizerConfig
+from hone.parameters import ChoiceParameter, Parameter, RangeParameter, SearchSpace
+
+from .base import Optimizer
+from .bayesian import BayesianOptimizer
+from .cmaes import CMAES
+from .genetic import GeneticOptimizer
+
+# Up to this many evaluations, Bayesian optimization had the lowest median error on most
+# benchmark problems; past it, each of its candidates costs too long to choose.
+BAYESIAN_UP_TO = 200
+
+
+def choose_optimizer(
+    search_space: SearchSpace | list[Parameter],
+    max_evaluations: int,
+    *,
+    initial_points: dict[str, Any] | list[dict[str, Any]] | None = None,
+    **config: Any,
+) -> Optimizer:
+    """The optimizer the benchmark suggests for this search space and budget, set up to use it.
+
+    - Up to 200 evaluations: Bayesian optimization, unless a choice parameter has choose_n.
+    - More, with choice parameters: the genetic optimizer, since CMA-ES searches numbers only.
+    - More, with integer parameters: the genetic optimizer, which did better on the benchmark's
+      integer and mixed problems.
+    - More, with continuous numbers only: CMA-ES.
+
+    Other settings, such as seed, maximize, noise or threshold, are passed on to the chosen
+    optimizer's config. The optimizer's class and config show what was chosen.
+    """
+    if not isinstance(search_space, SearchSpace):
+        search_space = SearchSpace(search_space)
+    choices = [p for p in search_space.parameters if isinstance(p, ChoiceParameter)]
+    settings = dict(config, max_evaluations=max_evaluations)
+    if max_evaluations <= BAYESIAN_UP_TO and not any(p.choose_n is not None for p in choices):
+        return BayesianOptimizer(search_space, BayesianConfig(**settings), initial_points=initial_points)
+    integers = [p for p in search_space.parameters if isinstance(p, RangeParameter) and p.integer and p.maximum > p.minimum]
+    if choices or integers:
+        return GeneticOptimizer(search_space, GeneticOptimizerConfig(**settings), initial_points=initial_points)
+    return CMAES(search_space, CMAESConfig(**settings), initial_points=initial_points)

hone/optimizers/cmaes.py

@@ -0,0 +1,232 @@
+"""CMA-ES: the covariance matrix adaptation evolution strategy.
+
+The update follows N. Hansen, "The CMA Evolution Strategy: A Tutorial" (arXiv:1604.00772),
+with positive recombination weights. Three additions adapt it to hone's search spaces:
+
+- Range parameters are scaled to [0, 1], on a log scale where they have one, and a sample
+  that falls outside is mirrored back in. The mirrored point is the one evaluated and used
+  in the update.
+- Integer parameters keep a minimum spread of a quarter of one integer step, so the search
+  can always move to a neighbouring value. Samples are widened to that spread, but the
+  distribution learns from the steps as drawn before widening, as in Hamano et al.,
+  "CMA-ES with Margin" (GECCO 2022), so the widening doesn't inflate the step size.
+- With restarts on, a run that converges or stalls starts again from a random point with
+  twice the population (Auger and Hansen, "A Restart CMA Evolution Strategy With Increasing
+  Population Size", CEC 2005).
+"""
+import math
+from typing import Any, List
+
+import numpy as np
+
+from hone.config import CMAESConfig
+from hone.optimizers.base import Individual, Optimizer, UnsupportedSearchSpace, _fitness
+from hone.parameters import ChoiceParameter, FixedParameter, Parameter, RangeParameter, SearchSpace
+
+INTEGER_SPREAD = 0.25  # minimum standard deviation of an integer variable, in integer steps
+MAX_CONDITION = 1e14  # restart when the covariance matrix is this badly conditioned
+MIN_STEP = 1e-12  # restart when every continuous step, in [0, 1] units, falls below this
+FLAT = 1e-12  # restart when recent generations' best values differ by less than this
+
+
+class CMAES(Optimizer):
+    """Covariance matrix adaptation evolution strategy, for range parameters.
+
+    Each generation samples candidates from a multivariate normal distribution, then moves
+    the distribution's centre toward the best of them and reshapes it to stretch along the
+    directions that improved. It learns the scale of each variable and how variables
+    interact, which makes it strong on smooth continuous problems.
+
+    Range parameters, continuous or integer, are searched; fixed parameters are passed
+    through. Choice parameters are not supported: use GeneticOptimizer for those.
+
+    The search starts at the centre of every range. With initial points, it evaluates them
+    first and starts at the best one instead; a smaller initial_step then keeps the search
+    close to it.
+
+    Args:
+        search_space: The parameters to optimize over, as a SearchSpace or a list of parameters.
+        config: A CMAESConfig. Defaults are used if omitted.
+        initial_points: Points to evaluate first: one dict of parameter values, or a list of them.
+    """
+    config_class = CMAESConfig
+    config: CMAESConfig
+
+    def __init__(
+        self,
+        search_space: SearchSpace | List[Parameter],
+        config: CMAESConfig | None = None,
+        *,
+        initial_points: dict[str, Any] | list[dict[str, Any]] | None = None,
+    ) -> None:
+        if not isinstance(search_space, SearchSpace):
+            search_space = SearchSpace(search_space)
+        choices = [p.name for p in search_space.parameters if isinstance(p, ChoiceParameter)]
+        if choices:
+            raise UnsupportedSearchSpace(
+                f"CMA-ES searches range parameters only; use GeneticOptimizer for choice parameters ({', '.join(choices)})"
+            )
+        searched = [p for p in search_space.parameters if isinstance(p, RangeParameter) and p.maximum > p.minimum]
+        if not searched:
+            raise UnsupportedSearchSpace("CMA-ES needs at least one range parameter with minimum below maximum")
+        self._searched = searched
+        self._constant = {
+            p.name: (p.from_unit(0.0) if isinstance(p, RangeParameter) else p.value)
+            for p in search_space.parameters
+            if isinstance(p, (RangeParameter, FixedParameter)) and p not in searched
+        }
+        self._integer = np.array([p.integer for p in searched])
+        super().__init__(search_space, config, initial_points=initial_points)
+
+    # --- state ----------------------------------------------------------------
+
+    def reset(self) -> None:
+        super().reset()
+        n = len(self._searched)
+        # Each generation waits for its slowest evaluation, so with many workers a population of
+        # at least twice their number keeps them busy.
+        self.population_size = self.config.population_size or max(4 + int(3 * math.log(n)), 2 * self._workers)
+        self._start(mean=np.full(n, 0.5))
+        self._evaluating_initial_points = False
+
+    def _start(self, mean: np.ndarray) -> None:
+        """Set up a fresh search distribution around mean for the current population size."""
+        n = len(mean)
+        lam = self.population_size
+        mu = lam // 2
+        weights = math.log((lam + 1) / 2) - np.log(np.arange(1, mu + 1))
+        self._weights = weights / weights.sum()
+        self._mu_eff = 1 / np.sum(self._weights**2)
+        mu_eff = self._mu_eff
+
+        self._c_sigma = (mu_eff + 2) / (n + mu_eff + 5)
+        self._d_sigma = 1 + 2 * max(0.0, math.sqrt((mu_eff - 1) / (n + 1)) - 1) + self._c_sigma
+        self._c_c = (4 + mu_eff / n) / (n + 4 + 2 * mu_eff / n)
+        self._c_1 = 2 / ((n + 1.3) ** 2 + mu_eff)
+        self._c_mu = min(1 - self._c_1, 2 * (mu_eff - 2 + 1 / mu_eff) / ((n + 2) ** 2 + mu_eff))
+        self._chi_n = math.sqrt(n) * (1 - 1 / (4 * n) + 1 / (21 * n**2))
+
+        self._mean = mean
+        self._sigma = self.config.initial_step
+        self._C = np.eye(n)
+        self._B = np.eye(n)
+        self._D = np.ones(n)
+        self._p_sigma = np.zeros(n)
+        self._p_c = np.zeros(n)
+        self._iteration = 0
+        self._recent_bests: list[float] = []
+        self._vectors: dict[Individual, np.ndarray] = {}  # each individual of the current generation -> its point in [0, 1]^n
+
+    # --- the generation -------------------------------------------------------
+
+    def _next_population(self, previous: list[Individual] | None, generation: int) -> list[Individual]:
+        if previous is None and self.initial_points:
+            self._evaluating_initial_points = True
+            return self._initial_individuals(generation)
+        if self._evaluating_initial_points:
+            assert previous is not None
+            self._evaluating_initial_points = False
+            succeeded = self._ranked([ind for ind in previous if not ind.failed])
+            if succeeded:
+                self._mean = self._encode(succeeded[0].params)
+        elif previous is not None:
+            self._update(previous)
+        spread = self._sigma * np.sqrt(np.diag(self._C))
+        minimum = self._minimum_spread()
+        # Integer variables are sampled wider when their spread falls below the minimum.
+        self._widen = np.where(spread < minimum, minimum / np.maximum(spread, 1e-300), 1.0)
+        return [self._sample(generation) for _ in range(self.population_size)]
+
+    def _sample(self, generation: int) -> Individual:
+        z = self.rng.standard_normal(len(self._mean))
+        x = _mirror(self._mean + self._sigma * self._widen * (self._B @ (self._D * z)))
+        individual = Individual(params=self._decode(x), generation=generation)
+        self._vectors[individual] = x
+        return individual
+
+    def _update(self, previous: list[Individual]) -> None:
+        ranked = [ind for ind in self._ranked(previous) if not ind.failed and ind in self._vectors]
+        if ranked:
+            self._adapt(np.array([self._vectors[ind] for ind in ranked[: len(self._weights)]]))
+            self._recent_bests.append(_fitness(ranked[0]))
+        self._vectors = {}
+        if self.config.restarts and self._should_restart():
+            self.population_size *= 2
+            self._start(mean=self.rng.uniform(0, 1, len(self._mean)))
+
+    def _adapt(self, best: np.ndarray) -> None:
+        """Move and reshape the distribution toward best, the top points ordered best first."""
+        n = len(self._mean)
+        weights = self._weights[: len(best)] / self._weights[: len(best)].sum()
+        # The paths, step size and covariance learn from the steps as sampled before integer
+        # widening, as in CMA-ES with margin; otherwise the widening would read as the search
+        # moving faster than it is, and inflate the step size.
+        steps = (best - self._mean) / (self._sigma * self._widen)
+        step = weights @ steps
+        self._mean = self._mean + self._sigma * self._widen * step
+
+        inv_sqrt_C = self._B @ np.diag(1 / self._D) @ self._B.T
+        self._p_sigma = (1 - self._c_sigma) * self._p_sigma + math.sqrt(
+            self._c_sigma * (2 - self._c_sigma) * self._mu_eff
+        ) * (inv_sqrt_C @ step)
+        self._iteration += 1
+        norm = np.linalg.norm(self._p_sigma)
+        h_sigma = norm / math.sqrt(1 - (1 - self._c_sigma) ** (2 * self._iteration)) < (1.4 + 2 / (n + 1)) * self._chi_n
+        self._p_c = (1 - self._c_c) * self._p_c + h_sigma * math.sqrt(self._c_c * (2 - self._c_c) * self._mu_eff) * step
+
+        rank_mu = steps.T @ (weights[:, None] * steps)
+        correction = (1 - h_sigma) * self._c_c * (2 - self._c_c)
+        self._C = (
+            (1 - self._c_1 - self._c_mu + self._c_1 * correction) * self._C
+            + self._c_1 * np.outer(self._p_c, self._p_c)
+            + self._c_mu * rank_mu
+        )
+        self._sigma *= math.exp((self._c_sigma / self._d_sigma) * (norm / self._chi_n - 1))
+
+        self._C = (self._C + self._C.T) / 2
+        eigenvalues, self._B = np.linalg.eigh(self._C)
+        self._D = np.sqrt(np.maximum(eigenvalues, 1e-300))
+
+    def _should_restart(self) -> bool:
+        continuous = ~self._integer
+        if continuous.any() and np.all(self._sigma * np.sqrt(np.diag(self._C))[continuous] < MIN_STEP):
+            return True
+        if (self._D.max() / self._D.min()) ** 2 > MAX_CONDITION:
+            return True
+        window = 10 + math.ceil(30 * len(self._mean) / self.population_size)
+        recent = self._recent_bests[-window:]
+        return len(recent) == window and max(recent) - min(recent) <= FLAT * max(1.0, abs(min(recent)))
+
+    def _minimum_spread(self) -> np.ndarray:
+        """The smallest standard deviation each variable keeps, in [0, 1] units: INTEGER_SPREAD integer steps near the mean.
+
+        On a log scale an integer step is wider at the bottom of the range than at the top,
+        so it is measured where the search currently is.
+        """
+        spread = np.zeros(len(self._searched))
+        for i, p in enumerate(self._searched):
+            if not p.integer:
+                continue
+            if p.log:
+                value = float(np.exp(np.log(p.minimum) + self._mean[i] * (np.log(p.maximum) - np.log(p.minimum))))
+                spread[i] = INTEGER_SPREAD * (p.to_unit(value + 1) - p.to_unit(value))
+            else:
+                spread[i] = INTEGER_SPREAD / (p.maximum - p.minimum)
+        return spread
+
+    # --- between [0, 1]^n and parameter values --------------------------------
+
+    def _encode(self, params: dict) -> np.ndarray:
+        return np.array([p.to_unit(params[p.name]) for p in self._searched], dtype=float)
+
+    def _decode(self, x: np.ndarray) -> dict:
+        values = dict(self._constant)
+        for parameter, u in zip(self._searched, x):
+            values[parameter.name] = parameter.from_unit(float(u))
+        return {name: values[name] for name in self.search_space.names}
+
+
+def _mirror(x: np.ndarray) -> np.ndarray:
+    """Reflect each coordinate back into [0, 1] at the edges."""
+    x = np.mod(x, 2.0)
+    return np.where(x > 1, 2 - x, x)

hone/optimizers/genetic/__init__.py

@@ -0,0 +1 @@
+from .optimizer import GeneticOptimizer