hone-python 0.9.0__py3-none-any.whl

hone/__init__.py

@@ -0,0 +1,40 @@
+from hone.config import BayesianConfig, CMAESConfig, GeneticOptimizerConfig, OptimizerConfig, SelectionMethod
+from hone.optimizers import (
+    CMAES,
+    BayesianOptimizer,
+    EvaluationFailed,
+    GeneticOptimizer,
+    Optimizer,
+    RandomSearch,
+    Trial,
+    UnsupportedSearchSpace,
+    choose_optimizer,
+)
+from hone.parameters import ChoiceParameter, FixedParameter, RangeParameter, SearchSpace
+from hone.result import GenerationStats, Progress, Result, RunInfo, print_progress
+
+__all__ = [
+    "BayesianConfig",
+    "BayesianOptimizer",
+    "CMAES",
+    "CMAESConfig",
+    "ChoiceParameter",
+    "EvaluationFailed",
+    "FixedParameter",
+    "GenerationStats",
+    "GeneticOptimizer",
+    "GeneticOptimizerConfig",
+    "Optimizer",
+    "OptimizerConfig",
+    "Progress",
+    "RandomSearch",
+    "RangeParameter",
+    "Result",
+    "RunInfo",
+    "SearchSpace",
+    "SelectionMethod",
+    "Trial",
+    "UnsupportedSearchSpace",
+    "choose_optimizer",
+    "print_progress",
+]

hone/benchmarks/__init__.py

@@ -0,0 +1,17 @@
+"""Benchmark problems and a runner for comparing optimizers. Run `python -m hone.benchmarks --help`."""
+from .ml import ml_problems
+from .problems import Problem, noisy_problems, quick_problems, standard_problems
+from .runner import (
+    BUDGETS,
+    DEFAULT_OPTIMIZERS,
+    Run,
+    SkipBudget,
+    Summary,
+    load_summaries,
+    regressions,
+    run_benchmark,
+    save_summaries,
+    summarize,
+    to_markdown,
+    wins,
+)

hone/benchmarks/__main__.py

@@ -0,0 +1,84 @@
+"""Command-line entry point: `python -m hone.benchmarks`."""
+import argparse
+import sys
+import time
+from collections.abc import Sequence
+from importlib.metadata import version
+
+from . import ml
+from .problems import noisy_problems, quick_problems, standard_problems
+from .runner import BUDGETS, load_summaries, regressions, run_benchmark, save_summaries, summarize, to_markdown
+
+
+QUICK_BUDGETS = (10, 50, 200)
+NOISY_BUDGETS = (10, 40, 200)
+
+
+def _join(numbers: list[int]) -> str:
+    """[10, 20, 50] -> "10, 20 and 50"."""
+    words = [f"{n:,}" for n in numbers]
+    return words[0] if len(words) == 1 else ", ".join(words[:-1]) + " and " + words[-1]
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m hone.benchmarks", description=__doc__)
+    suite = parser.add_mutually_exclusive_group()
+    suite.add_argument("--quick", action="store_true", help="run the small CI subset with fewer seeds and evaluations")
+    suite.add_argument("--ml", action="store_true", help="run the machine learning tuning problems instead (needs scikit-learn)")
+    suite.add_argument("--noisy", action="store_true", help="run the problems with noise added to every evaluation instead")
+    parser.add_argument("--seeds", type=int, help="number of seeds per problem (default 10, or 5 with --quick)")
+    parser.add_argument(
+        "--budgets",
+        type=int,
+        nargs="+",
+        help=f"evaluations per dimension, one run per budget (default {' '.join(map(str, BUDGETS))}, "
+        f"{' '.join(map(str, QUICK_BUDGETS))} with --quick, {' '.join(map(str, ml.BUDGETS))} with --ml, "
+        f"or {' '.join(map(str, NOISY_BUDGETS))} with --noisy)",
+    )
+    parser.add_argument("--jobs", type=int, default=1, help="share the runs among this many processes (Linux only)")
+    parser.add_argument("--tolerance", type=float, default=1e-4, help="error at or below which a run counts as solved")
+    parser.add_argument("--markdown", help="write the results table to this file")
+    parser.add_argument("--save", help="write the summaries as JSON to this file, for use as a baseline")
+    parser.add_argument("--baseline", help="compare against summaries saved with --save; exit 1 on a regression")
+    args = parser.parse_args(argv)
+
+    default_budgets: Sequence[int]
+    if args.quick:
+        problems, default_budgets = quick_problems(), QUICK_BUDGETS
+    elif args.ml:
+        problems, default_budgets = ml.ml_problems(), ml.BUDGETS
+    elif args.noisy:
+        problems, default_budgets = noisy_problems(), NOISY_BUDGETS
+    else:
+        problems, default_budgets = standard_problems(), BUDGETS
+    seeds = args.seeds or (5 if args.quick else 10)
+    budgets = sorted(set(args.budgets or default_budgets))
+
+    start = time.perf_counter()
+    summaries = summarize(run_benchmark(problems, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs), args.tolerance)
+    elapsed = time.perf_counter() - start
+
+    table = to_markdown(summaries, args.tolerance)
+    print(table)
+    setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
+    print(f"{setup}, in {elapsed:.0f}s")
+    if args.markdown:
+        with open(args.markdown, "w") as f:
+            f.write(
+                f"# Benchmark results\n\nGenerated by `python -m hone.benchmarks` with hone {version('hone-python')}: "
+                f"{setup}.\n\n{table}"
+            )
+    if args.save:
+        save_summaries(summaries, args.save)
+
+    if args.baseline:
+        found = regressions(summaries, load_summaries(args.baseline))
+        if found:
+            print("\nRegressions against the baseline:", *found, sep="\n  ")
+            return 1
+        print("\nNo regressions against the baseline.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

hone/benchmarks/ml.py

@@ -0,0 +1,133 @@
+"""Machine learning tuning problems: choosing a model's hyperparameters by cross-validation.
+
+Each problem trains a scikit-learn model on one of its bundled datasets and scores it by
+5-fold cross-validation with fixed folds, so every evaluation is a real training run and the
+same settings always give the same score. Each problem minimizes 1 − score: 1 − accuracy for
+classification, 1 − R² for regression.
+
+The true optimum is unknown, so each problem's optimum is the best value a long reference
+search found (see REFERENCE_SEARCH), and error is measured from there. Settings that span
+orders of magnitude are log-scale ranges.
+
+These problems need scikit-learn, which hone itself doesn't depend on: `pip install scikit-learn`.
+"""
+from functools import cache
+from typing import Callable
+
+from threadpoolctl import threadpool_limits
+
+from hone.parameters import ChoiceParameter, RangeParameter, SearchSpace
+
+from .problems import Problem
+
+# How each problem's optimum was found: the best of these runs, each with seeds 0 to 2.
+REFERENCE_SEARCH = (
+    "BayesianOptimizer with 200 evaluations, and CMA-ES (where it applies), the genetic optimizer "
+    "and random search with 1,000"
+)
+
+# Model tuning is done on small budgets, so these problems run at these budgets only.
+BUDGETS = (10, 20, 50)
+
+
+def _require_sklearn() -> None:
+    try:
+        import sklearn  # noqa: F401
+    except ImportError as error:
+        raise ImportError("the machine learning problems need scikit-learn: pip install scikit-learn") from error
+
+
+@cache
+def _data(name: str):
+    from sklearn import datasets
+
+    return getattr(datasets, f"load_{name}")(return_X_y=True)
+
+
+def _cross_validate(model, dataset: str, classification: bool) -> float:
+    """1 − the mean cross-validated score: accuracy for a classifier, R² for a regressor."""
+    from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score
+
+    X, y = _data(dataset)
+    folds = (StratifiedKFold if classification else KFold)(5, shuffle=True, random_state=0)
+    # The datasets are small, and threads only add overhead.
+    with threadpool_limits(limits=1):
+        score = cross_val_score(model, X, y, cv=folds, scoring="accuracy" if classification else "r2").mean()
+    return 1.0 - float(score)
+
+
+def svm_breast_cancer(C: float, gamma: float) -> float:
+    """A support vector classifier with an RBF kernel, on 569 breast tumour samples with 30 features."""
+    from sklearn.pipeline import make_pipeline
+    from sklearn.preprocessing import StandardScaler
+    from sklearn.svm import SVC
+
+    model = make_pipeline(StandardScaler(), SVC(C=C, gamma=gamma))
+    return _cross_validate(model, "breast_cancer", classification=True)
+
+
+def kernel_ridge_diabetes(alpha: float, gamma: float, kernel: str) -> float:
+    """Kernel ridge regression of disease progression from 10 measurements, on 442 diabetes patients."""
+    from sklearn.kernel_ridge import KernelRidge
+    from sklearn.pipeline import make_pipeline
+    from sklearn.preprocessing import StandardScaler
+
+    model = make_pipeline(StandardScaler(), KernelRidge(alpha=alpha, gamma=gamma, kernel=kernel))
+    return _cross_validate(model, "diabetes", classification=False)
+
+
+def random_forest_wine(max_depth: int, min_samples_leaf: int, max_features: float, criterion: str) -> float:
+    """A 20-tree random forest classifying 178 wines into 3 cultivars from 13 chemical measurements."""
+    from sklearn.ensemble import RandomForestClassifier
+
+    model = RandomForestClassifier(
+        n_estimators=20,
+        max_depth=max_depth,
+        min_samples_leaf=min_samples_leaf,
+        max_features=max_features,
+        criterion=criterion,
+        random_state=0,
+        n_jobs=1,
+    )
+    return _cross_validate(model, "wine", classification=True)
+
+
+# The best value of each problem found by REFERENCE_SEARCH.
+OPTIMA = {
+    "svm-breast-cancer": 0.014066138798323302,  # accuracy 98.6%, found by CMA-ES
+    "kernel-ridge-diabetes": 0.4961748264794844,  # R² 0.504, found by Bayesian optimization
+    "random-forest-wine": 0.005555555555555536,  # accuracy 99.4%, found by Bayesian optimization
+}
+
+SPACES: dict[str, tuple[Callable[..., float], list]] = {
+    "svm-breast-cancer": (
+        svm_breast_cancer,
+        [RangeParameter("C", 1e-3, 1e3, log=True), RangeParameter("gamma", 1e-5, 10, log=True)],
+    ),
+    "kernel-ridge-diabetes": (
+        kernel_ridge_diabetes,
+        [
+            RangeParameter("alpha", 1e-4, 100, log=True),
+            RangeParameter("gamma", 1e-5, 10, log=True),
+            ChoiceParameter("kernel", ["rbf", "laplacian"]),
+        ],
+    ),
+    "random-forest-wine": (
+        random_forest_wine,
+        [
+            RangeParameter("max_depth", 1, 12, integer=True),
+            RangeParameter("min_samples_leaf", 1, 20, integer=True),
+            RangeParameter("max_features", 0.05, 1.0),
+            ChoiceParameter("criterion", ["gini", "entropy"]),
+        ],
+    ),
+}
+
+
+def ml_problems() -> list[Problem]:
+    """The machine learning tuning problems. Needs scikit-learn."""
+    _require_sklearn()
+    return [
+        Problem(name=name, function=function, search_space=SearchSpace(parameters), optimum=OPTIMA[name], tags=frozenset({"ml"}))
+        for name, (function, parameters) in SPACES.items()
+    ]

hone/benchmarks/problems.py

@@ -0,0 +1,236 @@
+"""Test problems with known optima.
+
+The continuous functions follow the conventions of the BBOB benchmark: every variable lies
+in [-5, 5], and each function is shifted so its optimum sits at a fixed, problem-specific
+point away from the centre of the box. Optimizers that drift toward the centre would
+otherwise look better than they are.
+
+Each continuous function except sphere also comes in a rotated version, which turns the
+coordinate axes by a fixed random rotation. The plain versions are separable: every
+variable affects the value independently, so an optimizer can improve one variable at a
+time, and crossover that swaps whole variables between parents does unusually well. In
+the rotated versions the variables interact, as they usually do in engineering problems.
+"""
+import math
+import zlib
+from typing import Any, Callable
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict
+
+from hone.parameters import ChoiceParameter, Parameter, RangeParameter, SearchSpace
+
+BOUND = 5.0
+
+
+class Problem(BaseModel):
+    """An objective function, its search space, and the lowest value it can take."""
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    function: Callable[..., float]
+    search_space: SearchSpace
+    optimum: float
+    tags: frozenset[str]
+    noise: float = 0.0  # standard deviation of Gaussian noise the benchmark adds to each evaluation
+
+    @property
+    def dimensions(self) -> int:
+        return len(self.search_space.parameters)
+
+
+def _shift(name: str, dimensions: int, integer: bool = False) -> np.ndarray:
+    """A fixed optimum location for the named problem, within 60% of the bounds."""
+    rng = np.random.default_rng(zlib.crc32(name.encode()))
+    shift = rng.uniform(-0.6 * BOUND, 0.6 * BOUND, size=dimensions)
+    return np.round(shift) if integer else shift
+
+
+def _rotation(name: str, dimensions: int) -> np.ndarray:
+    """A fixed random rotation for the named problem: an orthogonal matrix with determinant 1."""
+    rng = np.random.default_rng(zlib.crc32(f"{name}/rotation".encode()))
+    q, r = np.linalg.qr(rng.standard_normal((dimensions, dimensions)))
+    q = q * np.sign(np.diag(r))  # makes the rotation uniformly random
+    if np.linalg.det(q) < 0:
+        q[:, 0] = -q[:, 0]
+    return q
+
+
+def _vector_problem(
+    name: str,
+    base: Callable[[np.ndarray], float],
+    dimensions: int,
+    parameters: list[Parameter] | None = None,
+    integer: bool = False,
+    rotate: bool = False,
+    tags: frozenset[str] = frozenset({"continuous"}),
+) -> Problem:
+    """Wrap a function of a vector z, minimal at z = 0, as a shifted problem over x0..x{n-1}.
+
+    With rotate, z is the shifted point turned by a fixed rotation, so the variables interact.
+    """
+    names = [f"x{i}" for i in range(dimensions)]
+    shift = _shift(name, dimensions, integer)
+    rotation = _rotation(name, dimensions) if rotate else np.eye(dimensions)
+
+    def function(**params: Any) -> float:
+        x = np.array([params[n] for n in names], dtype=float)
+        return float(base(rotation @ (x - shift)))
+
+    function.__name__ = name.replace("-", "_")
+    if parameters is None:
+        parameters = [RangeParameter(n, -BOUND, BOUND, integer=integer) for n in names]
+    return Problem(name=name, function=function, search_space=SearchSpace(parameters), optimum=0.0, tags=tags)
+
+
+# --- base functions, each with minimum 0 at z = 0 ---------------------------
+
+
+def sphere(z: np.ndarray) -> float:
+    return np.sum(z**2)
+
+
+def ellipsoid(z: np.ndarray) -> float:
+    """Sphere with axis weights from 1 to 1e6: tests how an optimizer handles badly scaled variables."""
+    weights = 10 ** (6 * np.arange(len(z)) / max(len(z) - 1, 1))
+    return np.sum(weights * z**2)
+
+
+def rosenbrock(z: np.ndarray) -> float:
+    """A long, curved valley. The usual form has its minimum at 1, so z is offset by 1."""
+    x = z + 1
+    return np.sum(100 * (x[1:] - x[:-1] ** 2) ** 2 + (1 - x[:-1]) ** 2)
+
+
+def rastrigin(z: np.ndarray) -> float:
+    """A regular grid of local minima around one global minimum."""
+    return 10 * len(z) + np.sum(z**2 - 10 * np.cos(2 * np.pi * z))
+
+
+def ackley(z: np.ndarray) -> float:
+    """A nearly flat outer region with a deep, narrow hole at the minimum."""
+    n = len(z)
+    return (
+        -20 * np.exp(-0.2 * np.sqrt(np.sum(z**2) / n))
+        - np.exp(np.sum(np.cos(2 * np.pi * z)) / n)
+        + 20
+        + math.e
+    )
+
+
+CONTINUOUS = {
+    "sphere": sphere,
+    "ellipsoid": ellipsoid,
+    "rosenbrock": rosenbrock,
+    "rastrigin": rastrigin,
+    "ackley": ackley,
+}
+
+# Sphere is the same under any rotation, so it has no rotated version.
+ROTATED = {name: base for name, base in CONTINUOUS.items() if name != "sphere"}
+
+
+def rotated(name: str, dimensions: int) -> Problem:
+    return _vector_problem(
+        f"rotated-{name}-{dimensions}d", ROTATED[name], dimensions, rotate=True, tags=frozenset({"continuous", "rotated"})
+    )
+
+
+# --- mixed problems --------------------------------------------------------
+
+
+def integer_sphere(dimensions: int) -> Problem:
+    """Sphere over integer variables only; the optimum is at an integer point."""
+    return _vector_problem(
+        f"integer-sphere-{dimensions}d", sphere, dimensions, integer=True, tags=frozenset({"integer"})
+    )
+
+
+def mixed_rastrigin(dimensions: int) -> Problem:
+    """Rastrigin with the first half of the variables continuous and the rest integer."""
+    name = f"mixed-rastrigin-{dimensions}d"
+    n_int = dimensions // 2
+    parameters = [
+        RangeParameter(f"x{i}", -BOUND, BOUND, integer=i >= dimensions - n_int) for i in range(dimensions)
+    ]
+    shift = _shift(name, dimensions)
+    shift[dimensions - n_int :] = np.round(shift[dimensions - n_int :])
+    names = [p.name for p in parameters]
+
+    def function(**params: Any) -> float:
+        x = np.array([params[n] for n in names], dtype=float)
+        return float(rastrigin(x - shift))
+
+    function.__name__ = name.replace("-", "_")
+    return Problem(name=name, function=function, search_space=SearchSpace(parameters), optimum=0.0, tags=frozenset({"mixed"}))
+
+
+MATERIALS = {
+    # A synthetic material choice: each material has its own best design point and a base cost.
+    "steel": 3.0,
+    "aluminium": 1.0,
+    "titanium": 0.0,
+    "composite": 2.0,
+}
+
+
+def material_selection(dimensions: int) -> Problem:
+    """A categorical choice that changes where the best continuous design lies.
+
+    Each material moves the optimum of the continuous variables and adds its own base cost,
+    so the optimizer has to search both together. Titanium, at cost 0, gives the global optimum.
+    """
+    name = f"material-selection-{dimensions}d"
+    rng = np.random.default_rng(zlib.crc32(name.encode()))
+    targets = {m: rng.uniform(-0.6 * BOUND, 0.6 * BOUND, size=dimensions) for m in MATERIALS}
+    names = [f"x{i}" for i in range(dimensions)]
+    parameters: list[Parameter] = [ChoiceParameter("material", list(MATERIALS))]
+    parameters += [RangeParameter(n, -BOUND, BOUND) for n in names]
+
+    def function(material: str, **params: Any) -> float:
+        x = np.array([params[n] for n in names], dtype=float)
+        return float(MATERIALS[material] + sphere(x - targets[material]))
+
+    function.__name__ = name.replace("-", "_")
+    return Problem(name=name, function=function, search_space=SearchSpace(parameters), optimum=0.0, tags=frozenset({"mixed", "categorical"}))
+
+
+def standard_problems(dimensions: tuple[int, ...] = (2, 5, 10)) -> list[Problem]:
+    """The full suite: every continuous function, plain and rotated, and every mixed problem, in each dimension."""
+    problems = [
+        _vector_problem(f"{name}-{d}d", base, d) for d in dimensions for name, base in CONTINUOUS.items()
+    ]
+    problems += [rotated(name, d) for d in dimensions for name in ROTATED]
+    for d in dimensions:
+        problems += [integer_sphere(d), mixed_rastrigin(d), material_selection(d)]
+    return problems
+
+
+def quick_problems() -> list[Problem]:
+    """A small subset for CI: one dimension, and a problem of each kind."""
+    return [
+        _vector_problem("sphere-5d", sphere, 5),
+        _vector_problem("ellipsoid-5d", ellipsoid, 5),
+        _vector_problem("rastrigin-5d", rastrigin, 5),
+        rotated("rastrigin", 5),
+        integer_sphere(5),
+        material_selection(5),
+    ]
+
+
+# Noise the noisy suite adds to each evaluation, as a standard deviation. The problems' values
+# near their optima are a few units, so this hides the differences an optimizer has to find.
+NOISE = 1.0
+
+
+def noisy_problems() -> list[Problem]:
+    """Problems with Gaussian noise added to every evaluation, scored by the true value of the result."""
+    plain = [
+        _vector_problem("sphere-5d", sphere, 5),
+        _vector_problem("rosenbrock-5d", rosenbrock, 5),
+        _vector_problem("rastrigin-5d", rastrigin, 5),
+        rotated("ackley", 5),
+        mixed_rastrigin(5),
+        material_selection(5),
+    ]
+    return [p.model_copy(update={"name": f"noisy-{p.name}", "noise": NOISE, "tags": p.tags | {"noisy"}}) for p in plain]