hone-python 0.9.0__py3-none-any.whl

hone/benchmarks/runner.py

@@ -0,0 +1,295 @@
+"""Run optimizers on benchmark problems and summarize how close they get to each optimum."""
+import functools
+import json
+import multiprocessing
+import statistics
+import zlib
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import Callable, Iterable
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict
+
+from hone.config import BayesianConfig, CMAESConfig, GeneticOptimizerConfig, OptimizerConfig
+from hone.optimizers import CMAES, BayesianOptimizer, GeneticOptimizer, Optimizer, RandomSearch, UnsupportedSearchSpace
+
+from .problems import Problem
+
+# Builds an optimizer for a problem, an evaluation budget and a seed. It may raise
+# UnsupportedSearchSpace for a problem it cannot search, which is then skipped, or
+# SkipBudget for a budget it isn't meant for.
+OptimizerFactory = Callable[[Problem, int, int], Optimizer]
+
+
+class SkipBudget(Exception):
+    """Raised by an optimizer factory to leave out a budget it isn't meant for."""
+
+
+# Bayesian optimization takes a fraction of a second per evaluation, which suits expensive
+# objectives but makes long benchmark runs slow, so it runs only up to this many evaluations.
+BAYESIAN_MAX_EVALUATIONS = 200
+
+POPULATION_SIZE = 50  # random search's batch size, which doesn't change its results
+
+# Evaluations per dimension that the benchmark gives each run. The small budgets show how
+# an optimizer does when each evaluation is expensive, as with a slow simulation.
+BUDGETS = (10, 20, 50, 100, 1000)
+
+FLOOR = 1e-8  # errors below this count as equal when comparing results
+
+
+def genetic(problem: Problem, budget: int, seed: int) -> Optimizer:
+    config = GeneticOptimizerConfig(max_evaluations=budget, seed=seed, noise=problem.noise > 0)
+    return GeneticOptimizer(problem.search_space, config)
+
+
+def cma_es(problem: Problem, budget: int, seed: int) -> Optimizer:
+    config = CMAESConfig(generations=budget, max_evaluations=budget, seed=seed, noise=problem.noise > 0)
+    return CMAES(problem.search_space, config)
+
+
+def bayesian(problem: Problem, budget: int, seed: int) -> Optimizer:
+    if budget > BAYESIAN_MAX_EVALUATIONS:
+        raise SkipBudget(f"{budget} evaluations is more than Bayesian optimization is run with")
+    config = BayesianConfig(max_evaluations=budget, seed=seed, noise=problem.noise > 0)
+    return BayesianOptimizer(problem.search_space, config)
+
+
+def random_search(problem: Problem, budget: int, seed: int) -> Optimizer:
+    config = OptimizerConfig(
+        population_size=POPULATION_SIZE, generations=budget, max_evaluations=budget, seed=seed, noise=problem.noise > 0
+    )
+    return RandomSearch(problem.search_space, config)
+
+
+DEFAULT_OPTIMIZERS: dict[str, OptimizerFactory] = {
+    "genetic": genetic,
+    "cma-es": cma_es,
+    "bayesian": bayesian,
+    "random": random_search,
+}
+
+
+class Run(BaseModel):
+    """One optimizer on one problem with one seed and one budget."""
+    model_config = ConfigDict(frozen=True)
+
+    problem: str
+    optimizer: str
+    budget_per_dimension: int
+    seed: int
+    error: float  # best value found minus the problem's optimum; 0 means the optimum was found
+    evaluations: int
+    trace: list[tuple[int, float]]  # (evaluations so far, error of the best so far), once per generation
+
+
+class Summary(BaseModel):
+    """All runs of one optimizer on one problem with one budget."""
+    model_config = ConfigDict(frozen=True)
+
+    problem: str
+    optimizer: str
+    budget_per_dimension: int
+    runs: int
+    median_error: float
+    lower_quartile: float
+    upper_quartile: float
+    solved: float  # share of runs whose error is at most the tolerance
+
+    @classmethod
+    def of(cls, runs: list[Run], tolerance: float) -> "Summary":
+        errors = sorted(run.error for run in runs)
+        quartiles = statistics.quantiles(errors, n=4, method="inclusive") if len(errors) > 1 else errors * 3
+        return cls(
+            problem=runs[0].problem,
+            optimizer=runs[0].optimizer,
+            budget_per_dimension=runs[0].budget_per_dimension,
+            runs=len(runs),
+            median_error=statistics.median(errors),
+            lower_quartile=quartiles[0],
+            upper_quartile=quartiles[2],
+            solved=sum(error <= tolerance for error in errors) / len(errors),
+        )
+
+
+def run_benchmark(
+    problems: Iterable[Problem],
+    optimizers: dict[str, OptimizerFactory] | None = None,
+    seeds: Iterable[int] = range(10),
+    budgets_per_dimension: Iterable[int] = BUDGETS,
+    jobs: int = 1,
+) -> list[Run]:
+    """Run every optimizer on every problem with every seed and every budget.
+
+    A budget of b evaluations per dimension gives a run b × dimensions evaluations. Each
+    budget is a separate run, so an optimizer can choose its settings from the budget.
+    An optimizer that cannot search a problem's parameters is skipped for that problem, and
+    one whose factory raises SkipBudget is skipped at that budget. With jobs > 1, runs are
+    shared among that many processes where the platform can fork them; the results are the same.
+    """
+    problems = list(problems)
+    optimizers = optimizers if optimizers is not None else DEFAULT_OPTIMIZERS
+    seeds, budgets = list(seeds), list(budgets_per_dimension)
+    tasks = []
+    for index, problem in enumerate(problems):
+        for name, factory in optimizers.items():
+            for per_dimension in budgets:
+                try:
+                    if seeds:
+                        factory(problem, per_dimension * problem.dimensions, seeds[0])
+                except UnsupportedSearchSpace:
+                    break
+                except SkipBudget:
+                    continue
+                tasks += [(index, name, per_dimension, seed) for seed in seeds]
+
+    global _shared
+    _shared = (problems, optimizers)
+    if jobs == 1 or "fork" not in multiprocessing.get_all_start_methods():
+        return [_task(task) for task in tasks]
+    # Forked processes inherit the problems and factories, which needn't be picklable.
+    with ProcessPoolExecutor(jobs, mp_context=multiprocessing.get_context("fork")) as pool:
+        return list(pool.map(_task, tasks))
+
+
+_shared: tuple[list[Problem], dict[str, OptimizerFactory]] = ([], {})  # what run_benchmark's tasks refer to
+
+
+def _task(task: tuple[int, str, int, int]) -> Run:
+    index, name, per_dimension, seed = task
+    problems, optimizers = _shared
+    problem = problems[index]
+    optimizer = optimizers[name](problem, per_dimension * problem.dimensions, seed)
+    return _run(problem, name, per_dimension, seed, optimizer)
+
+
+def _run(problem: Problem, name: str, per_dimension: int, seed: int, optimizer: Optimizer) -> Run:
+    result = optimizer.optimize(_with_noise(problem, seed) if problem.noise else problem.function)
+
+    def true_value(fitness: float, params: dict) -> float:
+        # A noisy run is scored by the noise-free value of the point it chose, not by the
+        # noisy value it saw there, which is likely to be a lucky one.
+        return problem.function(**params) if problem.noise else fitness
+
+    trace, best = [], float("inf")
+    for generation in result.generations:
+        if generation.best_fitness is not None and generation.best_params is not None:
+            best = min(best, true_value(generation.best_fitness, generation.best_params) - problem.optimum)
+        trace.append((generation.evaluations, max(best, 0.0)))
+    return Run(
+        problem=problem.name,
+        optimizer=name,
+        budget_per_dimension=per_dimension,
+        seed=seed,
+        error=max(true_value(result.best_fitness, result.best_params) - problem.optimum, 0.0),
+        evaluations=result.evaluations,
+        trace=trace,
+    )
+
+
+def _with_noise(problem: Problem, seed: int) -> Callable[..., float]:
+    """problem's function plus Gaussian noise of standard deviation problem.noise, repeatable for the seed."""
+    rng = np.random.default_rng([seed, zlib.crc32(problem.name.encode())])
+
+    @functools.wraps(problem.function)
+    def noisy(**params):
+        return problem.function(**params) + problem.noise * rng.standard_normal()
+
+    return noisy
+
+
+def summarize(runs: list[Run], tolerance: float = 1e-4) -> list[Summary]:
+    """One Summary per problem, optimizer and budget, in the order the runs were made."""
+    groups: dict[tuple[str, str, int], list[Run]] = {}
+    for run in runs:
+        groups.setdefault((run.problem, run.optimizer, run.budget_per_dimension), []).append(run)
+    return [Summary.of(group, tolerance) for group in groups.values()]
+
+
+def wins(summaries: list[Summary], floor: float = FLOOR) -> dict[int, dict[str, int]]:
+    """For each budget, how many problems each optimizer has the lowest median error on.
+
+    Ties count for every optimizer in them, and errors below floor count as equal. Only
+    problems that at least two optimizers ran on are counted.
+    """
+    optimizers = list(dict.fromkeys(s.optimizer for s in summaries))
+    groups: dict[tuple[int, str], list[Summary]] = {}
+    for s in summaries:
+        groups.setdefault((s.budget_per_dimension, s.problem), []).append(s)
+    counts: dict[int, dict[str, int]] = {}
+    for (budget, _), group in groups.items():
+        tally = counts.setdefault(budget, dict.fromkeys(optimizers, 0))
+        if len(group) < 2:
+            continue
+        best = min(max(s.median_error, floor) for s in group)
+        for s in group:
+            tally[s.optimizer] += max(s.median_error, floor) == best
+    return dict(sorted(counts.items()))
+
+
+def to_markdown(summaries: list[Summary], tolerance: float = 1e-4) -> str:
+    """An overview of wins at each budget, then a table per budget with a row per problem."""
+    optimizers = list(dict.fromkeys(s.optimizer for s in summaries))
+    budgets = sorted({s.budget_per_dimension for s in summaries})
+    by_key = {(s.problem, s.optimizer, s.budget_per_dimension): s for s in summaries}
+    problems = list(dict.fromkeys(s.problem for s in summaries))
+
+    lines = [
+        "## Overview",
+        "",
+        "Problems on which each optimizer has the lowest median error, at each budget. Ties count for each "
+        f"optimizer in them, and errors below {FLOOR:g} count as equal.",
+        "",
+        "| Evaluations per dimension | " + " | ".join(optimizers) + " |",
+        "| ---: | " + " | ".join("---:" for _ in optimizers) + " |",
+        *(f"| {b} | " + " | ".join(str(t[o]) for o in optimizers) + " |" for b, t in wins(summaries).items()),
+    ]
+    for budget in budgets:
+        lines += [
+            "",
+            f"## {budget} evaluations per dimension",
+            "",
+            "| Problem | " + " | ".join(f"{o} median | {o} solved" for o in optimizers) + " |",
+            "| --- | " + " | ".join("---: | ---:" for _ in optimizers) + " |",
+        ]
+        for problem in problems:
+            cells = []
+            for o in optimizers:
+                s = by_key.get((problem, o, budget))
+                cells += [f"{s.median_error:.3g}", f"{s.solved:.0%}"] if s else ["n/a", "n/a"]
+            lines.append(f"| {problem} | " + " | ".join(cells) + " |")
+    lines += [
+        "",
+        f"Median error over the runs (best value found minus the optimum); solved means error ≤ {tolerance:g}. "
+        "n/a: the optimizer does not support the problem's parameters, or isn't run at this budget.",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def save_summaries(summaries: list[Summary], path: str | Path) -> None:
+    Path(path).write_text(json.dumps([s.model_dump() for s in summaries], indent=2) + "\n")
+
+
+def load_summaries(path: str | Path) -> list[Summary]:
+    return [Summary.model_validate(s) for s in json.loads(Path(path).read_text())]
+
+
+def regressions(current: list[Summary], baseline: list[Summary], factor: float = 10.0, floor: float = FLOOR) -> list[str]:
+    """Problems and budgets where an optimizer's median error grew by more than factor against the baseline.
+
+    Errors below floor count as floor, so changes among values that are already essentially
+    zero don't count as regressions.
+    """
+    previous = {(s.problem, s.optimizer, s.budget_per_dimension): s for s in baseline}
+    found = []
+    for s in current:
+        before = previous.get((s.problem, s.optimizer, s.budget_per_dimension))
+        if before is None:
+            continue
+        if max(s.median_error, floor) > factor * max(before.median_error, floor):
+            found.append(
+                f"{s.optimizer} on {s.problem} at {s.budget_per_dimension} evaluations per dimension: "
+                f"median error {s.median_error:.3g}, baseline {before.median_error:.3g}"
+            )
+    return found

hone/config/__init__.py

@@ -0,0 +1,4 @@
+from .bayesian_config import BayesianConfig
+from .cmaes_config import CMAESConfig
+from .genetic_optimizer_config import GeneticOptimizerConfig, SelectionMethod
+from .optimizer_config import OptimizerConfig

hone/config/bayesian_config.py

@@ -0,0 +1,23 @@
+from typing import Any
+
+from pydantic import Field, model_validator
+
+from .optimizer_config import OptimizerConfig
+
+
+class BayesianConfig(OptimizerConfig):
+    """Configuration for Bayesian optimization. Usually only the budget, max_evaluations, needs setting.
+
+    Each generation is a batch of candidates chosen together. When max_evaluations is set and
+    generations isn't, generations defaults to the budget, so the run ends when the budget is spent.
+    """
+    population_size: int | None = Field(default=None, ge=1)  # type: ignore[assignment]  # candidates per batch; None: 1, or the number of workers
+    generations: int = Field(default=100, ge=1)
+    initial_samples: int | None = Field(default=None, ge=1)  # spread-out candidates before the model is used; None: 2 × (variables + 1)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _generations_from_budget(cls, data: Any) -> Any:
+        if isinstance(data, dict) and data.get("max_evaluations") is not None and data.get("generations") is None:
+            data = {**data, "generations": data["max_evaluations"]}
+        return data

hone/config/cmaes_config.py

@@ -0,0 +1,11 @@
+from pydantic import Field
+
+from .optimizer_config import OptimizerConfig
+
+
+class CMAESConfig(OptimizerConfig):
+    """Configuration for CMA-ES. The defaults suit most problems; usually only the budget needs setting."""
+    population_size: int | None = Field(default=None, ge=2)  # type: ignore[assignment]  # None: 4 + 3 ln(n) for n variables, or twice the workers if more
+    generations: int = Field(default=1000, ge=1)  # an upper bound; max_evaluations is the usual way to set a budget
+    initial_step: float = Field(default=0.3, gt=0, le=1)  # starting step size, as a fraction of each range
+    restarts: bool = True  # restart with double the population when a run converges or stalls (IPOP-CMA-ES)

hone/config/genetic_optimizer_config.py

@@ -0,0 +1,74 @@
+from enum import StrEnum
+
+from typing import Any
+
+from pydantic import Field, model_validator
+
+from .optimizer_config import OptimizerConfig
+
+
+class SelectionMethod(StrEnum):
+    """Selection methods for the genetic algorithm."""
+    TOURNAMENT = 'tournament'
+    ROULETTE = 'roulette'
+
+
+# With a budget and no population size, the population is sized for about this many
+# generations, within these bounds. Chosen from the benchmark's small budgets.
+GENERATIONS_PER_BUDGET = 20
+MIN_POPULATION = 16
+MAX_POPULATION = 50
+
+
+class GeneticOptimizerConfig(OptimizerConfig):
+    """Configuration for the genetic optimizer.
+
+    When max_evaluations is set, population_size and generations default to suit it: the
+    population to about a twentieth of the budget, between 16 and 50, and generations to
+    as many as the budget allows. Without a budget they default to 100 and 25.
+    """
+    # Chance each gene of a child is mutated. None: 1 / the number of variables, at least 0.1;
+    # with noise, 0.1, which the benchmark's noisy problems favour at large budgets.
+    mutation_rate: float | None = Field(default=None, ge=0, le=1)
+    # Size of a range mutation, as a fraction of the range width: the starting size when
+    # mutation adapts, and the size throughout when it doesn't.
+    mutation_scale: float = Field(default=0.1, gt=0)
+    # Each individual carries its own mutation size, which its children inherit and vary, so
+    # selection tunes it as the run goes. None: on, except with noise, where it didn't help.
+    adaptive_mutation: bool | None = None
+    crossover_rate: float = Field(default=0.8, ge=0, le=1)  # chance each gene of a child comes from its first parent
+    # Draw each range value of a child from between its parents' values, widened by blend_alpha
+    # times their distance on each side, rather than copying one.
+    blend_crossover: bool = True
+    blend_alpha: float = Field(default=0.1, ge=0)
+    elitism: bool = True
+    elitism_rate: float = Field(default=0.1, ge=0, lt=1)  # share of each generation carried over unchanged
+    selection_method: SelectionMethod = SelectionMethod.TOURNAMENT
+    tournament_size: int = Field(default=4, ge=2)
+    tournament_p: float = Field(default=1.0, ge=0, le=1)  # chance a tournament picks its best rather than its worst
+    # Share of children bred within niches of parents with the same choice parameter values, so
+    # each option keeps tuning its own numbers. Helps problems where the best choice is found
+    # late, on large budgets; costs on small ones. 0 turns niches off.
+    niche_share: float = Field(default=0.0, ge=0, lt=1)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _defaults_from_budget(cls, data: Any) -> Any:
+        if not isinstance(data, dict) or data.get("max_evaluations") is None:
+            return data
+        budget = data["max_evaluations"]
+        data = dict(data)
+        if data.get("population_size") is None:
+            population = min(max(budget // GENERATIONS_PER_BUDGET, MIN_POPULATION), MAX_POPULATION)
+            data["population_size"] = max(population, data.get("tournament_size") or 0)
+        if data.get("generations") is None:
+            data["generations"] = budget
+        return data
+
+    @model_validator(mode="after")
+    def _check_tournament_size(self):
+        if self.tournament_size > self.population_size:
+            raise ValueError(
+                f"tournament_size {self.tournament_size} cannot exceed population_size {self.population_size}"
+            )
+        return self

hone/config/optimizer_config.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Self
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class OptimizerConfig(BaseModel):
+    """Settings shared by every optimizer."""
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    population_size: int = Field(default=100, ge=2)  # individuals evaluated per generation
+    generations: int = Field(default=25, ge=1)
+    max_evaluations: int | None = Field(default=None, ge=1)  # stop after this many evaluations, even mid-generation
+    maximize: bool = False
+    threshold: float | None = None  # stop once the best fitness reaches this
+    seed: int | None = None  # fixes the random number generator, so runs can be repeated
+    noise: bool = False  # the objective is noisy: evaluating the same point twice can give different values
+    # With noise, evaluations kept back to re-evaluate the most promising points at the end, so
+    # the result is chosen, and its value reported, from averages rather than single lucky values
+    final_evaluations: int = Field(default=0, ge=0)
+    # The share of each generation's evaluations that must finish before the next generation is
+    # bred. Below 1, the slowest evaluations don't hold up the run: they finish in the background,
+    # and their results join the generation under way when they arrive. Results then depend on
+    # timing, so runs are no longer exactly repeatable.
+    min_finished: float = Field(default=1.0, gt=0, le=1)
+
+    @model_validator(mode="after")
+    def _check_final_evaluations(self):
+        if self.final_evaluations and self.max_evaluations is not None and self.final_evaluations >= self.max_evaluations:
+            raise ValueError(
+                f"final_evaluations {self.final_evaluations} leaves nothing of max_evaluations {self.max_evaluations} for the search"
+            )
+        return self
+
+    @property
+    def reserved_evaluations(self) -> int:
+        """Evaluations kept back for the final re-evaluation: none without noise."""
+        return self.final_evaluations if self.noise else 0
+
+    @classmethod
+    def from_file(cls, filepath: str | Path) -> Self:
+        """Create a config from a JSON or YAML file."""
+        filepath = Path(filepath)
+        text = filepath.read_text()
+        if filepath.suffix.lower() in {".yaml", ".yml"}:
+            return cls.model_validate(yaml.safe_load(text) or {})
+        return cls.model_validate_json(text)

hone/objective_function/__init__.py

@@ -0,0 +1 @@
+from .objective_function import ObjectiveFunction

hone/objective_function/objective_function.py

@@ -0,0 +1,47 @@
+import inspect
+import numbers
+from typing import Any, Callable
+
+from pydantic import BaseModel, ConfigDict
+
+from hone.parameters import SearchSpace
+
+
+class ObjectiveFunction(BaseModel):
+    """Objective function to be optimized, paired with the search space it is called over."""
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    function: Callable
+    search_space: SearchSpace
+
+    def model_post_init(self, context: Any) -> None:
+        # Checked after validation rather than in a validator, so a mismatch raises a plain ValueError.
+        self._check_signature()
+
+    def __call__(self, params: dict[str, Any]) -> float:
+        """Evaluate the function at params, which must return a real number."""
+        value = self.function(**params)
+        if not isinstance(value, numbers.Real):
+            raise TypeError(f"{self.name}() must return a number, but returned {value!r}")
+        return float(value)
+
+    def _check_signature(self):
+        """Fail before optimizing if the function cannot be called with the search space's parameters."""
+        signature = inspect.signature(self.function)
+        accepted = signature.parameters
+        takes_any_keyword = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in accepted.values())
+
+        names = set(self.search_space.names)
+        unknown = sorted(name for name in names if name not in accepted and not takes_any_keyword)
+        missing = sorted(
+            name
+            for name, p in accepted.items()
+            if name not in names
+            and p.default is inspect.Parameter.empty
+            and p.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
+        )
+        if unknown:
+            raise ValueError(f"{self.name}() does not accept parameters: {', '.join(unknown)}")
+        if missing:
+            raise ValueError(f"{self.name}() requires parameters missing from the search space: {', '.join(missing)}")

hone/optimizers/__init__.py

@@ -0,0 +1,6 @@
+from .bayesian import BayesianOptimizer
+from .base import EvaluationFailed, Individual, Optimizer, Trial, UnsupportedSearchSpace
+from .choice import choose_optimizer
+from .cmaes import CMAES
+from .genetic import GeneticOptimizer
+from .random_search import RandomSearch

hone/optimizers/_gp.py

@@ -0,0 +1,134 @@
+"""Gaussian process regression, the model behind Bayesian optimization.
+
+The kernel is Matérn 5/2 with one length scale per input, so the model can learn that some
+inputs matter more than others. Inputs are expected in [0, 1]; outputs are standardized.
+Hyperparameters are chosen by maximizing the marginal likelihood times weak priors (MAP).
+"""
+import math
+
+import numpy as np
+from scipy.linalg import cho_factor, cho_solve, solve_triangular
+from scipy.optimize import minimize
+
+SQRT5 = math.sqrt(5)
+
+# Bounds on the log hyperparameters: length scales, signal variance, noise variance.
+LOG_LENGTH = (math.log(0.01), math.log(20.0))
+LOG_SIGNAL = (math.log(0.05), math.log(20.0))
+LOG_NOISE = (math.log(1e-6), math.log(1.0))
+
+# Weak priors on the log hyperparameters, as (mean, standard deviation) of a normal.
+LENGTH_PRIOR = (math.log(0.3), 1.0)  # inputs are in [0, 1], so most length scales are a fraction of that
+SIGNAL_PRIOR = (0.0, 1.0)  # outputs are standardized
+NOISE_PRIOR = (math.log(1e-4), 2.0)  # little noise unless the data says otherwise
+
+
+def _matern(r: np.ndarray) -> np.ndarray:
+    return (1 + SQRT5 * r + 5 / 3 * r**2) * np.exp(-SQRT5 * r)
+
+
+def _scaled_distances(a: np.ndarray, b: np.ndarray, length: np.ndarray) -> np.ndarray:
+    a, b = a / length, b / length
+    squared = (a**2).sum(1)[:, None] + (b**2).sum(1)[None, :] - 2 * a @ b.T
+    return np.sqrt(np.maximum(squared, 0))
+
+
+class GaussianProcess:
+    """A Gaussian process fitted to inputs X, shape (n, d) in [0, 1], and outputs y, shape (n,)."""
+
+    def __init__(self, X: np.ndarray, y: np.ndarray, theta: np.ndarray | None = None) -> None:
+        self.X = np.asarray(X, dtype=float)
+        y = np.asarray(y, dtype=float)
+        self._offset = y.mean()
+        self._scale = y.std() if y.std() > 0 else 1.0
+        self.z = (y - self._offset) / self._scale
+        d = self.X.shape[1]
+        self.theta = theta if theta is not None else np.concatenate([np.full(d, LENGTH_PRIOR[0]), [0.0, math.log(1e-4)]])
+        self._condition()
+
+    # --- hyperparameters ------------------------------------------------------
+
+    def fit(self, restarts: int = 0, rng: np.random.Generator | None = None) -> "GaussianProcess":
+        """Choose hyperparameters by MAP, starting from the current ones and restarts random others."""
+        d = self.X.shape[1]
+        bounds = [LOG_LENGTH] * d + [LOG_SIGNAL, LOG_NOISE]
+        starts = [self.theta]
+        for _ in range(restarts):
+            assert rng is not None, "restarts need a random number generator"
+            starts.append(np.array([rng.uniform(low, high) for low, high in bounds]))
+        fits = [
+            minimize(self._objective, np.clip(start, *np.array(bounds).T), jac=True, method="L-BFGS-B", bounds=bounds)
+            for start in starts
+        ]
+        self.theta = min(fits, key=lambda found: found.fun).x
+        self._condition()
+        return self
+
+    def _objective(self, theta: np.ndarray) -> tuple[float, np.ndarray]:
+        """The negative log posterior of theta, and its gradient."""
+        d = self.X.shape[1]
+        length, signal, noise = np.exp(theta[:d]), math.exp(theta[d]), math.exp(theta[d + 1])
+        r = _scaled_distances(self.X, self.X, length)
+        correlation = _matern(r)
+        K = signal * correlation + (noise + 1e-10) * np.eye(len(self.X))
+        try:
+            factor = cho_factor(K, lower=True)
+        except np.linalg.LinAlgError:
+            return 1e10, np.zeros_like(theta)
+        alpha = cho_solve(factor, self.z)
+        L = factor[0]
+        nll = 0.5 * self.z @ alpha + np.log(np.diag(L)).sum() + 0.5 * len(self.z) * math.log(2 * math.pi)
+
+        W = np.outer(alpha, alpha) - cho_solve(factor, np.eye(len(self.X)))
+        # d k / d log(length_j) = signal * 5/3 * (1 + sqrt5 r) exp(-sqrt5 r) * (dx_j / length_j)^2
+        S = W * (signal * 5 / 3 * (1 + SQRT5 * r) * np.exp(-SQRT5 * r))
+        scaled = self.X / length
+        length_gradient = 2 * (S.sum(1) @ scaled**2) - 2 * np.einsum("ij,ik,kj->j", scaled, S, scaled)
+        gradient = -0.5 * np.concatenate(
+            [length_gradient, [np.sum(W * signal * correlation), noise * np.trace(W)]]
+        )
+
+        for index, (mean, sd) in [(slice(0, d), LENGTH_PRIOR), (d, SIGNAL_PRIOR), (d + 1, NOISE_PRIOR)]:
+            nll += 0.5 * np.sum(((theta[index] - mean) / sd) ** 2)
+            gradient[index] += (theta[index] - mean) / sd**2
+        return float(nll), gradient
+
+    # --- prediction -----------------------------------------------------------
+
+    def _condition(self) -> None:
+        d = self.X.shape[1]
+        self._length, self._signal = np.exp(self.theta[:d]), math.exp(self.theta[d])
+        noise = math.exp(self.theta[d + 1])
+        K = self._signal * _matern(_scaled_distances(self.X, self.X, self._length))
+        K[np.diag_indices_from(K)] += noise + 1e-10
+        jitter = 0.0
+        while True:
+            try:
+                self._L = np.linalg.cholesky(K + jitter * np.eye(len(K)))
+                break
+            except np.linalg.LinAlgError:
+                jitter = max(1e-8, jitter * 10)
+        self._alpha = cho_solve((self._L, True), self.z)
+        # Predictions multiply by the inverse of L, which is faster than solving with L each time.
+        self._L_inverse = solve_triangular(self._L, np.eye(len(K)), lower=True)
+
+    def predict(self, Xs: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+        """Posterior mean and standard deviation at Xs, in standardized units."""
+        k = self._signal * _matern(_scaled_distances(np.atleast_2d(Xs), self.X, self._length))
+        mean = k @ self._alpha
+        v = self._L_inverse @ k.T
+        variance = np.maximum(self._signal - (v**2).sum(0), 1e-12)
+        return mean, np.sqrt(variance)
+
+    def standardize(self, y: float | np.ndarray) -> float | np.ndarray:
+        return (np.asarray(y) - self._offset) / self._scale
+
+    def with_observation(self, x: np.ndarray, z: float) -> "GaussianProcess":
+        """This model with one more observation, given in standardized units, and the same hyperparameters."""
+        model = object.__new__(GaussianProcess)
+        model.X = np.vstack([self.X, x])
+        model._offset, model._scale = self._offset, self._scale
+        model.z = np.append(self.z, z)
+        model.theta = self.theta
+        model._condition()
+        return model