hone-python 0.14.0__tar.gz → 0.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {hone_python-0.14.0 → hone_python-0.15.0}/PKG-INFO +42 -12
  2. {hone_python-0.14.0 → hone_python-0.15.0}/README.md +39 -9
  3. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/__init__.py +4 -0
  4. hone_python-0.15.0/hone/benchmarks/__main__.py +255 -0
  5. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/constrained.py +16 -9
  6. hone_python-0.15.0/hone/benchmarks/cost.py +530 -0
  7. hone_python-0.15.0/hone/benchmarks/engineering.py +92 -0
  8. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/multi.py +127 -19
  9. hone_python-0.15.0/hone/benchmarks/multi_outputs.py +312 -0
  10. hone_python-0.15.0/hone/benchmarks/outputs.py +276 -0
  11. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/problems.py +2 -1
  12. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/runner.py +188 -29
  13. hone_python-0.15.0/hone/benchmarks/sharded.py +150 -0
  14. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/multi_objective_config.py +8 -4
  15. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/optimizer_config.py +24 -4
  16. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/_gp.py +41 -6
  17. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/_pool.py +37 -0
  18. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/base.py +113 -43
  19. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/bayesian.py +36 -16
  20. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/choice.py +16 -2
  21. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/core.py +142 -8
  22. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/nsga2.py +6 -1
  23. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/multi.py +224 -6
  24. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/multi_bayesian.py +17 -2
  25. {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/constraints.py +12 -0
  26. {hone_python-0.14.0 → hone_python-0.15.0}/hone/sensitivity_analysis.py +3 -1
  27. {hone_python-0.14.0 → hone_python-0.15.0}/pyproject.toml +14 -9
  28. hone_python-0.14.0/hone/benchmarks/__main__.py +0 -121
  29. {hone_python-0.14.0 → hone_python-0.15.0}/LICENSE +0 -0
  30. {hone_python-0.14.0 → hone_python-0.15.0}/hone/__init__.py +0 -0
  31. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/conditional.py +0 -0
  32. {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/ml.py +0 -0
  33. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/__init__.py +0 -0
  34. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/bayesian_config.py +0 -0
  35. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/cmaes_config.py +0 -0
  36. {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/genetic_optimizer_config.py +0 -0
  37. {hone_python-0.14.0 → hone_python-0.15.0}/hone/objective_function/__init__.py +0 -0
  38. {hone_python-0.14.0 → hone_python-0.15.0}/hone/objective_function/objective_function.py +0 -0
  39. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/__init__.py +0 -0
  40. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/cmaes.py +0 -0
  41. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/__init__.py +0 -0
  42. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/breeding.py +0 -0
  43. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/optimizer.py +0 -0
  44. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/selection.py +0 -0
  45. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/variation.py +0 -0
  46. {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/random_search.py +0 -0
  47. {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/__init__.py +0 -0
  48. {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/conditions.py +0 -0
  49. {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/parameters.py +0 -0
  50. {hone_python-0.14.0 → hone_python-0.15.0}/hone/pareto.py +0 -0
  51. {hone_python-0.14.0 → hone_python-0.15.0}/hone/py.typed +0 -0
  52. {hone_python-0.14.0 → hone_python-0.15.0}/hone/result.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hone-python
3
- Version: 0.14.0
3
+ Version: 0.15.0
4
4
  Summary: A python-based optimization engine/library.
5
5
  License-Expression: BSD-3-Clause
6
6
  License-File: LICENSE
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.13
13
13
  Classifier: Programming Language :: Python :: 3.14
14
14
  Classifier: Programming Language :: Python :: 3.15
15
15
  Requires-Dist: numpy (>=1.26.2)
16
- Requires-Dist: pydantic (>=2.7,<3.0.0)
16
+ Requires-Dist: pydantic (>=2.14.4,<3.0.0)
17
17
  Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
18
18
  Requires-Dist: scipy (>=1.13)
19
- Requires-Dist: threadpoolctl (>=3.5)
19
+ Requires-Dist: threadpoolctl (>=3.8.0)
20
20
  Project-URL: Changelog, https://github.com/GreatEmuWar/hone/blob/main/CHANGELOG.md
21
21
  Project-URL: Documentation, https://hone-python.dev/
22
22
  Project-URL: Source, https://github.com/GreatEmuWar/hone
@@ -58,7 +58,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
58
58
 
59
59
  ### Slow evaluations
60
60
 
61
- When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and on macOS and Windows the calling script needs an `if __name__ == "__main__":` guard.
61
+ When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
62
62
 
63
63
  ```python
64
64
  result = optimizer.optimize(run_simulation, workers=8, timeout=600)
@@ -127,7 +127,7 @@ After each generation the run's whole state is saved to the file. Run the same s
127
127
 
128
128
  With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
129
129
 
130
- A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
130
+ A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
131
131
 
132
132
  ### Failed evaluations
133
133
 
@@ -151,7 +151,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
151
151
 
152
152
  - Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
153
153
  - Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
154
- - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
154
+ - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
155
155
 
156
156
  `python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
157
157
 
@@ -244,7 +244,7 @@ With constraints on outputs, the objective returns a dict: its value under `"obj
244
244
 
245
245
  A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
246
246
 
247
- Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). Constraints on outputs aren't supported with `noise=True` yet, since a noisy design can pass a limit by luck. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
247
+ Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
248
248
 
249
249
  ### Conditional parameters
250
250
 
@@ -331,7 +331,7 @@ result.save("run.json")
331
331
  result = Result.load("run.json")
332
332
  ```
333
333
 
334
- `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result; for CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
334
+ `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
335
335
 
336
336
  ```python
337
337
  optimizer = Result.load("run.json").rebuild_optimizer()
@@ -370,8 +370,11 @@ result = optimizer.optimize(run_simulation)
370
370
  | Budget | Parameters | Chosen |
371
371
  | --- | --- | --- |
372
372
  | 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
373
- | More than 200 | Any choice or integer parameter | `GeneticOptimizer` |
374
- | More than 200 | Continuous numbers only | `CMAES` |
373
+ | 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
374
+ | Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
375
+ | Past the cutoff | Continuous numbers only | `CMAES` |
376
+
377
+ With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
375
378
 
376
379
  Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
377
380
 
@@ -412,7 +415,32 @@ print(choice.params, choice.values)
412
415
 
413
416
  `choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
414
417
 
415
- Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize`, `threshold` and `final_evaluations` don't apply.
418
+ Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
419
+
420
+ #### Noise with constraints on outputs
421
+
422
+ With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
423
+
424
+ ```python
425
+ from hone import NSGA2, NSGA2Config, Output, SearchSpace
426
+
427
+ search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
428
+ config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
429
+ result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
430
+ ```
431
+
432
+ Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
433
+
434
+ By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
435
+
436
+ What the rule guarantees, and what it doesn't:
437
+
438
+ - A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
439
+ - Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
440
+ - Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
441
+ - Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
442
+ - Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
443
+ - With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
416
444
 
417
445
  [examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
418
446
 
@@ -499,7 +527,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
499
527
 
500
528
  | Command | Does |
501
529
  | --- | --- |
502
- | `make test` | Install the dependencies and run the tests with coverage |
530
+ | `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
503
531
  | `make lint` | Run ruff, then mypy to check types |
504
532
  | `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
505
533
  | `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
@@ -508,6 +536,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
508
536
  | `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
509
537
  | `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
510
538
  | `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
539
+ | `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
540
+ | `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
511
541
  | `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
512
542
  | `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
513
543
 
@@ -34,7 +34,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
34
34
 
35
35
  ### Slow evaluations
36
36
 
37
- When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and on macOS and Windows the calling script needs an `if __name__ == "__main__":` guard.
37
+ When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
38
38
 
39
39
  ```python
40
40
  result = optimizer.optimize(run_simulation, workers=8, timeout=600)
@@ -103,7 +103,7 @@ After each generation the run's whole state is saved to the file. Run the same s
103
103
 
104
104
  With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
105
105
 
106
- A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
106
+ A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
107
107
 
108
108
  ### Failed evaluations
109
109
 
@@ -127,7 +127,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
127
127
 
128
128
  - Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
129
129
  - Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
130
- - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
130
+ - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
131
131
 
132
132
  `python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
133
133
 
@@ -220,7 +220,7 @@ With constraints on outputs, the objective returns a dict: its value under `"obj
220
220
 
221
221
  A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
222
222
 
223
- Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). Constraints on outputs aren't supported with `noise=True` yet, since a noisy design can pass a limit by luck. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
223
+ Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
224
224
 
225
225
  ### Conditional parameters
226
226
 
@@ -307,7 +307,7 @@ result.save("run.json")
307
307
  result = Result.load("run.json")
308
308
  ```
309
309
 
310
- `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result; for CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
310
+ `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
311
311
 
312
312
  ```python
313
313
  optimizer = Result.load("run.json").rebuild_optimizer()
@@ -346,8 +346,11 @@ result = optimizer.optimize(run_simulation)
346
346
  | Budget | Parameters | Chosen |
347
347
  | --- | --- | --- |
348
348
  | 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
349
- | More than 200 | Any choice or integer parameter | `GeneticOptimizer` |
350
- | More than 200 | Continuous numbers only | `CMAES` |
349
+ | 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
350
+ | Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
351
+ | Past the cutoff | Continuous numbers only | `CMAES` |
352
+
353
+ With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
351
354
 
352
355
  Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
353
356
 
@@ -388,7 +391,32 @@ print(choice.params, choice.values)
388
391
 
389
392
  `choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
390
393
 
391
- Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize`, `threshold` and `final_evaluations` don't apply.
394
+ Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
395
+
396
+ #### Noise with constraints on outputs
397
+
398
+ With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
399
+
400
+ ```python
401
+ from hone import NSGA2, NSGA2Config, Output, SearchSpace
402
+
403
+ search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
404
+ config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
405
+ result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
406
+ ```
407
+
408
+ Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
409
+
410
+ By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
411
+
412
+ What the rule guarantees, and what it doesn't:
413
+
414
+ - A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
415
+ - Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
416
+ - Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
417
+ - Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
418
+ - Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
419
+ - With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
392
420
 
393
421
  [examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
394
422
 
@@ -475,7 +503,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
475
503
 
476
504
  | Command | Does |
477
505
  | --- | --- |
478
- | `make test` | Install the dependencies and run the tests with coverage |
506
+ | `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
479
507
  | `make lint` | Run ruff, then mypy to check types |
480
508
  | `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
481
509
  | `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
@@ -484,6 +512,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
484
512
  | `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
485
513
  | `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
486
514
  | `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
515
+ | `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
516
+ | `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
487
517
  | `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
488
518
  | `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
489
519
 
@@ -1,8 +1,11 @@
1
1
  """Benchmark problems and a runner for comparing optimizers. Run `python -m hone.benchmarks --help`."""
2
2
  from .conditional import conditional_problems
3
3
  from .constrained import constrained_problems
4
+ from .engineering import engineering_problems, engineering_table
4
5
  from .ml import ml_problems
5
6
  from .multi import MultiProblem, multi_problems, run_multi_benchmark, summarize_multi
7
+ from .multi_outputs import cantilever_outputs, noisy_multi_output_problems, osy, tnk
8
+ from .outputs import noisy_output_problems, output_problems
6
9
  from .problems import Problem, noisy_problems, quick_problems, standard_problems
7
10
  from .runner import (
8
11
  BUDGETS,
@@ -10,6 +13,7 @@ from .runner import (
10
13
  Run,
11
14
  SkipBudget,
12
15
  Summary,
16
+ batch_optimizers,
13
17
  load_summaries,
14
18
  regressions,
15
19
  run_benchmark,
@@ -0,0 +1,255 @@
1
+ """Command-line entry point: `python -m hone.benchmarks`."""
2
+ import argparse
3
+ import sys
4
+ import time
5
+ from collections.abc import Sequence
6
+ from importlib.metadata import version
7
+ from pathlib import Path
8
+
9
+ from . import cost, engineering, ml
10
+ from .conditional import conditional_problems
11
+ from .multi import multi_markdown, multi_problems, run_multi_benchmark, summarize_multi
12
+ from .constrained import constrained_problems
13
+ from .outputs import noisy_output_problems, output_problems
14
+ from .problems import noisy_problems, quick_problems, standard_problems
15
+ from .runner import (
16
+ BUDGETS,
17
+ DEFAULT_OPTIMIZERS,
18
+ batch_optimizers,
19
+ load_summaries,
20
+ regressions,
21
+ run_benchmark,
22
+ save_summaries,
23
+ summarize,
24
+ to_markdown,
25
+ )
26
+
27
+
28
+ QUICK_BUDGETS = (10, 50, 200)
29
+ NOISY_BUDGETS = (10, 40, 200)
30
+ OUTPUT_BATCH_SIZES = (4, 8) # Bayesian optimization's batch sizes run on the problems with constraints on outputs, besides 1
31
+
32
+
33
+ def _join(numbers: list[int]) -> str:
34
+ """[10, 20, 50] -> "10, 20 and 50"."""
35
+ return _join_words([f"{n:,}" for n in numbers])
36
+
37
+
38
+ def _join_words(words: list[str]) -> str:
39
+ """["a", "b", "c"] -> "a, b and c"."""
40
+ return words[0] if len(words) == 1 else ", ".join(words[:-1]) + " and " + words[-1]
41
+
42
+
43
+ def main(argv: list[str] | None = None) -> int:
44
+ parser = argparse.ArgumentParser(prog="python -m hone.benchmarks", description=__doc__)
45
+ suite = parser.add_mutually_exclusive_group()
46
+ suite.add_argument("--quick", action="store_true", help="run the small CI subset with fewer seeds and evaluations")
47
+ suite.add_argument("--ml", action="store_true", help="run the machine learning tuning problems instead (needs scikit-learn)")
48
+ suite.add_argument("--noisy", action="store_true", help="run the problems with noise added to every evaluation instead")
49
+ suite.add_argument("--constrained", action="store_true", help="run the problems with constraints instead")
50
+ suite.add_argument(
51
+ "--outputs", action="store_true", help="run the problems with constraints on outputs instead, the noisy engineering designs included, with Bayesian optimization at batch sizes 1, 4 and 8"
52
+ )
53
+ suite.add_argument(
54
+ "--multi", action="store_true", help="run the problems with several objectives instead, scored by hypervolume"
55
+ )
56
+ suite.add_argument(
57
+ "--cost",
58
+ action="store_true",
59
+ help="time Bayesian optimization's candidates against evaluations and limits on outputs instead, with one profiled run; "
60
+ "--baseline then checks the ceiling at 500 evaluations",
61
+ )
62
+ suite.add_argument(
63
+ "--cost-multi",
64
+ action="store_true",
65
+ help="time MultiObjectiveBayesian's candidates (parego and models, 0 and 8 limits) as --cost does, without the "
66
+ "single-objective table or the profile; --noise times the noisy problem; --baseline checks the ceiling at 500 evaluations",
67
+ )
68
+ suite.add_argument(
69
+ "--conditional", action="store_true", help="run the problems with conditional parameters, and their flat versions, instead"
70
+ )
71
+ suite.add_argument(
72
+ "--engineering",
73
+ action="store_true",
74
+ help="run nothing: tabulate the engineering designs and g06, g07 and g10 against their published optima, "
75
+ "from saved summaries (see --summaries)",
76
+ )
77
+ parser.add_argument("--noise", action="store_true", help="with --cost-multi, also time the noisy problem (noise=True, which needs noise with limits), after the noise-free cells")
78
+ parser.add_argument("--seeds", type=int, help="number of seeds per problem (default 10, or 5 with --quick)")
79
+ parser.add_argument(
80
+ "--budgets",
81
+ type=int,
82
+ nargs="+",
83
+ help=f"evaluations per dimension, one run per budget (default {' '.join(map(str, BUDGETS))}, "
84
+ f"{' '.join(map(str, QUICK_BUDGETS))} with --quick, {' '.join(map(str, ml.BUDGETS))} with --ml, "
85
+ f"or {' '.join(map(str, NOISY_BUDGETS))} with --noisy)",
86
+ )
87
+ parser.add_argument(
88
+ "--batch-sizes",
89
+ type=int,
90
+ nargs="+",
91
+ metavar="SIZE",
92
+ help="also run Bayesian optimization choosing this many candidates per batch, as bayesian-SIZE (default 4 8 with --outputs)",
93
+ )
94
+ parser.add_argument("--jobs", type=int, default=1, help="share the runs among this many processes (Linux only)")
95
+ parser.add_argument("--tolerance", type=float, default=1e-4, help="error at or below which a run counts as solved")
96
+ parser.add_argument(
97
+ "--summaries",
98
+ nargs="+",
99
+ metavar="FILE",
100
+ default=list(engineering.SOURCES),
101
+ help="with --engineering, the saved summaries to read, a later file's results replacing an earlier one's "
102
+ f"(default {' '.join(engineering.SOURCES)})",
103
+ )
104
+ parser.add_argument("--markdown", help="write the results table to this file")
105
+ parser.add_argument("--save", help="write the summaries as JSON to this file, for use as a baseline")
106
+ parser.add_argument("--baseline", help="compare against summaries saved with --save; exit 1 on a regression")
107
+ args = parser.parse_args(argv)
108
+
109
+ if args.multi:
110
+ return _multi(args)
111
+ if args.cost:
112
+ return _cost(args)
113
+ if args.cost_multi:
114
+ return _cost_multi(args)
115
+ if args.engineering:
116
+ return _engineering(args)
117
+ default_budgets: Sequence[int]
118
+ if args.quick:
119
+ problems, default_budgets = quick_problems(), QUICK_BUDGETS
120
+ elif args.ml:
121
+ problems, default_budgets = ml.ml_problems(), ml.BUDGETS
122
+ elif args.noisy:
123
+ problems, default_budgets = noisy_problems(), NOISY_BUDGETS
124
+ elif args.constrained:
125
+ problems, default_budgets = constrained_problems(), BUDGETS
126
+ elif args.outputs:
127
+ problems, default_budgets = output_problems() + noisy_output_problems(), BUDGETS
128
+ elif args.conditional:
129
+ problems, default_budgets = conditional_problems(), BUDGETS
130
+ else:
131
+ problems, default_budgets = standard_problems(), BUDGETS
132
+ seeds = args.seeds or (5 if args.quick else 10)
133
+ budgets = sorted(set(args.budgets or default_budgets))
134
+ batch_sizes = args.batch_sizes if args.batch_sizes is not None else OUTPUT_BATCH_SIZES if args.outputs else ()
135
+ optimizers = {**DEFAULT_OPTIMIZERS, **batch_optimizers(batch_sizes)}
136
+
137
+ start = time.perf_counter()
138
+ runs = run_benchmark(problems, optimizers, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs)
139
+ summaries = summarize(runs, args.tolerance)
140
+ elapsed = time.perf_counter() - start
141
+
142
+ table = to_markdown(summaries, args.tolerance)
143
+ print(table)
144
+ setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
145
+ print(f"{setup}, in {elapsed:.0f}s")
146
+ if args.markdown:
147
+ with open(args.markdown, "w") as f:
148
+ f.write(
149
+ f"# Benchmark results\n\nGenerated by `python -m hone.benchmarks` with hone {version('hone-python')}: "
150
+ f"{setup}.\n\n{table}"
151
+ )
152
+ if args.save:
153
+ save_summaries(summaries, args.save)
154
+
155
+ if args.baseline:
156
+ found = regressions(summaries, load_summaries(args.baseline))
157
+ if found:
158
+ print("\nRegressions against the baseline:", *found, sep="\n ")
159
+ return 1
160
+ print("\nNo regressions against the baseline.")
161
+ return 0
162
+
163
+
164
+ def _multi(args: argparse.Namespace) -> int:
165
+ """The multi-objective suite: its own runs, scored by hypervolume gap."""
166
+ problems = multi_problems()
167
+ seeds = args.seeds or 10
168
+ budgets = sorted(set(args.budgets or BUDGETS))
169
+ start = time.perf_counter()
170
+ summaries = summarize_multi(run_multi_benchmark(problems, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs))
171
+ elapsed = time.perf_counter() - start
172
+ table = multi_markdown(summaries)
173
+ print(table)
174
+ setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
175
+ print(f"{setup}, in {elapsed:.0f}s")
176
+ if args.markdown:
177
+ with open(args.markdown, "w") as f:
178
+ f.write(f"# Multi-objective benchmark results\n\nGenerated by `python -m hone.benchmarks --multi` with hone {version('hone-python')}: {setup}.\n\n{table}")
179
+ if args.save:
180
+ with open(args.save, "w") as f:
181
+ f.write("[\n" + ",\n".join(s.model_dump_json() for s in summaries) + "\n]\n")
182
+ return 0
183
+
184
+
185
+ def _engineering(args: argparse.Namespace) -> int:
186
+ """The engineering designs against their published optima, from summaries already saved."""
187
+ summaries = [s for path in args.summaries for s in load_summaries(path)]
188
+ table = engineering.engineering_table(summaries)
189
+ print(table)
190
+ if args.markdown:
191
+ sources = _join_words([f"`{Path(path).name}`" for path in args.summaries])
192
+ with open(args.markdown, "w") as f:
193
+ f.write(
194
+ "# Engineering designs against their published optima\n\n"
195
+ f"Generated by `python -m hone.benchmarks --engineering` from {sources}, which the constrained and "
196
+ f"outputs suites save; nothing is run.\n\n{table}"
197
+ )
198
+ return 0
199
+
200
+
201
+ def _cost(args: argparse.Namespace) -> int:
202
+ """The cost of a Bayesian candidate: the table, one profiled run, and the ceiling against a saved table."""
203
+ print("Timing Bayesian candidates, one cell at a time:")
204
+ cells = cost.measure_cost(progress=True)
205
+ print("Profiling one run on g07:")
206
+ table = cost.CostTable(
207
+ hone=version("hone-python"),
208
+ machine=cost.machine(),
209
+ dimensions=cost.DIMENSIONS,
210
+ candidates=cost.CANDIDATES,
211
+ seeds=cost.SEEDS,
212
+ cells=cells,
213
+ profile=cost.profile_run(),
214
+ )
215
+ text = cost.cost_markdown(table)
216
+ print(text)
217
+ if args.markdown:
218
+ with open(args.markdown, "w") as f:
219
+ f.write(f"# The cost of a Bayesian candidate\n\nGenerated by `python -m hone.benchmarks --cost`.\n\n{text}")
220
+ if args.save:
221
+ cost.save_cost(table, args.save)
222
+ if args.baseline:
223
+ found = cost.ceiling_breaches(cells, cost.load_cost(args.baseline).cells)
224
+ if found:
225
+ print(f"\nOver the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations:", *found, sep="\n ")
226
+ return 1
227
+ print(f"\nWithin the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations.")
228
+ return 0
229
+
230
+
231
+ def _cost_multi(args: argparse.Namespace) -> int:
232
+ """The cost of a MultiObjectiveBayesian candidate: the table, and the ceiling against a saved table."""
233
+ print("Timing multi-objective Bayesian candidates, one cell at a time:")
234
+ table = cost.CostTable(
235
+ hone=version("hone-python"), machine=cost.machine(), dimensions=cost.DIMENSIONS, candidates=cost.CANDIDATES, seeds=cost.SEEDS,
236
+ cells=[], multi=cost.measure_multi_cost(progress=True) + (cost.measure_multi_cost(noise=True, progress=True) if args.noise else []),
237
+ )
238
+ text = cost.cost_markdown(table)
239
+ print(text)
240
+ if args.markdown:
241
+ with open(args.markdown, "w") as f:
242
+ f.write(f"# The cost of a multi-objective Bayesian candidate\n\nGenerated by `python -m hone.benchmarks --cost-multi`.\n\n{text}")
243
+ if args.save:
244
+ cost.save_cost(table, args.save)
245
+ if args.baseline:
246
+ found = cost.multi_ceiling_breaches(table.multi, cost.load_cost(args.baseline).multi)
247
+ if found:
248
+ print(f"\nOver the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations:", *found, sep="\n ")
249
+ return 1
250
+ print(f"\nWithin the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations.")
251
+ return 0
252
+
253
+
254
+ if __name__ == "__main__":
255
+ sys.exit(main())
@@ -6,8 +6,11 @@
6
6
  - rosenbrock-disk: Rosenbrock's function inside the disk x² + y² ≤ 2, whose optimum (1, 1) sits
7
7
  on the disk's edge.
8
8
  - pressure-vessel, tension-spring and welded-beam: three standard engineering design problems,
9
- scored against the best designs published for them. The welded beam's stress formulas need
10
- square roots, so its constraints are functions; the others are expressions.
9
+ scored against the best designs published for them, which Cagnina, Esquivel and Coello Coello
10
+ (2008), "Solving engineering optimization problems with the simple constrained particle swarm
11
+ optimizer", Informatica 32(3):319-326, list as problems E02, E04 and E01. The welded beam's
12
+ stress formulas need square roots, so its constraints are functions; the others are
13
+ expressions.
11
14
 
12
15
  Each engineering problem also comes with its limits on outputs, as the -outputs version: the
13
16
  objective reports the volume, stresses and so on that the limits apply to, as a simulation
@@ -81,7 +84,8 @@ def pressure_vessel() -> Problem:
81
84
  """A cylindrical vessel with hemispherical heads, at least 1,296,000 cubic inches, of least cost.
82
85
 
83
86
  Shell and head thicknesses come in sixteenths of an inch, so they are integers here. The best
84
- published design costs 6059.714 (Ts = 13/16, Th = 7/16, R = 42.098, L = 176.637).
87
+ published design costs 6059.714 (Ts = 13/16, Th = 7/16, R = 42.098, L = 176.637)
88
+ (Cagnina et al., 2008, problem E02).
85
89
  """
86
90
  shell, head = RangeParameter("shell", 1, 99, integer=True), RangeParameter("head", 1, 99, integer=True)
87
91
  radius, length = RangeParameter("radius", 10, 200), RangeParameter("length", 10, 200)
@@ -104,7 +108,14 @@ def pressure_vessel() -> Problem:
104
108
  def tension_spring() -> Problem:
105
109
  """A coil spring of least weight under limits on deflection, shear stress, surge frequency and diameter.
106
110
 
107
- The best published design weighs 0.0126652 (d = 0.051690, D = 0.356750, N = 11.287126).
111
+ The best published design weighs 0.0126652 (d = 0.051689, D = 0.356720, N = 11.288832).
112
+ Cagnina et al., 2008, problem E04, give the design (0.051690, 0.356750, 11.287126) with a
113
+ weight of 0.012665. Its rounded values break the shear stress limit by about 2e-5 (the stress
114
+ is 1.00002 times the limit), which is why they weigh 0.0126651, a little less than the optimum.
115
+ Celik and Kutucu, "Solving the tension/compression spring design problem by an improved
116
+ firefly algorithm", CEUR Workshop Proceedings 2255, paper 2, give the optimum as 0.0126652327883
117
+ at (0.051689156131, 0.356720026419, 11.288831695483), which satisfies every limit here. They
118
+ cite no source for it.
108
119
  """
109
120
  d, coil, n = RangeParameter("wire", 0.05, 2.0), RangeParameter("coil", 0.25, 1.3), RangeParameter("turns", 2.0, 15.0)
110
121
 
@@ -144,6 +155,7 @@ def welded_beam() -> Problem:
144
155
  """A beam welded to a support, of least cost under limits on shear, bending, buckling and deflection.
145
156
 
146
157
  The best published design costs 1.724852 (h = 0.205730, l = 3.470489, t = 9.036624, b = 0.205730).
158
+ Cagnina et al., 2008, problem E01, give the same cost and design with b = 0.205729.
147
159
  """
148
160
  weld, length = RangeParameter("weld", 0.1, 2.0), RangeParameter("length", 0.1, 10.0)
149
161
  thickness, width = RangeParameter("thickness", 0.1, 10.0), RangeParameter("width", 0.1, 2.0)
@@ -265,8 +277,3 @@ def constrained_problems(dimensions: tuple[int, ...] = (2, 5, 10)) -> list[Probl
265
277
  tension_spring_outputs(),
266
278
  welded_beam_outputs(),
267
279
  ]
268
-
269
-
270
- def output_problems() -> list[Problem]:
271
- """The engineering designs with their limits on outputs only, for a quick comparison with the parameter versions."""
272
- return [pressure_vessel_outputs(), tension_spring_outputs(), welded_beam_outputs()]