hone-python 0.14.0__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hone_python-0.14.0 → hone_python-0.15.0}/PKG-INFO +42 -12
- {hone_python-0.14.0 → hone_python-0.15.0}/README.md +39 -9
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/__init__.py +4 -0
- hone_python-0.15.0/hone/benchmarks/__main__.py +255 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/constrained.py +16 -9
- hone_python-0.15.0/hone/benchmarks/cost.py +530 -0
- hone_python-0.15.0/hone/benchmarks/engineering.py +92 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/multi.py +127 -19
- hone_python-0.15.0/hone/benchmarks/multi_outputs.py +312 -0
- hone_python-0.15.0/hone/benchmarks/outputs.py +276 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/problems.py +2 -1
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/runner.py +188 -29
- hone_python-0.15.0/hone/benchmarks/sharded.py +150 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/multi_objective_config.py +8 -4
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/optimizer_config.py +24 -4
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/_gp.py +41 -6
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/_pool.py +37 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/base.py +113 -43
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/bayesian.py +36 -16
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/choice.py +16 -2
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/core.py +142 -8
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/nsga2.py +6 -1
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/multi.py +224 -6
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/multi_bayesian.py +17 -2
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/constraints.py +12 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/sensitivity_analysis.py +3 -1
- {hone_python-0.14.0 → hone_python-0.15.0}/pyproject.toml +14 -9
- hone_python-0.14.0/hone/benchmarks/__main__.py +0 -121
- {hone_python-0.14.0 → hone_python-0.15.0}/LICENSE +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/conditional.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/benchmarks/ml.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/bayesian_config.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/cmaes_config.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/config/genetic_optimizer_config.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/objective_function/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/objective_function/objective_function.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/cmaes.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/breeding.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/optimizer.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/selection.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/genetic/variation.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/optimizers/random_search.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/__init__.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/conditions.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/parameters/parameters.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/pareto.py +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/py.typed +0 -0
- {hone_python-0.14.0 → hone_python-0.15.0}/hone/result.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hone-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: A python-based optimization engine/library.
|
|
5
5
|
License-Expression: BSD-3-Clause
|
|
6
6
|
License-File: LICENSE
|
|
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.15
|
|
15
15
|
Requires-Dist: numpy (>=1.26.2)
|
|
16
|
-
Requires-Dist: pydantic (>=2.
|
|
16
|
+
Requires-Dist: pydantic (>=2.14.4,<3.0.0)
|
|
17
17
|
Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
|
|
18
18
|
Requires-Dist: scipy (>=1.13)
|
|
19
|
-
Requires-Dist: threadpoolctl (>=3.
|
|
19
|
+
Requires-Dist: threadpoolctl (>=3.8.0)
|
|
20
20
|
Project-URL: Changelog, https://github.com/GreatEmuWar/hone/blob/main/CHANGELOG.md
|
|
21
21
|
Project-URL: Documentation, https://hone-python.dev/
|
|
22
22
|
Project-URL: Source, https://github.com/GreatEmuWar/hone
|
|
@@ -58,7 +58,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
|
|
|
58
58
|
|
|
59
59
|
### Slow evaluations
|
|
60
60
|
|
|
61
|
-
When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and
|
|
61
|
+
When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
|
|
62
62
|
|
|
63
63
|
```python
|
|
64
64
|
result = optimizer.optimize(run_simulation, workers=8, timeout=600)
|
|
@@ -127,7 +127,7 @@ After each generation the run's whole state is saved to the file. Run the same s
|
|
|
127
127
|
|
|
128
128
|
With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
|
|
129
129
|
|
|
130
|
-
A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
|
|
130
|
+
A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
|
|
131
131
|
|
|
132
132
|
### Failed evaluations
|
|
133
133
|
|
|
@@ -151,7 +151,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
|
|
|
151
151
|
|
|
152
152
|
- Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
|
|
153
153
|
- Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
|
|
154
|
-
- `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
|
|
154
|
+
- `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
|
|
155
155
|
|
|
156
156
|
`python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
|
|
157
157
|
|
|
@@ -244,7 +244,7 @@ With constraints on outputs, the objective returns a dict: its value under `"obj
|
|
|
244
244
|
|
|
245
245
|
A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
|
|
246
246
|
|
|
247
|
-
Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs).
|
|
247
|
+
Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
|
|
248
248
|
|
|
249
249
|
### Conditional parameters
|
|
250
250
|
|
|
@@ -331,7 +331,7 @@ result.save("run.json")
|
|
|
331
331
|
result = Result.load("run.json")
|
|
332
332
|
```
|
|
333
333
|
|
|
334
|
-
`rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result
|
|
334
|
+
`rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
|
|
335
335
|
|
|
336
336
|
```python
|
|
337
337
|
optimizer = Result.load("run.json").rebuild_optimizer()
|
|
@@ -370,8 +370,11 @@ result = optimizer.optimize(run_simulation)
|
|
|
370
370
|
| Budget | Parameters | Chosen |
|
|
371
371
|
| --- | --- | --- |
|
|
372
372
|
| 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
|
|
373
|
-
|
|
|
374
|
-
|
|
|
373
|
+
| 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
|
|
374
|
+
| Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
|
|
375
|
+
| Past the cutoff | Continuous numbers only | `CMAES` |
|
|
376
|
+
|
|
377
|
+
With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
|
|
375
378
|
|
|
376
379
|
Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
|
|
377
380
|
|
|
@@ -412,7 +415,32 @@ print(choice.params, choice.values)
|
|
|
412
415
|
|
|
413
416
|
`choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
|
|
414
417
|
|
|
415
|
-
Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize
|
|
418
|
+
Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
|
|
419
|
+
|
|
420
|
+
#### Noise with constraints on outputs
|
|
421
|
+
|
|
422
|
+
With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
|
|
423
|
+
|
|
424
|
+
```python
|
|
425
|
+
from hone import NSGA2, NSGA2Config, Output, SearchSpace
|
|
426
|
+
|
|
427
|
+
search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
|
|
428
|
+
config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
|
|
429
|
+
result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
|
|
433
|
+
|
|
434
|
+
By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
|
|
435
|
+
|
|
436
|
+
What the rule guarantees, and what it doesn't:
|
|
437
|
+
|
|
438
|
+
- A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
|
|
439
|
+
- Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
|
|
440
|
+
- Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
|
|
441
|
+
- Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
|
|
442
|
+
- Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
|
|
443
|
+
- With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
|
|
416
444
|
|
|
417
445
|
[examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
|
|
418
446
|
|
|
@@ -499,7 +527,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
|
|
|
499
527
|
|
|
500
528
|
| Command | Does |
|
|
501
529
|
| --- | --- |
|
|
502
|
-
| `make test` | Install the dependencies and run the tests
|
|
530
|
+
| `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
|
|
503
531
|
| `make lint` | Run ruff, then mypy to check types |
|
|
504
532
|
| `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
|
|
505
533
|
| `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
|
|
@@ -508,6 +536,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
|
|
|
508
536
|
| `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
|
|
509
537
|
| `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
|
|
510
538
|
| `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
|
|
539
|
+
| `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
|
|
540
|
+
| `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
|
|
511
541
|
| `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
|
|
512
542
|
| `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
|
|
513
543
|
|
|
@@ -34,7 +34,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
|
|
|
34
34
|
|
|
35
35
|
### Slow evaluations
|
|
36
36
|
|
|
37
|
-
When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and
|
|
37
|
+
When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
|
|
38
38
|
|
|
39
39
|
```python
|
|
40
40
|
result = optimizer.optimize(run_simulation, workers=8, timeout=600)
|
|
@@ -103,7 +103,7 @@ After each generation the run's whole state is saved to the file. Run the same s
|
|
|
103
103
|
|
|
104
104
|
With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
|
|
105
105
|
|
|
106
|
-
A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
|
|
106
|
+
A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
|
|
107
107
|
|
|
108
108
|
### Failed evaluations
|
|
109
109
|
|
|
@@ -127,7 +127,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
|
|
|
127
127
|
|
|
128
128
|
- Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
|
|
129
129
|
- Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
|
|
130
|
-
- `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
|
|
130
|
+
- `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
|
|
131
131
|
|
|
132
132
|
`python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
|
|
133
133
|
|
|
@@ -220,7 +220,7 @@ With constraints on outputs, the objective returns a dict: its value under `"obj
|
|
|
220
220
|
|
|
221
221
|
A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
|
|
222
222
|
|
|
223
|
-
Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs).
|
|
223
|
+
Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
|
|
224
224
|
|
|
225
225
|
### Conditional parameters
|
|
226
226
|
|
|
@@ -307,7 +307,7 @@ result.save("run.json")
|
|
|
307
307
|
result = Result.load("run.json")
|
|
308
308
|
```
|
|
309
309
|
|
|
310
|
-
`rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result
|
|
310
|
+
`rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
|
|
311
311
|
|
|
312
312
|
```python
|
|
313
313
|
optimizer = Result.load("run.json").rebuild_optimizer()
|
|
@@ -346,8 +346,11 @@ result = optimizer.optimize(run_simulation)
|
|
|
346
346
|
| Budget | Parameters | Chosen |
|
|
347
347
|
| --- | --- | --- |
|
|
348
348
|
| 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
|
|
349
|
-
|
|
|
350
|
-
|
|
|
349
|
+
| 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
|
|
350
|
+
| Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
|
|
351
|
+
| Past the cutoff | Continuous numbers only | `CMAES` |
|
|
352
|
+
|
|
353
|
+
With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
|
|
351
354
|
|
|
352
355
|
Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
|
|
353
356
|
|
|
@@ -388,7 +391,32 @@ print(choice.params, choice.values)
|
|
|
388
391
|
|
|
389
392
|
`choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
|
|
390
393
|
|
|
391
|
-
Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize
|
|
394
|
+
Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
|
|
395
|
+
|
|
396
|
+
#### Noise with constraints on outputs
|
|
397
|
+
|
|
398
|
+
With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
|
|
399
|
+
|
|
400
|
+
```python
|
|
401
|
+
from hone import NSGA2, NSGA2Config, Output, SearchSpace
|
|
402
|
+
|
|
403
|
+
search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
|
|
404
|
+
config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
|
|
405
|
+
result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
|
|
409
|
+
|
|
410
|
+
By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
|
|
411
|
+
|
|
412
|
+
What the rule guarantees, and what it doesn't:
|
|
413
|
+
|
|
414
|
+
- A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
|
|
415
|
+
- Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
|
|
416
|
+
- Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
|
|
417
|
+
- Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
|
|
418
|
+
- Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
|
|
419
|
+
- With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
|
|
392
420
|
|
|
393
421
|
[examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
|
|
394
422
|
|
|
@@ -475,7 +503,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
|
|
|
475
503
|
|
|
476
504
|
| Command | Does |
|
|
477
505
|
| --- | --- |
|
|
478
|
-
| `make test` | Install the dependencies and run the tests
|
|
506
|
+
| `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
|
|
479
507
|
| `make lint` | Run ruff, then mypy to check types |
|
|
480
508
|
| `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
|
|
481
509
|
| `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
|
|
@@ -484,6 +512,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
|
|
|
484
512
|
| `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
|
|
485
513
|
| `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
|
|
486
514
|
| `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
|
|
515
|
+
| `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
|
|
516
|
+
| `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
|
|
487
517
|
| `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
|
|
488
518
|
| `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
|
|
489
519
|
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
"""Benchmark problems and a runner for comparing optimizers. Run `python -m hone.benchmarks --help`."""
|
|
2
2
|
from .conditional import conditional_problems
|
|
3
3
|
from .constrained import constrained_problems
|
|
4
|
+
from .engineering import engineering_problems, engineering_table
|
|
4
5
|
from .ml import ml_problems
|
|
5
6
|
from .multi import MultiProblem, multi_problems, run_multi_benchmark, summarize_multi
|
|
7
|
+
from .multi_outputs import cantilever_outputs, noisy_multi_output_problems, osy, tnk
|
|
8
|
+
from .outputs import noisy_output_problems, output_problems
|
|
6
9
|
from .problems import Problem, noisy_problems, quick_problems, standard_problems
|
|
7
10
|
from .runner import (
|
|
8
11
|
BUDGETS,
|
|
@@ -10,6 +13,7 @@ from .runner import (
|
|
|
10
13
|
Run,
|
|
11
14
|
SkipBudget,
|
|
12
15
|
Summary,
|
|
16
|
+
batch_optimizers,
|
|
13
17
|
load_summaries,
|
|
14
18
|
regressions,
|
|
15
19
|
run_benchmark,
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Command-line entry point: `python -m hone.benchmarks`."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import cost, engineering, ml
|
|
10
|
+
from .conditional import conditional_problems
|
|
11
|
+
from .multi import multi_markdown, multi_problems, run_multi_benchmark, summarize_multi
|
|
12
|
+
from .constrained import constrained_problems
|
|
13
|
+
from .outputs import noisy_output_problems, output_problems
|
|
14
|
+
from .problems import noisy_problems, quick_problems, standard_problems
|
|
15
|
+
from .runner import (
|
|
16
|
+
BUDGETS,
|
|
17
|
+
DEFAULT_OPTIMIZERS,
|
|
18
|
+
batch_optimizers,
|
|
19
|
+
load_summaries,
|
|
20
|
+
regressions,
|
|
21
|
+
run_benchmark,
|
|
22
|
+
save_summaries,
|
|
23
|
+
summarize,
|
|
24
|
+
to_markdown,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
QUICK_BUDGETS = (10, 50, 200)
|
|
29
|
+
NOISY_BUDGETS = (10, 40, 200)
|
|
30
|
+
OUTPUT_BATCH_SIZES = (4, 8) # Bayesian optimization's batch sizes run on the problems with constraints on outputs, besides 1
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _join(numbers: list[int]) -> str:
|
|
34
|
+
"""[10, 20, 50] -> "10, 20 and 50"."""
|
|
35
|
+
return _join_words([f"{n:,}" for n in numbers])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _join_words(words: list[str]) -> str:
|
|
39
|
+
"""["a", "b", "c"] -> "a, b and c"."""
|
|
40
|
+
return words[0] if len(words) == 1 else ", ".join(words[:-1]) + " and " + words[-1]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main(argv: list[str] | None = None) -> int:
|
|
44
|
+
parser = argparse.ArgumentParser(prog="python -m hone.benchmarks", description=__doc__)
|
|
45
|
+
suite = parser.add_mutually_exclusive_group()
|
|
46
|
+
suite.add_argument("--quick", action="store_true", help="run the small CI subset with fewer seeds and evaluations")
|
|
47
|
+
suite.add_argument("--ml", action="store_true", help="run the machine learning tuning problems instead (needs scikit-learn)")
|
|
48
|
+
suite.add_argument("--noisy", action="store_true", help="run the problems with noise added to every evaluation instead")
|
|
49
|
+
suite.add_argument("--constrained", action="store_true", help="run the problems with constraints instead")
|
|
50
|
+
suite.add_argument(
|
|
51
|
+
"--outputs", action="store_true", help="run the problems with constraints on outputs instead, the noisy engineering designs included, with Bayesian optimization at batch sizes 1, 4 and 8"
|
|
52
|
+
)
|
|
53
|
+
suite.add_argument(
|
|
54
|
+
"--multi", action="store_true", help="run the problems with several objectives instead, scored by hypervolume"
|
|
55
|
+
)
|
|
56
|
+
suite.add_argument(
|
|
57
|
+
"--cost",
|
|
58
|
+
action="store_true",
|
|
59
|
+
help="time Bayesian optimization's candidates against evaluations and limits on outputs instead, with one profiled run; "
|
|
60
|
+
"--baseline then checks the ceiling at 500 evaluations",
|
|
61
|
+
)
|
|
62
|
+
suite.add_argument(
|
|
63
|
+
"--cost-multi",
|
|
64
|
+
action="store_true",
|
|
65
|
+
help="time MultiObjectiveBayesian's candidates (parego and models, 0 and 8 limits) as --cost does, without the "
|
|
66
|
+
"single-objective table or the profile; --noise times the noisy problem; --baseline checks the ceiling at 500 evaluations",
|
|
67
|
+
)
|
|
68
|
+
suite.add_argument(
|
|
69
|
+
"--conditional", action="store_true", help="run the problems with conditional parameters, and their flat versions, instead"
|
|
70
|
+
)
|
|
71
|
+
suite.add_argument(
|
|
72
|
+
"--engineering",
|
|
73
|
+
action="store_true",
|
|
74
|
+
help="run nothing: tabulate the engineering designs and g06, g07 and g10 against their published optima, "
|
|
75
|
+
"from saved summaries (see --summaries)",
|
|
76
|
+
)
|
|
77
|
+
parser.add_argument("--noise", action="store_true", help="with --cost-multi, also time the noisy problem (noise=True, which needs noise with limits), after the noise-free cells")
|
|
78
|
+
parser.add_argument("--seeds", type=int, help="number of seeds per problem (default 10, or 5 with --quick)")
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--budgets",
|
|
81
|
+
type=int,
|
|
82
|
+
nargs="+",
|
|
83
|
+
help=f"evaluations per dimension, one run per budget (default {' '.join(map(str, BUDGETS))}, "
|
|
84
|
+
f"{' '.join(map(str, QUICK_BUDGETS))} with --quick, {' '.join(map(str, ml.BUDGETS))} with --ml, "
|
|
85
|
+
f"or {' '.join(map(str, NOISY_BUDGETS))} with --noisy)",
|
|
86
|
+
)
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--batch-sizes",
|
|
89
|
+
type=int,
|
|
90
|
+
nargs="+",
|
|
91
|
+
metavar="SIZE",
|
|
92
|
+
help="also run Bayesian optimization choosing this many candidates per batch, as bayesian-SIZE (default 4 8 with --outputs)",
|
|
93
|
+
)
|
|
94
|
+
parser.add_argument("--jobs", type=int, default=1, help="share the runs among this many processes (Linux only)")
|
|
95
|
+
parser.add_argument("--tolerance", type=float, default=1e-4, help="error at or below which a run counts as solved")
|
|
96
|
+
parser.add_argument(
|
|
97
|
+
"--summaries",
|
|
98
|
+
nargs="+",
|
|
99
|
+
metavar="FILE",
|
|
100
|
+
default=list(engineering.SOURCES),
|
|
101
|
+
help="with --engineering, the saved summaries to read, a later file's results replacing an earlier one's "
|
|
102
|
+
f"(default {' '.join(engineering.SOURCES)})",
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument("--markdown", help="write the results table to this file")
|
|
105
|
+
parser.add_argument("--save", help="write the summaries as JSON to this file, for use as a baseline")
|
|
106
|
+
parser.add_argument("--baseline", help="compare against summaries saved with --save; exit 1 on a regression")
|
|
107
|
+
args = parser.parse_args(argv)
|
|
108
|
+
|
|
109
|
+
if args.multi:
|
|
110
|
+
return _multi(args)
|
|
111
|
+
if args.cost:
|
|
112
|
+
return _cost(args)
|
|
113
|
+
if args.cost_multi:
|
|
114
|
+
return _cost_multi(args)
|
|
115
|
+
if args.engineering:
|
|
116
|
+
return _engineering(args)
|
|
117
|
+
default_budgets: Sequence[int]
|
|
118
|
+
if args.quick:
|
|
119
|
+
problems, default_budgets = quick_problems(), QUICK_BUDGETS
|
|
120
|
+
elif args.ml:
|
|
121
|
+
problems, default_budgets = ml.ml_problems(), ml.BUDGETS
|
|
122
|
+
elif args.noisy:
|
|
123
|
+
problems, default_budgets = noisy_problems(), NOISY_BUDGETS
|
|
124
|
+
elif args.constrained:
|
|
125
|
+
problems, default_budgets = constrained_problems(), BUDGETS
|
|
126
|
+
elif args.outputs:
|
|
127
|
+
problems, default_budgets = output_problems() + noisy_output_problems(), BUDGETS
|
|
128
|
+
elif args.conditional:
|
|
129
|
+
problems, default_budgets = conditional_problems(), BUDGETS
|
|
130
|
+
else:
|
|
131
|
+
problems, default_budgets = standard_problems(), BUDGETS
|
|
132
|
+
seeds = args.seeds or (5 if args.quick else 10)
|
|
133
|
+
budgets = sorted(set(args.budgets or default_budgets))
|
|
134
|
+
batch_sizes = args.batch_sizes if args.batch_sizes is not None else OUTPUT_BATCH_SIZES if args.outputs else ()
|
|
135
|
+
optimizers = {**DEFAULT_OPTIMIZERS, **batch_optimizers(batch_sizes)}
|
|
136
|
+
|
|
137
|
+
start = time.perf_counter()
|
|
138
|
+
runs = run_benchmark(problems, optimizers, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs)
|
|
139
|
+
summaries = summarize(runs, args.tolerance)
|
|
140
|
+
elapsed = time.perf_counter() - start
|
|
141
|
+
|
|
142
|
+
table = to_markdown(summaries, args.tolerance)
|
|
143
|
+
print(table)
|
|
144
|
+
setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
|
|
145
|
+
print(f"{setup}, in {elapsed:.0f}s")
|
|
146
|
+
if args.markdown:
|
|
147
|
+
with open(args.markdown, "w") as f:
|
|
148
|
+
f.write(
|
|
149
|
+
f"# Benchmark results\n\nGenerated by `python -m hone.benchmarks` with hone {version('hone-python')}: "
|
|
150
|
+
f"{setup}.\n\n{table}"
|
|
151
|
+
)
|
|
152
|
+
if args.save:
|
|
153
|
+
save_summaries(summaries, args.save)
|
|
154
|
+
|
|
155
|
+
if args.baseline:
|
|
156
|
+
found = regressions(summaries, load_summaries(args.baseline))
|
|
157
|
+
if found:
|
|
158
|
+
print("\nRegressions against the baseline:", *found, sep="\n ")
|
|
159
|
+
return 1
|
|
160
|
+
print("\nNo regressions against the baseline.")
|
|
161
|
+
return 0
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _multi(args: argparse.Namespace) -> int:
|
|
165
|
+
"""The multi-objective suite: its own runs, scored by hypervolume gap."""
|
|
166
|
+
problems = multi_problems()
|
|
167
|
+
seeds = args.seeds or 10
|
|
168
|
+
budgets = sorted(set(args.budgets or BUDGETS))
|
|
169
|
+
start = time.perf_counter()
|
|
170
|
+
summaries = summarize_multi(run_multi_benchmark(problems, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs))
|
|
171
|
+
elapsed = time.perf_counter() - start
|
|
172
|
+
table = multi_markdown(summaries)
|
|
173
|
+
print(table)
|
|
174
|
+
setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
|
|
175
|
+
print(f"{setup}, in {elapsed:.0f}s")
|
|
176
|
+
if args.markdown:
|
|
177
|
+
with open(args.markdown, "w") as f:
|
|
178
|
+
f.write(f"# Multi-objective benchmark results\n\nGenerated by `python -m hone.benchmarks --multi` with hone {version('hone-python')}: {setup}.\n\n{table}")
|
|
179
|
+
if args.save:
|
|
180
|
+
with open(args.save, "w") as f:
|
|
181
|
+
f.write("[\n" + ",\n".join(s.model_dump_json() for s in summaries) + "\n]\n")
|
|
182
|
+
return 0
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _engineering(args: argparse.Namespace) -> int:
|
|
186
|
+
"""The engineering designs against their published optima, from summaries already saved."""
|
|
187
|
+
summaries = [s for path in args.summaries for s in load_summaries(path)]
|
|
188
|
+
table = engineering.engineering_table(summaries)
|
|
189
|
+
print(table)
|
|
190
|
+
if args.markdown:
|
|
191
|
+
sources = _join_words([f"`{Path(path).name}`" for path in args.summaries])
|
|
192
|
+
with open(args.markdown, "w") as f:
|
|
193
|
+
f.write(
|
|
194
|
+
"# Engineering designs against their published optima\n\n"
|
|
195
|
+
f"Generated by `python -m hone.benchmarks --engineering` from {sources}, which the constrained and "
|
|
196
|
+
f"outputs suites save; nothing is run.\n\n{table}"
|
|
197
|
+
)
|
|
198
|
+
return 0
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _cost(args: argparse.Namespace) -> int:
|
|
202
|
+
"""The cost of a Bayesian candidate: the table, one profiled run, and the ceiling against a saved table."""
|
|
203
|
+
print("Timing Bayesian candidates, one cell at a time:")
|
|
204
|
+
cells = cost.measure_cost(progress=True)
|
|
205
|
+
print("Profiling one run on g07:")
|
|
206
|
+
table = cost.CostTable(
|
|
207
|
+
hone=version("hone-python"),
|
|
208
|
+
machine=cost.machine(),
|
|
209
|
+
dimensions=cost.DIMENSIONS,
|
|
210
|
+
candidates=cost.CANDIDATES,
|
|
211
|
+
seeds=cost.SEEDS,
|
|
212
|
+
cells=cells,
|
|
213
|
+
profile=cost.profile_run(),
|
|
214
|
+
)
|
|
215
|
+
text = cost.cost_markdown(table)
|
|
216
|
+
print(text)
|
|
217
|
+
if args.markdown:
|
|
218
|
+
with open(args.markdown, "w") as f:
|
|
219
|
+
f.write(f"# The cost of a Bayesian candidate\n\nGenerated by `python -m hone.benchmarks --cost`.\n\n{text}")
|
|
220
|
+
if args.save:
|
|
221
|
+
cost.save_cost(table, args.save)
|
|
222
|
+
if args.baseline:
|
|
223
|
+
found = cost.ceiling_breaches(cells, cost.load_cost(args.baseline).cells)
|
|
224
|
+
if found:
|
|
225
|
+
print(f"\nOver the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations:", *found, sep="\n ")
|
|
226
|
+
return 1
|
|
227
|
+
print(f"\nWithin the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations.")
|
|
228
|
+
return 0
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _cost_multi(args: argparse.Namespace) -> int:
|
|
232
|
+
"""The cost of a MultiObjectiveBayesian candidate: the table, and the ceiling against a saved table."""
|
|
233
|
+
print("Timing multi-objective Bayesian candidates, one cell at a time:")
|
|
234
|
+
table = cost.CostTable(
|
|
235
|
+
hone=version("hone-python"), machine=cost.machine(), dimensions=cost.DIMENSIONS, candidates=cost.CANDIDATES, seeds=cost.SEEDS,
|
|
236
|
+
cells=[], multi=cost.measure_multi_cost(progress=True) + (cost.measure_multi_cost(noise=True, progress=True) if args.noise else []),
|
|
237
|
+
)
|
|
238
|
+
text = cost.cost_markdown(table)
|
|
239
|
+
print(text)
|
|
240
|
+
if args.markdown:
|
|
241
|
+
with open(args.markdown, "w") as f:
|
|
242
|
+
f.write(f"# The cost of a multi-objective Bayesian candidate\n\nGenerated by `python -m hone.benchmarks --cost-multi`.\n\n{text}")
|
|
243
|
+
if args.save:
|
|
244
|
+
cost.save_cost(table, args.save)
|
|
245
|
+
if args.baseline:
|
|
246
|
+
found = cost.multi_ceiling_breaches(table.multi, cost.load_cost(args.baseline).multi)
|
|
247
|
+
if found:
|
|
248
|
+
print(f"\nOver the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations:", *found, sep="\n ")
|
|
249
|
+
return 1
|
|
250
|
+
print(f"\nWithin the ceiling of ×{cost.CEILING} at {cost.CEILING_AT} evaluations.")
|
|
251
|
+
return 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
if __name__ == "__main__":
|
|
255
|
+
sys.exit(main())
|
|
@@ -6,8 +6,11 @@
|
|
|
6
6
|
- rosenbrock-disk: Rosenbrock's function inside the disk x² + y² ≤ 2, whose optimum (1, 1) sits
|
|
7
7
|
on the disk's edge.
|
|
8
8
|
- pressure-vessel, tension-spring and welded-beam: three standard engineering design problems,
|
|
9
|
-
scored against the best designs published for them
|
|
10
|
-
|
|
9
|
+
scored against the best designs published for them, which Cagnina, Esquivel and Coello Coello
|
|
10
|
+
(2008), "Solving engineering optimization problems with the simple constrained particle swarm
|
|
11
|
+
optimizer", Informatica 32(3):319-326, list as problems E02, E04 and E01. The welded beam's
|
|
12
|
+
stress formulas need square roots, so its constraints are functions; the others are
|
|
13
|
+
expressions.
|
|
11
14
|
|
|
12
15
|
Each engineering problem also comes with its limits on outputs, as the -outputs version: the
|
|
13
16
|
objective reports the volume, stresses and so on that the limits apply to, as a simulation
|
|
@@ -81,7 +84,8 @@ def pressure_vessel() -> Problem:
|
|
|
81
84
|
"""A cylindrical vessel with hemispherical heads, at least 1,296,000 cubic inches, of least cost.
|
|
82
85
|
|
|
83
86
|
Shell and head thicknesses come in sixteenths of an inch, so they are integers here. The best
|
|
84
|
-
published design costs 6059.714 (Ts = 13/16, Th = 7/16, R = 42.098, L = 176.637)
|
|
87
|
+
published design costs 6059.714 (Ts = 13/16, Th = 7/16, R = 42.098, L = 176.637)
|
|
88
|
+
(Cagnina et al., 2008, problem E02).
|
|
85
89
|
"""
|
|
86
90
|
shell, head = RangeParameter("shell", 1, 99, integer=True), RangeParameter("head", 1, 99, integer=True)
|
|
87
91
|
radius, length = RangeParameter("radius", 10, 200), RangeParameter("length", 10, 200)
|
|
@@ -104,7 +108,14 @@ def pressure_vessel() -> Problem:
|
|
|
104
108
|
def tension_spring() -> Problem:
|
|
105
109
|
"""A coil spring of least weight under limits on deflection, shear stress, surge frequency and diameter.
|
|
106
110
|
|
|
107
|
-
The best published design weighs 0.0126652 (d = 0.
|
|
111
|
+
The best published design weighs 0.0126652 (d = 0.051689, D = 0.356720, N = 11.288832).
|
|
112
|
+
Cagnina et al., 2008, problem E04, give the design (0.051690, 0.356750, 11.287126) with a
|
|
113
|
+
weight of 0.012665. Its rounded values break the shear stress limit by about 2e-5 (the stress
|
|
114
|
+
is 1.00002 times the limit), which is why they weigh 0.0126651, a little less than the optimum.
|
|
115
|
+
Celik and Kutucu, "Solving the tension/compression spring design problem by an improved
|
|
116
|
+
firefly algorithm", CEUR Workshop Proceedings 2255, paper 2, give the optimum as 0.0126652327883
|
|
117
|
+
at (0.051689156131, 0.356720026419, 11.288831695483), which satisfies every limit here. They
|
|
118
|
+
cite no source for it.
|
|
108
119
|
"""
|
|
109
120
|
d, coil, n = RangeParameter("wire", 0.05, 2.0), RangeParameter("coil", 0.25, 1.3), RangeParameter("turns", 2.0, 15.0)
|
|
110
121
|
|
|
@@ -144,6 +155,7 @@ def welded_beam() -> Problem:
|
|
|
144
155
|
"""A beam welded to a support, of least cost under limits on shear, bending, buckling and deflection.
|
|
145
156
|
|
|
146
157
|
The best published design costs 1.724852 (h = 0.205730, l = 3.470489, t = 9.036624, b = 0.205730).
|
|
158
|
+
Cagnina et al., 2008, problem E01, give the same cost and design with b = 0.205729.
|
|
147
159
|
"""
|
|
148
160
|
weld, length = RangeParameter("weld", 0.1, 2.0), RangeParameter("length", 0.1, 10.0)
|
|
149
161
|
thickness, width = RangeParameter("thickness", 0.1, 10.0), RangeParameter("width", 0.1, 2.0)
|
|
@@ -265,8 +277,3 @@ def constrained_problems(dimensions: tuple[int, ...] = (2, 5, 10)) -> list[Probl
|
|
|
265
277
|
tension_spring_outputs(),
|
|
266
278
|
welded_beam_outputs(),
|
|
267
279
|
]
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
def output_problems() -> list[Problem]:
|
|
271
|
-
"""The engineering designs with their limits on outputs only, for a quick comparison with the parameter versions."""
|
|
272
|
-
return [pressure_vessel_outputs(), tension_spring_outputs(), welded_beam_outputs()]
|