hone-python 0.13.0__tar.gz → 0.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {hone_python-0.13.0 → hone_python-0.15.0}/PKG-INFO +75 -11
  2. {hone_python-0.13.0 → hone_python-0.15.0}/README.md +72 -8
  3. {hone_python-0.13.0 → hone_python-0.15.0}/hone/__init__.py +2 -1
  4. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/__init__.py +4 -0
  5. hone_python-0.15.0/hone/benchmarks/__main__.py +255 -0
  6. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/constrained.py +114 -7
  7. hone_python-0.15.0/hone/benchmarks/cost.py +530 -0
  8. hone_python-0.15.0/hone/benchmarks/engineering.py +92 -0
  9. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/multi.py +127 -19
  10. hone_python-0.15.0/hone/benchmarks/multi_outputs.py +312 -0
  11. hone_python-0.15.0/hone/benchmarks/outputs.py +276 -0
  12. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/problems.py +2 -1
  13. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/runner.py +202 -32
  14. hone_python-0.15.0/hone/benchmarks/sharded.py +150 -0
  15. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/multi_objective_config.py +8 -4
  16. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/optimizer_config.py +24 -4
  17. {hone_python-0.13.0 → hone_python-0.15.0}/hone/objective_function/objective_function.py +8 -2
  18. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/_gp.py +41 -6
  19. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/_pool.py +37 -0
  20. hone_python-0.15.0/hone/optimizers/base.py +367 -0
  21. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/bayesian.py +171 -24
  22. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/choice.py +16 -2
  23. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/cmaes.py +4 -2
  24. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/core.py +156 -9
  25. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/genetic/nsga2.py +23 -3
  26. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/genetic/optimizer.py +6 -3
  27. hone_python-0.15.0/hone/optimizers/genetic/selection.py +51 -0
  28. hone_python-0.15.0/hone/optimizers/multi.py +480 -0
  29. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/multi_bayesian.py +51 -10
  30. {hone_python-0.13.0 → hone_python-0.15.0}/hone/parameters/__init__.py +1 -1
  31. {hone_python-0.13.0 → hone_python-0.15.0}/hone/parameters/conditions.py +18 -2
  32. {hone_python-0.13.0 → hone_python-0.15.0}/hone/parameters/constraints.py +189 -36
  33. {hone_python-0.13.0 → hone_python-0.15.0}/hone/parameters/parameters.py +90 -10
  34. {hone_python-0.13.0 → hone_python-0.15.0}/hone/pareto.py +10 -2
  35. {hone_python-0.13.0 → hone_python-0.15.0}/hone/result.py +25 -3
  36. {hone_python-0.13.0 → hone_python-0.15.0}/hone/sensitivity_analysis.py +27 -12
  37. {hone_python-0.13.0 → hone_python-0.15.0}/pyproject.toml +14 -9
  38. hone_python-0.13.0/hone/benchmarks/__main__.py +0 -121
  39. hone_python-0.13.0/hone/optimizers/base.py +0 -247
  40. hone_python-0.13.0/hone/optimizers/genetic/selection.py +0 -28
  41. hone_python-0.13.0/hone/optimizers/multi.py +0 -230
  42. {hone_python-0.13.0 → hone_python-0.15.0}/LICENSE +0 -0
  43. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/conditional.py +0 -0
  44. {hone_python-0.13.0 → hone_python-0.15.0}/hone/benchmarks/ml.py +0 -0
  45. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/__init__.py +0 -0
  46. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/bayesian_config.py +0 -0
  47. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/cmaes_config.py +0 -0
  48. {hone_python-0.13.0 → hone_python-0.15.0}/hone/config/genetic_optimizer_config.py +0 -0
  49. {hone_python-0.13.0 → hone_python-0.15.0}/hone/objective_function/__init__.py +0 -0
  50. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/__init__.py +0 -0
  51. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/genetic/__init__.py +0 -0
  52. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/genetic/breeding.py +0 -0
  53. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/genetic/variation.py +0 -0
  54. {hone_python-0.13.0 → hone_python-0.15.0}/hone/optimizers/random_search.py +0 -0
  55. {hone_python-0.13.0 → hone_python-0.15.0}/hone/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hone-python
3
- Version: 0.13.0
3
+ Version: 0.15.0
4
4
  Summary: A python-based optimization engine/library.
5
5
  License-Expression: BSD-3-Clause
6
6
  License-File: LICENSE
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3.13
13
13
  Classifier: Programming Language :: Python :: 3.14
14
14
  Classifier: Programming Language :: Python :: 3.15
15
15
  Requires-Dist: numpy (>=1.26.2)
16
- Requires-Dist: pydantic (>=2.7,<3.0.0)
16
+ Requires-Dist: pydantic (>=2.14.4,<3.0.0)
17
17
  Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
18
18
  Requires-Dist: scipy (>=1.13)
19
- Requires-Dist: threadpoolctl (>=3.5)
19
+ Requires-Dist: threadpoolctl (>=3.8.0)
20
20
  Project-URL: Changelog, https://github.com/GreatEmuWar/hone/blob/main/CHANGELOG.md
21
21
  Project-URL: Documentation, https://hone-python.dev/
22
22
  Project-URL: Source, https://github.com/GreatEmuWar/hone
@@ -58,7 +58,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
58
58
 
59
59
  ### Slow evaluations
60
60
 
61
- When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and on macOS and Windows the calling script needs an `if __name__ == "__main__":` guard.
61
+ When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
62
62
 
63
63
  ```python
64
64
  result = optimizer.optimize(run_simulation, workers=8, timeout=600)
@@ -111,6 +111,8 @@ def good_enough_or_overnight(progress):
111
111
  return good_enough or progress.elapsed > 8 * 3600
112
112
  ```
113
113
 
114
+ With constraints on outputs (see "Constraints on outputs" under Constraints), the best so far can break one while nothing has satisfied them all; check `progress.feasible` too before stopping on its value.
115
+
114
116
  Pressing Ctrl+C stops the run too. Either way, `optimize()` stops the workers and returns the best result so far, with `stopped` set.
115
117
 
116
118
  ### Resuming a run
@@ -125,7 +127,7 @@ After each generation the run's whole state is saved to the file. Run the same s
125
127
 
126
128
  With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
127
129
 
128
- A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
130
+ A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
129
131
 
130
132
  ### Failed evaluations
131
133
 
@@ -149,7 +151,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
149
151
 
150
152
  - Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
151
153
  - Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
152
- - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
154
+ - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
153
155
 
154
156
  `python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
155
157
 
@@ -212,6 +214,38 @@ Constraints should be quick to check: hone checks them for every candidate it co
212
214
 
213
215
  Starting points must satisfy the constraints. If the constraints leave nothing to search, or a region too small to find in 100,000 random draws, the optimizer raises `InfeasibleSearchSpace` when it's created, before any evaluation. Each optimizer keeps to constraints in its own way: random search and Bayesian optimization only consider points that satisfy them, CMA-ES draws a candidate again while it breaks one, and the genetic optimizer moves such a child back toward its fitter parent until it is feasible, which tends to land it near the boundary, where constrained optima usually are. On the benchmark's constrained problems the choice between optimizers follows the same pattern as without constraints; see [benchmarks/README.md](benchmarks/README.md#constrained-problems). [examples/pressure_vessel.py](examples/pressure_vessel.py) designs a pressure vessel of least cost under three constraints and finds the best published design. [examples/maintenance_schedule.py](examples/maintenance_schedule.py) chooses a maintenance schedule for a production line, scored by a noisy simulation of a year, with a constraint on technician hours and a function constraint that depends on the chosen shift; it halves the expected yearly cost of a sensible schedule chosen by hand.
214
216
 
217
+ #### Constraints on outputs
218
+
219
+ Some limits can't be checked until a design has been evaluated: the peak stress a simulation reports, a temperature, a latency. Write them the same way, with `Output` standing for the reported value, and pass them in the same list:
220
+
221
+ ```python
222
+ from hone import BayesianOptimizer, Output, SearchSpace
223
+
224
+ stress = Output("stress")
225
+ search_space = SearchSpace(
226
+ [width, height],
227
+ constraints=[
228
+ width * height <= 40, # on the parameters: checked before evaluating
229
+ stress <= 250, # on an output: judged once the evaluation reports it
230
+ stress <= 50 * width, # outputs and parameters can mix
231
+ ],
232
+ )
233
+
234
+ def simulate(width, height):
235
+ ...
236
+ return {"objective": mass, "stress": peak_stress}
237
+
238
+ result = BayesianOptimizer(search_space).optimize(simulate)
239
+ result.feasible # whether best_params satisfies every limit on the outputs
240
+ result.best_outputs # {"stress": ...} at best_params
241
+ ```
242
+
243
+ With constraints on outputs, the objective returns a dict: its value under `"objective"`, and a number for each output the constraints use. With several objectives, the outputs go in the same dict as the objectives, and a name can be both, so `Output("cost") <= 20` limits an objective. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`, and NaN or infinity anywhere in it marks the evaluation as failed. `tell(trial, {...})` takes the same dict. Outputs work everywhere parameters do in an expression, including `test >> constraint`, as in `material["steel"] >> (stress <= 400)`. They can't appear in a test, on the left of `>>`, because which parameters the objective is passed can't depend on what it returns. A function constraint takes outputs by name too: `Constraint.function(f, outputs=["stress"])`.
244
+
245
+ A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
246
+
247
+ Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
248
+
215
249
  ### Conditional parameters
216
250
 
217
251
  Some settings only exist when another takes certain values: a kernel's settings only for the model that has kernels, a third layer's size only with three layers or more. Write each condition as `test >> parameters`, read "then", and pass them to `SearchSpace`:
@@ -297,7 +331,7 @@ result.save("run.json")
297
331
  result = Result.load("run.json")
298
332
  ```
299
333
 
300
- `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result; for CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
334
+ `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
301
335
 
302
336
  ```python
303
337
  optimizer = Result.load("run.json").rebuild_optimizer()
@@ -336,8 +370,11 @@ result = optimizer.optimize(run_simulation)
336
370
  | Budget | Parameters | Chosen |
337
371
  | --- | --- | --- |
338
372
  | 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
339
- | More than 200 | Any choice or integer parameter | `GeneticOptimizer` |
340
- | More than 200 | Continuous numbers only | `CMAES` |
373
+ | 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
374
+ | Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
375
+ | Past the cutoff | Continuous numbers only | `CMAES` |
376
+
377
+ With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
341
378
 
342
379
  Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
343
380
 
@@ -378,7 +415,32 @@ print(choice.params, choice.values)
378
415
 
379
416
  `choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
380
417
 
381
- Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize`, `threshold` and `final_evaluations` don't apply.
418
+ Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
419
+
420
+ #### Noise with constraints on outputs
421
+
422
+ With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
423
+
424
+ ```python
425
+ from hone import NSGA2, NSGA2Config, Output, SearchSpace
426
+
427
+ search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
428
+ config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
429
+ result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
430
+ ```
431
+
432
+ Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
433
+
434
+ By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
435
+
436
+ What the rule guarantees, and what it doesn't:
437
+
438
+ - A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
439
+ - Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
440
+ - Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
441
+ - Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
442
+ - Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
443
+ - With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
382
444
 
383
445
  [examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
384
446
 
@@ -465,7 +527,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
465
527
 
466
528
  | Command | Does |
467
529
  | --- | --- |
468
- | `make test` | Install the dependencies and run the tests with coverage |
530
+ | `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
469
531
  | `make lint` | Run ruff, then mypy to check types |
470
532
  | `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
471
533
  | `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
@@ -474,6 +536,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
474
536
  | `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
475
537
  | `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
476
538
  | `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
539
+ | `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
540
+ | `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
477
541
  | `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
478
542
  | `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
479
543
 
@@ -34,7 +34,7 @@ The optimizer minimizes by default; set `maximize=True` in the config to maximiz
34
34
 
35
35
  ### Slow evaluations
36
36
 
37
- When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, and on macOS and Windows the calling script needs an `if __name__ == "__main__":` guard.
37
+ When each evaluation takes a while, run them in parallel. `workers` evaluates that many candidates at a time in background processes; the objective function must be defined at the top level of a module, not in a notebook or at a prompt, and not be a lambda or a nested function; otherwise a `TypeError` says so before any worker starts. The same goes for a function constraint. On macOS, Windows, and Linux from Python 3.14, where workers are started afresh and import the objective by name, the calling script needs an `if __name__ == "__main__":` guard. Under the fork start method (Linux before Python 3.14), any function works.
38
38
 
39
39
  ```python
40
40
  result = optimizer.optimize(run_simulation, workers=8, timeout=600)
@@ -87,6 +87,8 @@ def good_enough_or_overnight(progress):
87
87
  return good_enough or progress.elapsed > 8 * 3600
88
88
  ```
89
89
 
90
+ With constraints on outputs (see "Constraints on outputs" under Constraints), the best so far can break one while nothing has satisfied them all; check `progress.feasible` too before stopping on its value.
91
+
90
92
  Pressing Ctrl+C stops the run too. Either way, `optimize()` stops the workers and returns the best result so far, with `stopped` set.
91
93
 
92
94
  ### Resuming a run
@@ -101,7 +103,7 @@ After each generation the run's whole state is saved to the file. Run the same s
101
103
 
102
104
  With ask and tell, call `optimizer.save_checkpoint(path)` whenever you like, and `optimizer.load_checkpoint(path)` on a new optimizer set up the same way. Trials that were handed out but not yet told are handed out again by the next `ask()`, and can still be told by their ids.
103
105
 
104
- A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself.
106
+ A checkpoint is a pickle, which can run code when it is loaded, so only load checkpoints you saved yourself. Pickle checkpoints are deprecated: from 0.15.0 loading one, with `load_checkpoint()` or by carrying on a run with `optimize(checkpoint=...)`, gives a `FutureWarning`. A safe format arrives in 0.16.0, and loading pickle checkpoints will be removed in 0.19.0. Saving still writes pickles until 0.16.0, and doesn't warn.
105
107
 
106
108
  ### Failed evaluations
107
109
 
@@ -125,7 +127,7 @@ Otherwise the optimizer believes every value it sees, and its best result is lik
125
127
 
126
128
  - Every evaluation of the same point is averaged, and a point's fitness is that average. The genetic optimizer often breeds a point again, which re-evaluates it for free, and ranks its parents by their latest averages.
127
129
  - Bayesian optimization measures improvement against the best value its model predicts at any evaluated point, rather than against the luckiest single value.
128
- - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search.
130
+ - `final_evaluations` keeps that many evaluations back for the end, to evaluate the most promising points again (up to 5 of them, each at least twice) and return the one with the best average. Its `best_fitness` is then an average too, and so a fairer estimate of what the result will achieve. It's off by default, because on small budgets the evaluations are worth more to the search, except with constraints on outputs (below), where a tenth of `max_evaluations` is kept back, at least 10 and at most half; `final_evaluations=0` turns that off.
129
131
 
130
132
  `python -m hone.benchmarks --noisy` measures this: it adds noise to every evaluation and scores each run by the true, noise-free value of the point it returns.
131
133
 
@@ -188,6 +190,38 @@ Constraints should be quick to check: hone checks them for every candidate it co
188
190
 
189
191
  Starting points must satisfy the constraints. If the constraints leave nothing to search, or a region too small to find in 100,000 random draws, the optimizer raises `InfeasibleSearchSpace` when it's created, before any evaluation. Each optimizer keeps to constraints in its own way: random search and Bayesian optimization only consider points that satisfy them, CMA-ES draws a candidate again while it breaks one, and the genetic optimizer moves such a child back toward its fitter parent until it is feasible, which tends to land it near the boundary, where constrained optima usually are. On the benchmark's constrained problems the choice between optimizers follows the same pattern as without constraints; see [benchmarks/README.md](benchmarks/README.md#constrained-problems). [examples/pressure_vessel.py](examples/pressure_vessel.py) designs a pressure vessel of least cost under three constraints and finds the best published design. [examples/maintenance_schedule.py](examples/maintenance_schedule.py) chooses a maintenance schedule for a production line, scored by a noisy simulation of a year, with a constraint on technician hours and a function constraint that depends on the chosen shift; it halves the expected yearly cost of a sensible schedule chosen by hand.
190
192
 
193
+ #### Constraints on outputs
194
+
195
+ Some limits can't be checked until a design has been evaluated: the peak stress a simulation reports, a temperature, a latency. Write them the same way, with `Output` standing for the reported value, and pass them in the same list:
196
+
197
+ ```python
198
+ from hone import BayesianOptimizer, Output, SearchSpace
199
+
200
+ stress = Output("stress")
201
+ search_space = SearchSpace(
202
+ [width, height],
203
+ constraints=[
204
+ width * height <= 40, # on the parameters: checked before evaluating
205
+ stress <= 250, # on an output: judged once the evaluation reports it
206
+ stress <= 50 * width, # outputs and parameters can mix
207
+ ],
208
+ )
209
+
210
+ def simulate(width, height):
211
+ ...
212
+ return {"objective": mass, "stress": peak_stress}
213
+
214
+ result = BayesianOptimizer(search_space).optimize(simulate)
215
+ result.feasible # whether best_params satisfies every limit on the outputs
216
+ result.best_outputs # {"stress": ...} at best_params
217
+ ```
218
+
219
+ With constraints on outputs, the objective returns a dict: its value under `"objective"`, and a number for each output the constraints use. With several objectives, the outputs go in the same dict as the objectives, and a name can be both, so `Output("cost") <= 20` limits an objective. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`, and NaN or infinity anywhere in it marks the evaluation as failed. `tell(trial, {...})` takes the same dict. Outputs work everywhere parameters do in an expression, including `test >> constraint`, as in `material["steel"] >> (stress <= 400)`. They can't appear in a test, on the left of `>>`, because which parameters the objective is passed can't depend on what it returns. A function constraint takes outputs by name too: `Constraint.function(f, outputs=["stress"])`.
220
+
221
+ A design that breaks a limit on an output has already been evaluated, so it can't be skipped; instead it is ranked by Deb's feasibility rule, with no penalty weight to tune. A design that satisfies every limit beats one that doesn't, those that do rank by their value, and those that don't rank by their total violation. Each violation is a share of its limit when the limit is a number, as in `stress <= 250`, so 10% over a stress limit counts the same as 10% over a deflection limit. A limit that varies with the design, such as `stress <= 50 * width`, a limit of zero and a function constraint count in their own units, so when several limits share a problem, write each with its number on one side, and have a function return a share, such as `stress / LIMIT - 1`. `best_params` is the best design that satisfies every limit. If none did, it is the one that came closest, and `feasible` is `False`. Each generation's statistics count the designs that broke a limit in `infeasible` and leave them out of the fitness figures. A threshold is reached only by a design that satisfies every limit. On a front, only such designs appear, and each point carries its `outputs`.
222
+
223
+ Bayesian optimization makes the most of these limits. It models each limit with a Gaussian process, learning from the designs that broke one, and multiplies its expected improvement by the probability that a candidate satisfies them all, treating the limits as independent, so two limits that say nearly the same thing make it more cautious there than it need be; until it finds a design that does, it looks for one. The other optimizers only learn which designs ranked lower. On the benchmark's engineering problems with their limits on outputs, Bayesian optimization comes close to its results when the same limits are known before evaluating, while the genetic optimizer and CMA-ES lose more; see [benchmarks/README.md](benchmarks/README.md#constraints-on-outputs). With `noise=True`, a design's outputs are averaged like its value and each limit is judged on the averages, since a noisy design can pass a limit by luck; the final round is then on by default (`final_evaluations`, above), so the result's feasibility rests on several draws rather than one. And each limit is judged with a margin of safety: a design passes only if its average clears the limit by about 1.64 standard errors with many repeats and more with few (`feasibility_confidence=0.95`, a Student t quantile), the noise in each limit estimated from every design evaluated more than once. A design on the boundary of a limit, where the best usually is, otherwise passes half the time however often it's averaged; with the margin, on the noisy engineering benchmarks, results that report satisfying their limits but truly break one fall from up to half of Bayesian optimization's runs to at most a tenth, for a design 0.1 to 2 percentage points further from the optimum. `feasibility_confidence=None` judges on the average alone. The confidence is per limit, for a given design, before selection: because a result is chosen on the same draws that judge it, the share of returned designs that truly break a limit can be higher than 1 − `feasibility_confidence`, and a design on several limits must stand further back, as the margin is per limit. The noise is pooled on the assumption that it is about the same everywhere; a limit that varies with the design, such as `Output("stress") <= 50 * width`, is better written as a ratio, `Output("stress") / (50 * width) <= 1`. `sensitivity()` accepts them and measures only the objective. [examples/welded_beam.py](examples/welded_beam.py) designs a welded beam whose stresses, buckling load and deflection come from an analysis of each design, and gets within about 1% of the best published design in 150 evaluations.
224
+
191
225
  ### Conditional parameters
192
226
 
193
227
  Some settings only exist when another takes certain values: a kernel's settings only for the model that has kernels, a third layer's size only with three layers or more. Write each condition as `test >> parameters`, read "then", and pass them to `SearchSpace`:
@@ -273,7 +307,7 @@ result.save("run.json")
273
307
  result = Result.load("run.json")
274
308
  ```
275
309
 
276
- `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective and it returns the same result; for CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
310
+ `rebuild_optimizer()` sets up a new optimizer exactly as the result's was, so a saved run can be repeated. Run it on the same objective, with the same versions of hone and NumPy, and it returns the same result: a seed fixes the random draws, not the method, so a new version of hone can choose different candidates. `result.run` records both versions. This release changes seeded results where the margin of safety applies, with noise and constraints on outputs. For CMA-ES and Bayesian optimization, also use the same number of workers, `result.run.workers`. An optimizer class of your own is rebuilt with `rebuild_optimizer(MyOptimizer)`.
277
311
 
278
312
  ```python
279
313
  optimizer = Result.load("run.json").rebuild_optimizer()
@@ -312,8 +346,11 @@ result = optimizer.optimize(run_simulation)
312
346
  | Budget | Parameters | Chosen |
313
347
  | --- | --- | --- |
314
348
  | 200 evaluations or fewer | Any, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
315
- | More than 200 | Any choice or integer parameter | `GeneticOptimizer` |
316
- | More than 200 | Continuous numbers only | `CMAES` |
349
+ | 500 evaluations or fewer | Constraints on outputs, noise off, unless a choice parameter has `choose_n` | `BayesianOptimizer` |
350
+ | Past the cutoff (200, or 500 as above) | Any choice or integer parameter | `GeneticOptimizer` |
351
+ | Past the cutoff | Continuous numbers only | `CMAES` |
352
+
353
+ With constraints on outputs and noise off, Bayesian optimization stays the choice to 500 evaluations, because on the 14 problems of [benchmarks/choose-outputs-results.md](benchmarks/choose-outputs-results.md) it beat the old pick on every one. At 500 it spends about 1 to 5 minutes of its own time per run on 2 to 5 parameters with up to 4 limits, and about 12 to 19 with 10 parameters and 8 limits, on GitHub's runners (processors differ by about ×2.4; [benchmarks/choose-outputs-timing-results.md](benchmarks/choose-outputs-timing-results.md)), and the time per candidate grows as the run converges. With noise the cutoff stays at 200, because Bayesian optimization's final-round feasibility isn't reliable yet. To keep the old choice, construct the optimizer the old rule would pick: `GeneticOptimizer` with choice or integer parameters, `CMAES` otherwise, such as `CMAES(search_space, CMAESConfig(max_evaluations=400))`.
317
354
 
318
355
  Other settings, such as `seed`, `maximize`, `noise` or `threshold`, are passed on to the chosen optimizer's config. On the benchmark's 180 problem and budget pairs, its pick had a median error 1.37 times the best optimizer's, on geometric mean, and was the best on 146; always using CMA-ES gives 7.9 times the best, and it can't run the 15 problems with choice parameters. Its biggest misses are plain Rastrigin with 1,000 evaluations per dimension and plain ellipsoid with 50 to 100, where the genetic optimizer wins because the variables don't interact, which a rule can't know in advance; on the rotated versions CMA-ES wins.
319
356
 
@@ -354,7 +391,32 @@ print(choice.params, choice.values)
354
391
 
355
392
  `choose_pareto_optimizer(search_space, objectives, max_evaluations)` picks `MultiObjectiveBayesian` up to 200 evaluations, unless a choice parameter has `choose_n`, and `NSGA2` otherwise, as the benchmark suggests; other settings are passed on to the chosen optimizer's config. On it, Bayesian optimization's median front beats NSGA2's on 12 of the 13 problem and budget pairs both run on and don't tie.
356
393
 
357
- Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize`, `threshold` and `final_evaluations` don't apply.
394
+ Everything else works as with one objective: constraints, conditions, workers and time limits, failed evaluations, checkpoints, `ask()` and `tell(trial, {...})`, starting points, `save()`, `load()` and `rebuild_optimizer()`. With `noise=True`, each point on the front is an average, with its number of evaluations and the `standard_error` of each objective. `print_progress` prints the size of the front and its hypervolume. A dict with missing or extra keys, or a value that isn't a number, raises `TypeError`; `maximize` and `threshold` don't apply, and `final_evaluations` applies only with noise and constraints on outputs, below.
395
+
396
+ #### Noise with constraints on outputs
397
+
398
+ With `noise=True` and a limit on an output, as in `Output("stress") <= 250`, a point's outputs are averaged like its values, and the point is judged on the averages. The limit is on the true output, and a single noisy draw can pass it by luck.
399
+
400
+ ```python
401
+ from hone import NSGA2, NSGA2Config, Output, SearchSpace
402
+
403
+ search_space = SearchSpace([width, height], constraints=[Output("stress") <= 250])
404
+ config = NSGA2Config(objectives={"mass": "minimize", "deflection": "minimize"}, noise=True, max_evaluations=2000, seed=0)
405
+ result = NSGA2(search_space, config).optimize(analyse) # analyse returns mass, deflection and stress
406
+ ```
407
+
408
+ Every draw counts toward a point's averages, including those that break a limit; a draw that broke it is also counted in `result.infeasible`. A point passes a limit only if its average clears it with the margin of safety: about 1.64 standard errors with many repeats, more with few (`feasibility_confidence=0.95`; `None` judges on the averages alone). The front is that of the points that pass, with dominance judged on their average values. Each `ParetoPoint.outputs` holds average outputs, and `ParetoResult.feasible` says whether any point passes on its averages.
409
+
410
+ By default a tenth of `max_evaluations` is kept back for a final round, at least 10 and at most half. With the margin on, the round comes in two halves. The first evaluates points spread along the front, chosen by hypervolume contribution, and topped up until every limit has at least 3 degrees of freedom to estimate its noise. The second goes to up to 4 front points whose pass is least certain. With the margin off there is one round. `final_evaluations=0` keeps nothing back. `final_evaluations` with noise and no constraints on outputs is refused, since there is nothing to judge. Without noise, or with noise and no constraints on outputs, nothing changes.
411
+
412
+ What the rule guarantees, and what it doesn't:
413
+
414
+ - A design that truly breaks any one limit passes with probability at most 1 − `feasibility_confidence`. That is a rate per reported point, judged before selection, not the chance that every point on the front is feasible. Each limit has its own margin, so a point on several limits must stand further back from each.
415
+ - Selection is stronger on a front. A point that passes a limit by luck dominates its truly feasible neighbours and removes them from the front, so the share of returned points that truly break a limit can exceed 1 − `feasibility_confidence`, most where the whole front sits on limits.
416
+ - Concentrated repeats bias what survives. A point evaluated several times regresses toward its mean, while one evaluated once keeps its lucky draw, so points that survive unrepeated are those well inside the limits, and the front leans toward the interior. A region of it can disappear when a false pass is exposed.
417
+ - Dominance is judged on averages and ignores noise in the objectives. A point can be on the front because of a lucky objective draw. The margin protects the limits, not the objectives.
418
+ - Each limit's noise is pooled over every point evaluated more than once, which assumes it is about the same everywhere. A front spans more of the space than one optimum, so this is stretched further.
419
+ - With a small budget the reserve is the minimum of 10, so each half is 5 evaluations and the front is judged on few measured points. Fronts then come back smaller, and honest.
358
420
 
359
421
  [examples/cantilever_beam.py](examples/cantilever_beam.py) designs a beam in steel, aluminium or carbon fibre that is light, cheap and stiff, three objectives, under a stress limit that depends on the material, and chooses the stiffest design under a mass and a cost limit. Tuning for more than three objectives is out of scope for now: with many objectives nearly every design is on the front, and the search loses its pull.
360
422
 
@@ -441,7 +503,7 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
441
503
 
442
504
  | Command | Does |
443
505
  | --- | --- |
444
- | `make test` | Install the dependencies and run the tests with coverage |
506
+ | `make test` | Install the dependencies and run the tests on every core, then the few that measure wall-clock time on their own; CI adds coverage |
445
507
  | `make lint` | Run ruff, then mypy to check types |
446
508
  | `make docs` | Serve the documentation site at http://127.0.0.1:8000, rebuilding as files change (see [Docs](#docs)) |
447
509
  | `make docs-build` | Build the documentation site into `site/`, failing on any warning, as CI does |
@@ -450,6 +512,8 @@ Requires Python 3.12 or later and [Poetry](https://python-poetry.org/) 2.
450
512
  | `make benchmark-noisy` | Run the noisy problems and update `benchmarks/noisy-results.md` |
451
513
  | `make benchmark-ml` | Run the machine learning tuning problems and update `benchmarks/ml-results.md` |
452
514
  | `make benchmark-constrained` | Run the constrained problems and update `benchmarks/constrained-results.md` |
515
+ | `make benchmark-outputs` | Run the problems with constraints on outputs and update `benchmarks/outputs-results.md` |
516
+ | `make benchmark-cost` | Time Bayesian optimization's candidates and update `benchmarks/cost-results.md`, the ceiling later changes are held to |
453
517
  | `make benchmark-conditional` | Run the conditional problems and update `benchmarks/conditional-results.md` |
454
518
  | `make benchmark-multi` | Run the multi-objective problems and update `benchmarks/multi-results.md` |
455
519
 
@@ -15,7 +15,7 @@ from hone.optimizers import (
15
15
  choose_optimizer,
16
16
  choose_pareto_optimizer,
17
17
  )
18
- from hone.parameters import ChoiceParameter, Condition, Constraint, FixedParameter, InfeasibleSearchSpace, RangeParameter, SearchSpace
18
+ from hone.parameters import ChoiceParameter, Condition, Constraint, FixedParameter, InfeasibleSearchSpace, Output, RangeParameter, SearchSpace
19
19
  from hone.pareto import Front, ParetoGenerationStats, ParetoPoint, ParetoProgress, ParetoResult
20
20
  from hone.result import GenerationStats, Progress, Result, RunInfo, print_progress
21
21
  from hone.sensitivity_analysis import Interval, ParameterSensitivity, SensitivityReport, sensitivity
@@ -47,6 +47,7 @@ __all__ = [
47
47
  "NSGA2Config",
48
48
  "Optimizer",
49
49
  "OptimizerConfig",
50
+ "Output",
50
51
  "ParameterSensitivity",
51
52
  "ParetoGenerationStats",
52
53
  "ParetoPoint",
@@ -1,8 +1,11 @@
1
1
  """Benchmark problems and a runner for comparing optimizers. Run `python -m hone.benchmarks --help`."""
2
2
  from .conditional import conditional_problems
3
3
  from .constrained import constrained_problems
4
+ from .engineering import engineering_problems, engineering_table
4
5
  from .ml import ml_problems
5
6
  from .multi import MultiProblem, multi_problems, run_multi_benchmark, summarize_multi
7
+ from .multi_outputs import cantilever_outputs, noisy_multi_output_problems, osy, tnk
8
+ from .outputs import noisy_output_problems, output_problems
6
9
  from .problems import Problem, noisy_problems, quick_problems, standard_problems
7
10
  from .runner import (
8
11
  BUDGETS,
@@ -10,6 +13,7 @@ from .runner import (
10
13
  Run,
11
14
  SkipBudget,
12
15
  Summary,
16
+ batch_optimizers,
13
17
  load_summaries,
14
18
  regressions,
15
19
  run_benchmark,