hone-python 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hone/__init__.py CHANGED
@@ -1,33 +1,55 @@
1
- from hone.config import BayesianConfig, CMAESConfig, GeneticOptimizerConfig, OptimizerConfig, SelectionMethod
1
+ from hone.config import BayesianConfig, CMAESConfig, GeneticOptimizerConfig, MultiObjectiveBayesianConfig, MultiObjectiveConfig, NSGA2Config, OptimizerConfig, SelectionMethod
2
2
  from hone.optimizers import (
3
3
  CMAES,
4
4
  BayesianOptimizer,
5
5
  EvaluationFailed,
6
6
  GeneticOptimizer,
7
+ MultiObjectiveBayesian,
8
+ MultiObjectiveOptimizer,
9
+ MultiObjectiveRandomSearch,
10
+ NSGA2,
7
11
  Optimizer,
8
12
  RandomSearch,
9
13
  Trial,
10
14
  UnsupportedSearchSpace,
11
15
  choose_optimizer,
16
+ choose_pareto_optimizer,
12
17
  )
13
- from hone.parameters import ChoiceParameter, Constraint, FixedParameter, InfeasibleSearchSpace, RangeParameter, SearchSpace
18
+ from hone.parameters import ChoiceParameter, Condition, Constraint, FixedParameter, InfeasibleSearchSpace, RangeParameter, SearchSpace
19
+ from hone.pareto import Front, ParetoGenerationStats, ParetoPoint, ParetoProgress, ParetoResult
14
20
  from hone.result import GenerationStats, Progress, Result, RunInfo, print_progress
15
21
 
16
22
  __all__ = [
17
23
  "BayesianConfig",
18
24
  "BayesianOptimizer",
25
+ "ChoiceParameter",
26
+ "choose_optimizer",
27
+ "choose_pareto_optimizer",
19
28
  "CMAES",
20
29
  "CMAESConfig",
21
- "ChoiceParameter",
30
+ "Condition",
22
31
  "Constraint",
23
32
  "EvaluationFailed",
24
33
  "FixedParameter",
34
+ "Front",
25
35
  "GenerationStats",
26
36
  "GeneticOptimizer",
27
37
  "GeneticOptimizerConfig",
28
38
  "InfeasibleSearchSpace",
39
+ "MultiObjectiveBayesian",
40
+ "MultiObjectiveBayesianConfig",
41
+ "MultiObjectiveConfig",
42
+ "MultiObjectiveOptimizer",
43
+ "MultiObjectiveRandomSearch",
44
+ "NSGA2",
45
+ "NSGA2Config",
29
46
  "Optimizer",
30
47
  "OptimizerConfig",
48
+ "ParetoGenerationStats",
49
+ "ParetoPoint",
50
+ "ParetoProgress",
51
+ "ParetoResult",
52
+ "print_progress",
31
53
  "Progress",
32
54
  "RandomSearch",
33
55
  "RangeParameter",
@@ -37,6 +59,4 @@ __all__ = [
37
59
  "SelectionMethod",
38
60
  "Trial",
39
61
  "UnsupportedSearchSpace",
40
- "choose_optimizer",
41
- "print_progress",
42
62
  ]
@@ -1,6 +1,8 @@
1
1
  """Benchmark problems and a runner for comparing optimizers. Run `python -m hone.benchmarks --help`."""
2
+ from .conditional import conditional_problems
2
3
  from .constrained import constrained_problems
3
4
  from .ml import ml_problems
5
+ from .multi import MultiProblem, multi_problems, run_multi_benchmark, summarize_multi
4
6
  from .problems import Problem, noisy_problems, quick_problems, standard_problems
5
7
  from .runner import (
6
8
  BUDGETS,
@@ -6,6 +6,8 @@ from collections.abc import Sequence
6
6
  from importlib.metadata import version
7
7
 
8
8
  from . import ml
9
+ from .conditional import conditional_problems
10
+ from .multi import multi_markdown, multi_problems, run_multi_benchmark, summarize_multi
9
11
  from .constrained import constrained_problems
10
12
  from .problems import noisy_problems, quick_problems, standard_problems
11
13
  from .runner import BUDGETS, load_summaries, regressions, run_benchmark, save_summaries, summarize, to_markdown
@@ -28,6 +30,12 @@ def main(argv: list[str] | None = None) -> int:
28
30
  suite.add_argument("--ml", action="store_true", help="run the machine learning tuning problems instead (needs scikit-learn)")
29
31
  suite.add_argument("--noisy", action="store_true", help="run the problems with noise added to every evaluation instead")
30
32
  suite.add_argument("--constrained", action="store_true", help="run the problems with constraints instead")
33
+ suite.add_argument(
34
+ "--multi", action="store_true", help="run the problems with several objectives instead, scored by hypervolume"
35
+ )
36
+ suite.add_argument(
37
+ "--conditional", action="store_true", help="run the problems with conditional parameters, and their flat versions, instead"
38
+ )
31
39
  parser.add_argument("--seeds", type=int, help="number of seeds per problem (default 10, or 5 with --quick)")
32
40
  parser.add_argument(
33
41
  "--budgets",
@@ -44,6 +52,8 @@ def main(argv: list[str] | None = None) -> int:
44
52
  parser.add_argument("--baseline", help="compare against summaries saved with --save; exit 1 on a regression")
45
53
  args = parser.parse_args(argv)
46
54
 
55
+ if args.multi:
56
+ return _multi(args)
47
57
  default_budgets: Sequence[int]
48
58
  if args.quick:
49
59
  problems, default_budgets = quick_problems(), QUICK_BUDGETS
@@ -53,6 +63,8 @@ def main(argv: list[str] | None = None) -> int:
53
63
  problems, default_budgets = noisy_problems(), NOISY_BUDGETS
54
64
  elif args.constrained:
55
65
  problems, default_budgets = constrained_problems(), BUDGETS
66
+ elif args.conditional:
67
+ problems, default_budgets = conditional_problems(), BUDGETS
56
68
  else:
57
69
  problems, default_budgets = standard_problems(), BUDGETS
58
70
  seeds = args.seeds or (5 if args.quick else 10)
@@ -84,5 +96,26 @@ def main(argv: list[str] | None = None) -> int:
84
96
  return 0
85
97
 
86
98
 
99
+ def _multi(args: argparse.Namespace) -> int:
100
+ """The multi-objective suite: its own runs, scored by hypervolume gap."""
101
+ problems = multi_problems()
102
+ seeds = args.seeds or 10
103
+ budgets = sorted(set(args.budgets or BUDGETS))
104
+ start = time.perf_counter()
105
+ summaries = summarize_multi(run_multi_benchmark(problems, seeds=range(seeds), budgets_per_dimension=budgets, jobs=args.jobs))
106
+ elapsed = time.perf_counter() - start
107
+ table = multi_markdown(summaries)
108
+ print(table)
109
+ setup = f"{len(problems)} problems, {seeds} seeds, budgets of {_join(budgets)} evaluations per dimension"
110
+ print(f"{setup}, in {elapsed:.0f}s")
111
+ if args.markdown:
112
+ with open(args.markdown, "w") as f:
113
+ f.write(f"# Multi-objective benchmark results\n\nGenerated by `python -m hone.benchmarks --multi` with hone {version('hone-python')}: {setup}.\n\n{table}")
114
+ if args.save:
115
+ with open(args.save, "w") as f:
116
+ f.write("[\n" + ",\n".join(s.model_dump_json() for s in summaries) + "\n]\n")
117
+ return 0
118
+
119
+
87
120
  if __name__ == "__main__":
88
121
  sys.exit(main())
@@ -0,0 +1,109 @@
1
+ """Conditional test problems, each in two versions: with conditions, and with every parameter switched on.
2
+
3
+ - branches-3x2d and branches-3x5d: a choice of three branches, each a function of its own
4
+ numbers. The best branch is a steep, narrow bowl reaching 0; the decoys are a wide, shallow
5
+ bowl whose floor is 1 and a shallow Rastrigin grid whose floor is 0.5. Random points score
6
+ better in the decoys, so an optimizer has to look past them.
7
+ - nested-9: a choice of model; one model has a choice of kernel, and one kernel has numbers of
8
+ its own, so the best point is two conditions deep. The other model is a wide, shallow decoy
9
+ whose floor is 0.25.
10
+ - layers-7: a layer count from 1 to 5 whose larger values switch on more numbers, as a
11
+ network's units per layer would be. Three layers is best. Its conditions test a range, so
12
+ CMA-ES can search it.
13
+
14
+ The "-flat" version of each has the same parameters, function and optimum, with no conditions:
15
+ every parameter is always on and the function ignores those it doesn't use, which was the way
16
+ to search such a space before conditions. Comparing the two shows what conditions are worth.
17
+ """
18
+ from typing import Any, Callable
19
+
20
+ import numpy as np
21
+
22
+ from hone.parameters import ChoiceParameter, Condition, Parameter, RangeParameter, SearchSpace
23
+
24
+ from .problems import BOUND, Problem, _shift, rastrigin, sphere
25
+
26
+
27
+ def _problem(name: str, function: Callable[..., float], parameters: list[Parameter], conditions: list[Condition]) -> list[Problem]:
28
+ function.__name__ = name.replace("-", "_")
29
+ return [
30
+ Problem(
31
+ name=name,
32
+ function=function,
33
+ search_space=SearchSpace(parameters, conditions=conditions),
34
+ optimum=0.0,
35
+ tags=frozenset({"conditional"}),
36
+ ),
37
+ Problem(
38
+ name=f"{name}-flat",
39
+ function=function,
40
+ search_space=SearchSpace(parameters),
41
+ optimum=0.0,
42
+ tags=frozenset({"conditional", "flat"}),
43
+ ),
44
+ ]
45
+
46
+
47
+ def _numbers(prefix: str, count: int) -> list[RangeParameter]:
48
+ return [RangeParameter(f"{prefix}{i}", -BOUND, BOUND) for i in range(count)]
49
+
50
+
51
+ def _vector(params: dict[str, Any], numbers: list[RangeParameter]) -> np.ndarray:
52
+ return np.array([params[p.name] for p in numbers], dtype=float)
53
+
54
+
55
+ def branches(dimensions: int) -> list[Problem]:
56
+ name = f"branches-3x{dimensions}d"
57
+ branch = ChoiceParameter("branch", ["narrow", "wide", "grid"])
58
+ numbers = {b: _numbers(f"{b}_", dimensions) for b in branch.choices}
59
+ shifts = {b: _shift(f"{name}/{b}", dimensions) for b in branch.choices}
60
+ # (function, scale, floor) for each branch
61
+ bases = {"narrow": (sphere, 4.0, 0.0), "wide": (sphere, 0.05, 1.0), "grid": (rastrigin, 0.1, 0.5)}
62
+
63
+ def function(branch: str, **params: Any) -> float:
64
+ base, scale, floor = bases[branch]
65
+ return floor + scale * float(base(_vector(params, numbers[branch]) - shifts[branch]))
66
+
67
+ parameters: list[Parameter] = [branch, *(p for b in branch.choices for p in numbers[b])]
68
+ return _problem(name, function, parameters, [branch[b] >> numbers[b] for b in branch.choices])
69
+
70
+
71
+ def nested() -> list[Problem]:
72
+ name = "nested-9"
73
+ model = ChoiceParameter("model", ["kernel_machine", "forest"])
74
+ kernel = ChoiceParameter("kernel", ["linear", "rbf"])
75
+ shared, width, forest = _numbers("s", 2), _numbers("g", 2), _numbers("f", 3)
76
+ centres = {key: _shift(f"{name}/{key}", len(group)) for key, group in [("s", shared), ("g", width), ("f", forest)]}
77
+
78
+ def function(model: str, kernel: str | None = None, **params: Any) -> float:
79
+ if model == "forest":
80
+ return 0.25 + 0.05 * float(sphere(_vector(params, forest) - centres["f"]))
81
+ value = float(sphere(_vector(params, shared) - centres["s"]))
82
+ if kernel == "linear":
83
+ return 0.5 + value
84
+ return value + 4 * float(sphere(_vector(params, width) - centres["g"]))
85
+
86
+ parameters: list[Parameter] = [model, kernel, *shared, *width, *forest]
87
+ conditions = [model["kernel_machine"] >> [kernel, *shared], kernel["rbf"] >> width, model["forest"] >> forest]
88
+ return _problem(name, function, parameters, conditions)
89
+
90
+
91
+ def layers() -> list[Problem]:
92
+ name = "layers-7"
93
+ count = RangeParameter("layers", 1, 5, integer=True)
94
+ rate = RangeParameter("rate", -BOUND, BOUND)
95
+ units = _numbers("units_", 5)
96
+ centre = _shift(f"{name}/units", 5)
97
+ best_rate = float(_shift(f"{name}/rate", 1)[0])
98
+
99
+ def function(layers: int, rate: float, **params: Any) -> float:
100
+ used = units[:layers]
101
+ return 0.5 * abs(layers - 3) + float(sphere(_vector(params, used) - centre[:layers])) + (rate - best_rate) ** 2
102
+
103
+ parameters: list[Parameter] = [count, rate, *units]
104
+ return _problem(name, function, parameters, [(count >= i + 1) >> units[i] for i in range(1, 5)])
105
+
106
+
107
+ def conditional_problems() -> list[Problem]:
108
+ """Each conditional problem, followed by its flat version."""
109
+ return [*branches(2), *branches(5), *nested(), *layers()]
hone/benchmarks/ml.py CHANGED
@@ -16,7 +16,7 @@ from typing import Callable
16
16
 
17
17
  from threadpoolctl import threadpool_limits
18
18
 
19
- from hone.parameters import ChoiceParameter, RangeParameter, SearchSpace
19
+ from hone.parameters import ChoiceParameter, Condition, Parameter, RangeParameter, SearchSpace
20
20
 
21
21
  from .problems import Problem
22
22
 
@@ -92,11 +92,56 @@ def random_forest_wine(max_depth: int, min_samples_leaf: int, max_features: floa
92
92
  return _cross_validate(model, "wine", classification=True)
93
93
 
94
94
 
95
+ def model_selection_breast_cancer(
96
+ model: str,
97
+ C: float | None = None,
98
+ gamma: float | None = None,
99
+ max_depth: int | None = None,
100
+ min_samples_leaf: int | None = None,
101
+ max_features: float | None = None,
102
+ n_neighbors: int | None = None,
103
+ weights: str | None = None,
104
+ ) -> float:
105
+ """One of three classifiers, and its settings, on the 569 breast tumour samples svm-breast-cancer uses.
106
+
107
+ A support vector classifier with an RBF kernel, a 20-tree random forest, or k nearest
108
+ neighbours. With conditions, only the chosen model's settings are searched.
109
+ """
110
+ from sklearn.ensemble import RandomForestClassifier
111
+ from sklearn.neighbors import KNeighborsClassifier
112
+ from sklearn.pipeline import make_pipeline
113
+ from sklearn.preprocessing import StandardScaler
114
+ from sklearn.svm import SVC
115
+
116
+ if model == "svm":
117
+ estimator = make_pipeline(StandardScaler(), SVC(C=C, gamma=gamma))
118
+ elif model == "forest":
119
+ estimator = RandomForestClassifier(
120
+ n_estimators=20, max_depth=max_depth, min_samples_leaf=min_samples_leaf, max_features=max_features, random_state=0, n_jobs=1
121
+ )
122
+ else:
123
+ estimator = make_pipeline(StandardScaler(), KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights))
124
+ return _cross_validate(estimator, "breast_cancer", classification=True)
125
+
126
+
127
+ _MODEL = ChoiceParameter("model", ["svm", "forest", "knn"])
128
+ _SVM = [RangeParameter("C", 1e-3, 1e3, log=True), RangeParameter("gamma", 1e-5, 10, log=True)]
129
+ _FOREST = [
130
+ RangeParameter("max_depth", 1, 12, integer=True),
131
+ RangeParameter("min_samples_leaf", 1, 20, integer=True),
132
+ RangeParameter("max_features", 0.05, 1.0),
133
+ ]
134
+ _KNN: list[Parameter] = [RangeParameter("n_neighbors", 1, 50, integer=True), ChoiceParameter("weights", ["uniform", "distance"])]
135
+ MODEL_SELECTION: list[Parameter] = [_MODEL, *_SVM, *_FOREST, *_KNN]
136
+ MODEL_CONDITIONS: list[Condition] = [_MODEL["svm"] >> _SVM, _MODEL["forest"] >> _FOREST, _MODEL["knn"] >> _KNN]
137
+
95
138
  # The best value of each problem found by REFERENCE_SEARCH.
96
139
  OPTIMA = {
97
140
  "svm-breast-cancer": 0.014066138798323302, # accuracy 98.6%, found by CMA-ES
98
141
  "kernel-ridge-diabetes": 0.4961748264794844, # R² 0.504, found by Bayesian optimization
99
142
  "random-forest-wine": 0.005555555555555536, # accuracy 99.4%, found by Bayesian optimization
143
+ # Every reference run chose the support vector classifier, and the best matched svm-breast-cancer's.
144
+ "model-selection-breast-cancer": 0.014066138798323302,
100
145
  }
101
146
 
102
147
  SPACES: dict[str, tuple[Callable[..., float], list]] = {
@@ -125,9 +170,30 @@ SPACES: dict[str, tuple[Callable[..., float], list]] = {
125
170
 
126
171
 
127
172
  def ml_problems() -> list[Problem]:
128
- """The machine learning tuning problems. Needs scikit-learn."""
173
+ """The machine learning tuning problems. Needs scikit-learn.
174
+
175
+ model-selection-breast-cancer comes twice: with conditions, and as model-selection-breast-cancer-flat
176
+ with every setting always searched, as before conditions, to show what they are worth.
177
+ """
129
178
  _require_sklearn()
130
- return [
179
+ problems = [
131
180
  Problem(name=name, function=function, search_space=SearchSpace(parameters), optimum=OPTIMA[name], tags=frozenset({"ml"}))
132
181
  for name, (function, parameters) in SPACES.items()
133
182
  ]
183
+ optimum = OPTIMA["model-selection-breast-cancer"]
184
+ return problems + [
185
+ Problem(
186
+ name="model-selection-breast-cancer",
187
+ function=model_selection_breast_cancer,
188
+ search_space=SearchSpace(MODEL_SELECTION, conditions=MODEL_CONDITIONS),
189
+ optimum=optimum,
190
+ tags=frozenset({"ml", "conditional"}),
191
+ ),
192
+ Problem(
193
+ name="model-selection-breast-cancer-flat",
194
+ function=model_selection_breast_cancer,
195
+ search_space=SearchSpace(MODEL_SELECTION),
196
+ optimum=optimum,
197
+ tags=frozenset({"ml", "conditional", "flat"}),
198
+ ),
199
+ ]