laplacekit-bayesopt 0.1a0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- laplacekit_bayesopt-0.1a0/PKG-INFO +144 -0
- laplacekit_bayesopt-0.1a0/README.md +125 -0
- laplacekit_bayesopt-0.1a0/pyproject.toml +106 -0
- laplacekit_bayesopt-0.1a0/pyproject.toml.orig +82 -0
- laplacekit_bayesopt-0.1a0/src/laplacekit_bayesopt/__init__.py +64 -0
- laplacekit_bayesopt-0.1a0/src/laplacekit_bayesopt/paths.py +134 -0
- laplacekit_bayesopt-0.1a0/src/laplacekit_bayesopt/py.typed +0 -0
- laplacekit_bayesopt-0.1a0/src/laplacekit_bayesopt/surrogate.py +519 -0
- laplacekit_bayesopt-0.1a0/src/laplacekit_bayesopt/training.py +251 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: laplacekit-bayesopt
|
|
3
|
+
Version: 0.1a0
|
|
4
|
+
Summary: Bayesian optimization with laplacekit: Laplace-approximated neural networks as BoTorch surrogates
|
|
5
|
+
Author: Agustinus Kristiadi
|
|
6
|
+
Author-email: Agustinus Kristiadi <agustinus@kristia.de>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
12
|
+
Requires-Dist: laplacekit>=0.1a0,<0.2
|
|
13
|
+
Requires-Dist: laplacekit-marglik>=0.1a0,<0.2
|
|
14
|
+
Requires-Dist: botorch>=0.16
|
|
15
|
+
Requires-Dist: gpytorch>=1.14
|
|
16
|
+
Requires-Python: >=3.13
|
|
17
|
+
Project-URL: Homepage, https://github.com/laplacekit/laplacekit-bayesopt
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# laplacekit-bayesopt
|
|
21
|
+
|
|
22
|
+
A <code><b>laplacekit</b></code> plugin that turns a network into a
|
|
23
|
+
[BoTorch](https://botorch.org) surrogate, Bayesian optimization with
|
|
24
|
+
Laplace-approximated neural networks.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install laplacekit-bayesopt
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Usage
|
|
33
|
+
|
|
34
|
+
`LaplaceBNNSurrogate` is a BoTorch `Model`. Use it in a BoTorch loop in place of
|
|
35
|
+
`SingleTaskGP`:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import torch
|
|
39
|
+
from botorch.acquisition import LogExpectedImprovement
|
|
40
|
+
from botorch.models.transforms.input import Normalize
|
|
41
|
+
from botorch.models.transforms.outcome import Standardize
|
|
42
|
+
from botorch.optim import optimize_acqf
|
|
43
|
+
from torch import nn
|
|
44
|
+
|
|
45
|
+
from laplacekit_bayesopt import LaplaceBNNSurrogate
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def make_net():
|
|
49
|
+
return nn.Sequential(
|
|
50
|
+
nn.Linear(2, 50), nn.ReLU(), nn.Linear(50, 50), nn.ReLU(), nn.Linear(50, 1)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
for _ in range(rounds):
|
|
55
|
+
model = LaplaceBNNSurrogate(
|
|
56
|
+
make_net(),
|
|
57
|
+
train_X,
|
|
58
|
+
train_Y,
|
|
59
|
+
input_transform=Normalize(d=2),
|
|
60
|
+
outcome_transform=Standardize(m=1),
|
|
61
|
+
)
|
|
62
|
+
acq = LogExpectedImprovement(model, best_f=train_Y.max())
|
|
63
|
+
x, _ = optimize_acqf(acq, bounds=bounds, q=1, num_restarts=10, raw_samples=20)
|
|
64
|
+
train_X, train_Y = torch.cat([train_X, x]), torch.cat([train_Y, objective(x)])
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
As with `SingleTaskGP`, each round builds a new model from all the data, and the data
|
|
68
|
+
can go in by keyword (`train_X=`, `train_Y=`) or through
|
|
69
|
+
`LaplaceBNNSurrogate.construct_inputs(dataset, net=make_net())`, as Ax builds models. The
|
|
70
|
+
constructor trains the network it is given in place. A fresh network starts over; the
|
|
71
|
+
network of the last round starts from its trained weights, a warm start.
|
|
72
|
+
|
|
73
|
+
The posterior is the linearized Laplace approximation, which is a Gaussian process:
|
|
74
|
+
|
|
75
|
+
- `model.posterior(X)` is joint over the `q` points of each batch, so Monte Carlo
|
|
76
|
+
acquisition functions (`qLogNoisyExpectedImprovement`, `qLogExpectedHypervolumeImprovement`, ...)
|
|
77
|
+
see the correlations between points and outcomes.
|
|
78
|
+
- `model.condition_on_observations(X, Y)` conditions the GP on new points without
|
|
79
|
+
training. BoTorch's fantasies need this, so `qKnowledgeGradient` works.
|
|
80
|
+
- BoTorch's `PathwiseThompsonSampling` works on it: importing the package registers the
|
|
81
|
+
model with `draw_matheron_paths`. Each path is one function f(x; μ) + J(x) ε drawn from
|
|
82
|
+
the posterior, smooth in x, so it works with `optimize_acqf` and `optimize_acqf_discrete`
|
|
83
|
+
alike.
|
|
84
|
+
|
|
85
|
+
### Training
|
|
86
|
+
|
|
87
|
+
A `Trainer` trains the network and fits the Laplace approximation:
|
|
88
|
+
|
|
89
|
+
| Trainer | What it does |
|
|
90
|
+
| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
91
|
+
| `PosthocMarglik()` (default) | Adam on the mean squared error, then tunes the prior precision and the noise on the marginal likelihood. |
|
|
92
|
+
| `OnlineMarglik()` | Tunes the prior precision and the noise during training ([Immer et al. 2021](https://arxiv.org/abs/2104.04975)), through `lk.marglik.train`. |
|
|
93
|
+
|
|
94
|
+
They are laplace-bayesopt's `marglik_mode="posthoc"` and `"online"`. Both use KFAC over
|
|
95
|
+
all weights by default. `PosthocMarglik(subset=lk.subsets.last_layer)` gives a last-layer
|
|
96
|
+
Laplace BNN, and `hessian_approx=` takes another curvature:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
import laplacekit as lk
|
|
100
|
+
|
|
101
|
+
from laplacekit_bayesopt import PosthocMarglik
|
|
102
|
+
|
|
103
|
+
trainer = PosthocMarglik(epochs=500, subset=lk.subsets.last_layer, hessian_approx=lk.DenseGGN())
|
|
104
|
+
model = LaplaceBNNSurrogate(make_net(), train_X, train_Y, trainer=trainer)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Training and path draws use torch's global RNG, so `torch.manual_seed`
|
|
108
|
+
makes a run repeatable.
|
|
109
|
+
|
|
110
|
+
`train_Yvar=` fixes the observation noise variance instead of learning it, as in
|
|
111
|
+
`SingleTaskGP`. It is in the space of `train_Y` and goes through the outcome transform; one
|
|
112
|
+
value for all points and outcomes is supported. `model.noise_variance` is the noise, learned
|
|
113
|
+
or given, and `model.laplace_posterior` the Laplace posterior. Any callable with the
|
|
114
|
+
signature of `Trainer.__call__` works as a trainer.
|
|
115
|
+
|
|
116
|
+
## Examples
|
|
117
|
+
|
|
118
|
+
`examples/` holds runnable scripts:
|
|
119
|
+
|
|
120
|
+
- `singletask.py`: Branin with log expected improvement.
|
|
121
|
+
- `thompson.py`: Branin over 8,192 Sobol candidates with Thompson sampling.
|
|
122
|
+
- `multitask.py`: a composite objective over three outcomes, with Monte Carlo log expected
|
|
123
|
+
improvement.
|
|
124
|
+
- `multiobjective.py`: the Pareto front of Branin-Currin with log expected hypervolume
|
|
125
|
+
improvement.
|
|
126
|
+
|
|
127
|
+
## Development
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
make sync # uv sync
|
|
131
|
+
make check # ruff, ty, pytest
|
|
132
|
+
make examples
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## Citation
|
|
136
|
+
|
|
137
|
+
```bibtex
|
|
138
|
+
@inproceedings{kristiadi2023promises,
|
|
139
|
+
title={Promises and Pitfalls of the Linearized {L}aplace in {B}ayesian Optimization},
|
|
140
|
+
author={Kristiadi, Agustinus and Immer, Alexander and Eschenhagen, Runa and Fortuin, Vincent},
|
|
141
|
+
booktitle={AABI},
|
|
142
|
+
year={2023}
|
|
143
|
+
}
|
|
144
|
+
```
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# laplacekit-bayesopt
|
|
2
|
+
|
|
3
|
+
A <code><b>laplacekit</b></code> plugin that turns a network into a
|
|
4
|
+
[BoTorch](https://botorch.org) surrogate, Bayesian optimization with
|
|
5
|
+
Laplace-approximated neural networks.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install laplacekit-bayesopt
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
`LaplaceBNNSurrogate` is a BoTorch `Model`. Use it in a BoTorch loop in place of
|
|
16
|
+
`SingleTaskGP`:
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import torch
|
|
20
|
+
from botorch.acquisition import LogExpectedImprovement
|
|
21
|
+
from botorch.models.transforms.input import Normalize
|
|
22
|
+
from botorch.models.transforms.outcome import Standardize
|
|
23
|
+
from botorch.optim import optimize_acqf
|
|
24
|
+
from torch import nn
|
|
25
|
+
|
|
26
|
+
from laplacekit_bayesopt import LaplaceBNNSurrogate
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def make_net():
|
|
30
|
+
return nn.Sequential(
|
|
31
|
+
nn.Linear(2, 50), nn.ReLU(), nn.Linear(50, 50), nn.ReLU(), nn.Linear(50, 1)
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
for _ in range(rounds):
|
|
36
|
+
model = LaplaceBNNSurrogate(
|
|
37
|
+
make_net(),
|
|
38
|
+
train_X,
|
|
39
|
+
train_Y,
|
|
40
|
+
input_transform=Normalize(d=2),
|
|
41
|
+
outcome_transform=Standardize(m=1),
|
|
42
|
+
)
|
|
43
|
+
acq = LogExpectedImprovement(model, best_f=train_Y.max())
|
|
44
|
+
x, _ = optimize_acqf(acq, bounds=bounds, q=1, num_restarts=10, raw_samples=20)
|
|
45
|
+
train_X, train_Y = torch.cat([train_X, x]), torch.cat([train_Y, objective(x)])
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
As with `SingleTaskGP`, each round builds a new model from all the data, and the data
|
|
49
|
+
can go in by keyword (`train_X=`, `train_Y=`) or through
|
|
50
|
+
`LaplaceBNNSurrogate.construct_inputs(dataset, net=make_net())`, as Ax builds models. The
|
|
51
|
+
constructor trains the network it is given in place. A fresh network starts over; the
|
|
52
|
+
network of the last round starts from its trained weights, a warm start.
|
|
53
|
+
|
|
54
|
+
The posterior is the linearized Laplace approximation, which is a Gaussian process:
|
|
55
|
+
|
|
56
|
+
- `model.posterior(X)` is joint over the `q` points of each batch, so Monte Carlo
|
|
57
|
+
acquisition functions (`qLogNoisyExpectedImprovement`, `qLogExpectedHypervolumeImprovement`, ...)
|
|
58
|
+
see the correlations between points and outcomes.
|
|
59
|
+
- `model.condition_on_observations(X, Y)` conditions the GP on new points without
|
|
60
|
+
training. BoTorch's fantasies need this, so `qKnowledgeGradient` works.
|
|
61
|
+
- BoTorch's `PathwiseThompsonSampling` works on it: importing the package registers the
|
|
62
|
+
model with `draw_matheron_paths`. Each path is one function f(x; μ) + J(x) ε drawn from
|
|
63
|
+
the posterior, smooth in x, so it works with `optimize_acqf` and `optimize_acqf_discrete`
|
|
64
|
+
alike.
|
|
65
|
+
|
|
66
|
+
### Training
|
|
67
|
+
|
|
68
|
+
A `Trainer` trains the network and fits the Laplace approximation:
|
|
69
|
+
|
|
70
|
+
| Trainer | What it does |
|
|
71
|
+
| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
72
|
+
| `PosthocMarglik()` (default) | Adam on the mean squared error, then tunes the prior precision and the noise on the marginal likelihood. |
|
|
73
|
+
| `OnlineMarglik()` | Tunes the prior precision and the noise during training ([Immer et al. 2021](https://arxiv.org/abs/2104.04975)), through `lk.marglik.train`. |
|
|
74
|
+
|
|
75
|
+
They are laplace-bayesopt's `marglik_mode="posthoc"` and `"online"`. Both use KFAC over
|
|
76
|
+
all weights by default. `PosthocMarglik(subset=lk.subsets.last_layer)` gives a last-layer
|
|
77
|
+
Laplace BNN, and `hessian_approx=` takes another curvature:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import laplacekit as lk
|
|
81
|
+
|
|
82
|
+
from laplacekit_bayesopt import PosthocMarglik
|
|
83
|
+
|
|
84
|
+
trainer = PosthocMarglik(epochs=500, subset=lk.subsets.last_layer, hessian_approx=lk.DenseGGN())
|
|
85
|
+
model = LaplaceBNNSurrogate(make_net(), train_X, train_Y, trainer=trainer)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Training and path draws use torch's global RNG, so `torch.manual_seed`
|
|
89
|
+
makes a run repeatable.
|
|
90
|
+
|
|
91
|
+
`train_Yvar=` fixes the observation noise variance instead of learning it, as in
|
|
92
|
+
`SingleTaskGP`. It is in the space of `train_Y` and goes through the outcome transform; one
|
|
93
|
+
value for all points and outcomes is supported. `model.noise_variance` is the noise, learned
|
|
94
|
+
or given, and `model.laplace_posterior` the Laplace posterior. Any callable with the
|
|
95
|
+
signature of `Trainer.__call__` works as a trainer.
|
|
96
|
+
|
|
97
|
+
## Examples
|
|
98
|
+
|
|
99
|
+
`examples/` holds runnable scripts:
|
|
100
|
+
|
|
101
|
+
- `singletask.py`: Branin with log expected improvement.
|
|
102
|
+
- `thompson.py`: Branin over 8,192 Sobol candidates with Thompson sampling.
|
|
103
|
+
- `multitask.py`: a composite objective over three outcomes, with Monte Carlo log expected
|
|
104
|
+
improvement.
|
|
105
|
+
- `multiobjective.py`: the Pareto front of Branin-Currin with log expected hypervolume
|
|
106
|
+
improvement.
|
|
107
|
+
|
|
108
|
+
## Development
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
make sync # uv sync
|
|
112
|
+
make check # ruff, ty, pytest
|
|
113
|
+
make examples
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Citation
|
|
117
|
+
|
|
118
|
+
```bibtex
|
|
119
|
+
@inproceedings{kristiadi2023promises,
|
|
120
|
+
title={Promises and Pitfalls of the Linearized {L}aplace in {B}ayesian Optimization},
|
|
121
|
+
author={Kristiadi, Agustinus and Immer, Alexander and Eschenhagen, Runa and Fortuin, Vincent},
|
|
122
|
+
booktitle={AABI},
|
|
123
|
+
year={2023}
|
|
124
|
+
}
|
|
125
|
+
```
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "laplacekit-bayesopt"
|
|
3
|
+
version = "0.1a0"
|
|
4
|
+
description = "Bayesian optimization with laplacekit: Laplace-approximated neural networks as BoTorch surrogates"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.13"
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Development Status :: 3 - Alpha",
|
|
10
|
+
"Operating System :: OS Independent",
|
|
11
|
+
"Programming Language :: Python :: 3.13",
|
|
12
|
+
"Programming Language :: Python :: 3.14",
|
|
13
|
+
]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"laplacekit>=0.1a0,<0.2",
|
|
16
|
+
"laplacekit-marglik>=0.1a0,<0.2",
|
|
17
|
+
"botorch>=0.16",
|
|
18
|
+
"gpytorch>=1.14",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[[project.authors]]
|
|
22
|
+
name = "Agustinus Kristiadi"
|
|
23
|
+
email = "agustinus@kristia.de"
|
|
24
|
+
|
|
25
|
+
[project.entry-points."laplacekit.plugins"]
|
|
26
|
+
laplacekit_bayesopt = "laplacekit_bayesopt:plugin"
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/laplacekit/laplacekit-bayesopt"
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["uv_build>=0.13.0,<0.14"]
|
|
33
|
+
build-backend = "uv_build"
|
|
34
|
+
|
|
35
|
+
[dependency-groups]
|
|
36
|
+
dev = [
|
|
37
|
+
"pytest>=8",
|
|
38
|
+
"ty>=0.0.104",
|
|
39
|
+
"ruff>=0.14",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[[tool.uv.index]]
|
|
43
|
+
name = "pytorch-cpu"
|
|
44
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
45
|
+
explicit = true
|
|
46
|
+
|
|
47
|
+
[[tool.uv.sources.torch]]
|
|
48
|
+
index = "pytorch-cpu"
|
|
49
|
+
|
|
50
|
+
[tool.uv.sources.laplacekit]
|
|
51
|
+
git = "https://github.com/laplacekit/laplacekit"
|
|
52
|
+
branch = "main"
|
|
53
|
+
|
|
54
|
+
[tool.uv.sources.laplacekit-marglik]
|
|
55
|
+
git = "https://github.com/laplacekit/laplacekit"
|
|
56
|
+
branch = "main"
|
|
57
|
+
subdirectory = "plugins/marglik"
|
|
58
|
+
|
|
59
|
+
[tool.ty.environment]
|
|
60
|
+
python-version = "3.13"
|
|
61
|
+
|
|
62
|
+
[tool.ruff]
|
|
63
|
+
line-length = 100
|
|
64
|
+
target-version = "py313"
|
|
65
|
+
|
|
66
|
+
[tool.ruff.lint]
|
|
67
|
+
select = [
|
|
68
|
+
"E",
|
|
69
|
+
"F",
|
|
70
|
+
"I",
|
|
71
|
+
"UP",
|
|
72
|
+
"B",
|
|
73
|
+
"D",
|
|
74
|
+
]
|
|
75
|
+
ignore = [
|
|
76
|
+
"D105",
|
|
77
|
+
"D107",
|
|
78
|
+
"D202",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
[tool.ruff.lint.isort]
|
|
82
|
+
known-first-party = [
|
|
83
|
+
"laplacekit",
|
|
84
|
+
"laplacekit_bayesopt",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
[tool.ruff.lint.flake8-bugbear]
|
|
88
|
+
extend-immutable-calls = ["laplacekit_bayesopt.training.PosthocMarglik"]
|
|
89
|
+
|
|
90
|
+
[tool.ruff.lint.pydocstyle]
|
|
91
|
+
convention = "google"
|
|
92
|
+
|
|
93
|
+
[tool.ruff.lint.per-file-ignores]
|
|
94
|
+
"tests/**" = ["D"]
|
|
95
|
+
"examples/**" = [
|
|
96
|
+
"D",
|
|
97
|
+
"E402",
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
[tool.pytest.ini_options]
|
|
101
|
+
testpaths = ["tests"]
|
|
102
|
+
filterwarnings = ["ignore:`torch.jit.script` is not supported:FutureWarning:torch.jit._script"]
|
|
103
|
+
|
|
104
|
+
[tool.pyright]
|
|
105
|
+
typeCheckingMode = "standard"
|
|
106
|
+
pythonVersion = "3.13"
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "laplacekit-bayesopt"
|
|
3
|
+
version = "0.1a0"
|
|
4
|
+
description = "Bayesian optimization with laplacekit: Laplace-approximated neural networks as BoTorch surrogates"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [{ name = "Agustinus Kristiadi", email = "agustinus@kristia.de" }]
|
|
7
|
+
license = "MIT"
|
|
8
|
+
requires-python = ">=3.13"
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 3 - Alpha",
|
|
11
|
+
"Operating System :: OS Independent",
|
|
12
|
+
"Programming Language :: Python :: 3.13",
|
|
13
|
+
"Programming Language :: Python :: 3.14",
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"laplacekit>=0.1a0,<0.2",
|
|
17
|
+
"laplacekit-marglik>=0.1a0,<0.2",
|
|
18
|
+
"botorch>=0.16",
|
|
19
|
+
"gpytorch>=1.14",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.entry-points."laplacekit.plugins"]
|
|
23
|
+
laplacekit_bayesopt = "laplacekit_bayesopt:plugin"
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/laplacekit/laplacekit-bayesopt"
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["uv_build>=0.13.0,<0.14"]
|
|
30
|
+
build-backend = "uv_build"
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = ["pytest>=8", "ty>=0.0.104", "ruff>=0.14"]
|
|
34
|
+
|
|
35
|
+
[[tool.uv.index]]
|
|
36
|
+
name = "pytorch-cpu"
|
|
37
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
38
|
+
explicit = true
|
|
39
|
+
|
|
40
|
+
[tool.uv.sources]
|
|
41
|
+
torch = [{ index = "pytorch-cpu" }]
|
|
42
|
+
laplacekit = { git = "https://github.com/laplacekit/laplacekit", branch = "main" }
|
|
43
|
+
laplacekit-marglik = { git = "https://github.com/laplacekit/laplacekit", branch = "main", subdirectory = "plugins/marglik" }
|
|
44
|
+
|
|
45
|
+
[tool.ty.environment]
|
|
46
|
+
python-version = "3.13"
|
|
47
|
+
|
|
48
|
+
[tool.ruff]
|
|
49
|
+
line-length = 100
|
|
50
|
+
target-version = "py313"
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint]
|
|
53
|
+
select = ["E", "F", "I", "UP", "B", "D"]
|
|
54
|
+
# D105, D107: dunders and `__init__` are covered by the class docstring. D202: the house
|
|
55
|
+
# style puts a blank line after a function's docstring.
|
|
56
|
+
ignore = ["D105", "D107", "D202"]
|
|
57
|
+
|
|
58
|
+
[tool.ruff.lint.isort]
|
|
59
|
+
known-first-party = ["laplacekit", "laplacekit_bayesopt"]
|
|
60
|
+
|
|
61
|
+
[tool.ruff.lint.flake8-bugbear]
|
|
62
|
+
# Frozen dataclasses, safe as argument defaults (B008).
|
|
63
|
+
extend-immutable-calls = ["laplacekit_bayesopt.training.PosthocMarglik"]
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint.pydocstyle]
|
|
66
|
+
convention = "google"
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint.per-file-ignores]
|
|
69
|
+
"tests/**" = ["D"]
|
|
70
|
+
"examples/**" = ["D", "E402"]
|
|
71
|
+
|
|
72
|
+
[tool.pytest.ini_options]
|
|
73
|
+
testpaths = ["tests"]
|
|
74
|
+
filterwarnings = [
|
|
75
|
+
# torch's own torch._decomp calls torch.jit.script, which warns on Python 3.14+. Not our code.
|
|
76
|
+
"ignore:`torch.jit.script` is not supported:FutureWarning:torch.jit._script",
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
# VS Code's Pylance reads this: it marks a class that misses a protocol method in red.
|
|
80
|
+
[tool.pyright]
|
|
81
|
+
typeCheckingMode = "standard"
|
|
82
|
+
pythonVersion = "3.13"
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
r"""Bayesian optimization with Laplace-approximated neural networks, as BoTorch models.
|
|
2
|
+
|
|
3
|
+
`LaplaceBNNSurrogate` is a BoTorch `Model`. Build it from the data, as `SingleTaskGP`, and use
|
|
4
|
+
it with BoTorch's acquisition functions and optimizers:
|
|
5
|
+
|
|
6
|
+
```python
|
|
7
|
+
from botorch.acquisition import LogExpectedImprovement
|
|
8
|
+
from botorch.optim import optimize_acqf
|
|
9
|
+
from laplacekit_bayesopt import LaplaceBNNSurrogate
|
|
10
|
+
|
|
11
|
+
for _ in range(rounds):
|
|
12
|
+
model = LaplaceBNNSurrogate(make_net(), train_X, train_Y)
|
|
13
|
+
acq = LogExpectedImprovement(model, best_f=train_Y.max())
|
|
14
|
+
x, _ = optimize_acqf(acq, bounds=bounds, q=1, num_restarts=10, raw_samples=512)
|
|
15
|
+
train_X, train_Y = torch.cat([train_X, x]), torch.cat([train_Y, objective(x)])
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The posterior is the linearized Laplace approximation (Kristiadi et al. 2023, "Promises
|
|
19
|
+
and Pitfalls of the Linearized Laplace in Bayesian Optimization"). It is a Gaussian
|
|
20
|
+
process, so `LaplaceBNNSurrogate.posterior` is joint over the points of a batch, and
|
|
21
|
+
`LaplaceBNNSurrogate.condition_on_observations` conditions it on new points without training,
|
|
22
|
+
which BoTorch's fantasies (`qKnowledgeGradient`, say) rely on.
|
|
23
|
+
|
|
24
|
+
Importing the package registers the model with BoTorch's `draw_matheron_paths`, so
|
|
25
|
+
`PathwiseThompsonSampling` draws functions from the linearized posterior.
|
|
26
|
+
|
|
27
|
+
A `Trainer` trains the network and fits the approximation: `PosthocMarglik` (the default)
|
|
28
|
+
tunes the prior precision and the noise on the marginal likelihood after training,
|
|
29
|
+
`OnlineMarglik` during training. A reused network starts from its trained weights (a warm
|
|
30
|
+
start); pass a fresh one for a fresh start.
|
|
31
|
+
|
|
32
|
+
This is the successor of `laplace-bayesopt`, built on **laplacekit** instead of
|
|
33
|
+
`laplace-torch`.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from importlib import metadata
|
|
37
|
+
|
|
38
|
+
import laplacekit as lk
|
|
39
|
+
from laplacekit_bayesopt import paths as _paths # noqa: F401 # registers the paths
|
|
40
|
+
from laplacekit_bayesopt.surrogate import LaplaceBNNSurrogate
|
|
41
|
+
from laplacekit_bayesopt.training import OnlineMarglik, PosthocMarglik, Trainer
|
|
42
|
+
|
|
43
|
+
__all__ = ["LaplaceBNNSurrogate", "OnlineMarglik", "PosthocMarglik", "Trainer"]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def health() -> tuple[lk.HealthCheck, ...]:
|
|
47
|
+
"""Check for `lk.health()` that BoTorch and GPyTorch are installed, with their versions.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
One check per library.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
return tuple(_installed(dist) for dist in ("botorch", "gpytorch"))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _installed(dist: str) -> lk.HealthCheck:
|
|
57
|
+
try:
|
|
58
|
+
return lk.HealthCheck(dist, "ok", metadata.version(dist))
|
|
59
|
+
except metadata.PackageNotFoundError:
|
|
60
|
+
return lk.HealthCheck(dist, "error", "not installed (required)")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
plugin = lk.Plugin(supports=">=0.1a0,<0.2", health=health)
|
|
64
|
+
"""The record behind the `laplacekit.plugins` entry point."""
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
r"""Sample paths of `LaplaceBNNSurrogate` for BoTorch's pathwise sampling.
|
|
2
|
+
|
|
3
|
+
A path is one function drawn from the linearized Laplace posterior,
|
|
4
|
+
$f_s(x) = f(x; \mu) + J(x) \varepsilon_s$, with one weight draw
|
|
5
|
+
$\varepsilon_s \sim \mathcal{N}(0, \Sigma)$ for the whole input space. Importing the
|
|
6
|
+
package registers `LaplaceBNNSurrogate` with BoTorch's `draw_matheron_paths`, so BoTorch's
|
|
7
|
+
pathwise tools work on it unchanged:
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from botorch.acquisition.thompson_sampling import PathwiseThompsonSampling
|
|
11
|
+
|
|
12
|
+
acq = PathwiseThompsonSampling(model)
|
|
13
|
+
x, _ = optimize_acqf(acq, bounds=bounds, q=1, num_restarts=10, raw_samples=512)
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The draws come from torch's global RNG, so `torch.manual_seed` makes them repeatable.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import math
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
from botorch.sampling.pathwise import SamplePath
|
|
24
|
+
from botorch.sampling.pathwise.posterior_samplers import DrawMatheronPaths
|
|
25
|
+
from botorch.sampling.pathwise.prior_samplers import draw_kernel_feature_paths
|
|
26
|
+
from botorch.sampling.pathwise.update_strategies import gaussian_update
|
|
27
|
+
from torch import Size, Tensor
|
|
28
|
+
|
|
29
|
+
from laplacekit.core import CapabilityError
|
|
30
|
+
from laplacekit_bayesopt.surrogate import LaplaceBNNSurrogate
|
|
31
|
+
|
|
32
|
+
__all__: list[str] = []
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _LinearizedPaths(SamplePath):
|
|
36
|
+
"""`sample_shape` functions drawn from the linearized posterior of `model`.
|
|
37
|
+
|
|
38
|
+
Called on inputs `[*batch, n, d]` in the original space, the paths return
|
|
39
|
+
`[*broadcast(batch, sample_shape), n]` for one outcome. For several outcomes, the
|
|
40
|
+
inputs carry an extra dim of size 1 before `n`, as `MatheronPathModel` passes them,
|
|
41
|
+
and the result is `[..., m, n]`, the layout of BoTorch's multi-output GP paths. The
|
|
42
|
+
outcomes are in the original space.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(self, model: LaplaceBNNSurrogate, sample_shape: Size) -> None:
|
|
46
|
+
super().__init__()
|
|
47
|
+
|
|
48
|
+
noise = model.laplace_posterior.sample_centered(math.prod(sample_shape))
|
|
49
|
+
self.surrogate = model
|
|
50
|
+
self.sample_shape = sample_shape
|
|
51
|
+
self.eps = {k: v.reshape(*sample_shape, *v.shape[1:]) for k, v in noise.items()}
|
|
52
|
+
|
|
53
|
+
def set_ensemble_as_batch(self, ensemble_as_batch: bool) -> None:
|
|
54
|
+
"""Do nothing: the model is not an ensemble.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
ensemble_as_batch: Ignored.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def forward(self, X: Tensor, **kwargs: Any) -> Tensor:
|
|
61
|
+
"""The paths' values at `X`.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
X: The inputs, `[*batch, n, d]`, or `[*batch, 1, n, d]` for several outcomes.
|
|
65
|
+
**kwargs: Ignored.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
`[*batch', n]` for one outcome, `[*batch', m, n]` for several, where `batch'`
|
|
69
|
+
broadcasts `batch` with the sample shape.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
model = self.surrogate
|
|
73
|
+
m = model.num_outputs
|
|
74
|
+
|
|
75
|
+
if m > 1:
|
|
76
|
+
X = X.squeeze(-3)
|
|
77
|
+
|
|
78
|
+
X_tf = model.transform_inputs(X)
|
|
79
|
+
n, d = X_tf.shape[-2:]
|
|
80
|
+
batch = torch.broadcast_shapes(X_tf.shape[:-2], self.sample_shape)
|
|
81
|
+
lead = len(batch) - len(self.sample_shape)
|
|
82
|
+
|
|
83
|
+
inputs = X_tf.expand(*batch, n, d).reshape(-1, n, d)
|
|
84
|
+
mean = {k: v.detach() for k, v in model.laplace_posterior.mean.items()}
|
|
85
|
+
eps = {
|
|
86
|
+
k: e.reshape(*[1] * lead, *e.shape)
|
|
87
|
+
.expand(*batch, *e.shape[len(self.sample_shape) :])
|
|
88
|
+
.reshape(inputs.shape[0], *e.shape[len(self.sample_shape) :])
|
|
89
|
+
.to(mean[k])
|
|
90
|
+
for k, e in self.eps.items()
|
|
91
|
+
}
|
|
92
|
+
fn = model.laplace_posterior.model
|
|
93
|
+
|
|
94
|
+
# f(x; μ) + J(x) ε for each input set with its own ε, one JVP per set.
|
|
95
|
+
def one(x: Tensor, e: dict[str, Tensor]) -> Tensor:
|
|
96
|
+
f, lin = torch.func.jvp(lambda p: torch.as_tensor(fn(p, x)), (mean,), (e,))[:2]
|
|
97
|
+
|
|
98
|
+
return f + lin
|
|
99
|
+
|
|
100
|
+
out = torch.vmap(one)(inputs, eps).reshape(*batch, n, m)
|
|
101
|
+
|
|
102
|
+
if hasattr(model, "outcome_transform"):
|
|
103
|
+
out, _ = model.outcome_transform.untransform(out, X=X_tf)
|
|
104
|
+
|
|
105
|
+
return out.squeeze(-1) if m == 1 else out.transpose(-1, -2)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@DrawMatheronPaths.register(LaplaceBNNSurrogate)
|
|
109
|
+
def _draw_linearized_paths(
|
|
110
|
+
model: LaplaceBNNSurrogate,
|
|
111
|
+
*,
|
|
112
|
+
sample_shape: Size,
|
|
113
|
+
prior_sampler: object = draw_kernel_feature_paths,
|
|
114
|
+
update_strategy: object = gaussian_update,
|
|
115
|
+
) -> _LinearizedPaths:
|
|
116
|
+
"""Draw `sample_shape` paths of the linearized Laplace posterior.
|
|
117
|
+
|
|
118
|
+
`draw_matheron_paths` passes a GP prior sampler and update strategy. The Laplace paths
|
|
119
|
+
have neither, so only BoTorch's defaults are accepted.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
if prior_sampler is not draw_kernel_feature_paths or update_strategy is not gaussian_update:
|
|
123
|
+
raise CapabilityError(
|
|
124
|
+
"LaplaceBNNSurrogate draws its paths from the weight posterior, so it takes no "
|
|
125
|
+
"prior_sampler or update_strategy; leave them at BoTorch's defaults"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
if model._conditioning is not None:
|
|
129
|
+
raise CapabilityError(
|
|
130
|
+
"paths of LaplaceBNNSurrogate come from the fitted posterior, so they can't "
|
|
131
|
+
"follow condition_on_observations; build a new model with the data instead"
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
return _LinearizedPaths(model, sample_shape)
|
|
File without changes
|