pyperch 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyperch/__init__.py +5 -0
- pyperch/optim/__init__.py +11 -0
- pyperch/optim/base.py +51 -0
- pyperch/optim/ga.py +284 -0
- pyperch/optim/rhc.py +154 -0
- pyperch/optim/sa.py +191 -0
- pyperch/search/__init__.py +4 -0
- pyperch/search/grid.py +22 -0
- pyperch/search/optuna.py +8 -0
- pyperch/search/random.py +25 -0
- pyperch/utils/__init__.py +4 -0
- pyperch/utils/random.py +9 -0
- pyperch/utils/tracking.py +23 -0
- pyperch-0.3.0.dist-info/METADATA +32 -0
- pyperch-0.3.0.dist-info/RECORD +16 -0
- pyperch-0.3.0.dist-info/WHEEL +4 -0
pyperch/__init__.py
ADDED
pyperch/optim/base.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RandomizedOptimizer(torch.optim.Optimizer):
|
|
9
|
+
"""Base class for randomized optimizers that operate on PyTorch parameters."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, params: Iterable[torch.nn.Parameter], defaults: dict):
|
|
12
|
+
super().__init__(params, defaults)
|
|
13
|
+
self.reset_counters()
|
|
14
|
+
|
|
15
|
+
def reset_counters(self) -> None:
|
|
16
|
+
"""Reset optimization counters without changing model parameters."""
|
|
17
|
+
self.function_evals = 0
|
|
18
|
+
self.proposed_steps = 0
|
|
19
|
+
self.accepted_steps = 0
|
|
20
|
+
self.rejected_steps = 0
|
|
21
|
+
self.best_loss: float | None = None
|
|
22
|
+
|
|
23
|
+
@torch.no_grad()
|
|
24
|
+
def _parameters(self) -> list[torch.nn.Parameter]:
|
|
25
|
+
"""Return trainable parameters managed by this optimizer."""
|
|
26
|
+
return [
|
|
27
|
+
p for group in self.param_groups for p in group["params"] if p.requires_grad
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
@torch.no_grad()
|
|
31
|
+
def _clone_params(self) -> list[torch.Tensor]:
|
|
32
|
+
"""Copy the current trainable parameter values."""
|
|
33
|
+
return [p.detach().clone() for p in self._parameters()]
|
|
34
|
+
|
|
35
|
+
@torch.no_grad()
|
|
36
|
+
def _restore_params(self, values: list[torch.Tensor]) -> None:
|
|
37
|
+
"""Restore trainable parameters from a copied parameter list."""
|
|
38
|
+
for p, value in zip(self._parameters(), values):
|
|
39
|
+
p.copy_(value)
|
|
40
|
+
|
|
41
|
+
def _record_eval(self, loss: float | None = None) -> None:
|
|
42
|
+
"""Record one objective evaluation and optionally update the best loss."""
|
|
43
|
+
self.function_evals += 1
|
|
44
|
+
|
|
45
|
+
if loss is not None:
|
|
46
|
+
self._update_best_loss(loss)
|
|
47
|
+
|
|
48
|
+
def _update_best_loss(self, loss: float) -> None:
|
|
49
|
+
"""Update the best loss when improved."""
|
|
50
|
+
if self.best_loss is None or loss < self.best_loss:
|
|
51
|
+
self.best_loss = loss
|
pyperch/optim/ga.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""Genetic Algorithm optimizer.
|
|
2
|
+
|
|
3
|
+
Randomized Optimization methods for PyPerch.
|
|
4
|
+
|
|
5
|
+
Based on the original PyPerch optimizers by Jakub Owczarek
|
|
6
|
+
(BSD 3-Clause License).
|
|
7
|
+
|
|
8
|
+
These were also inspired by ABAGAIL’s randomized optimization algorithms - https://github.com/pushkar/ABAGAIL.
|
|
9
|
+
|
|
10
|
+
Substantial refactoring and redesign by John Mansfield (2026).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from .base import RandomizedOptimizer
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class GA(RandomizedOptimizer):
|
|
23
|
+
"""Genetic algorithm optimizer for arbitrary PyTorch models.
|
|
24
|
+
|
|
25
|
+
This optimizer builds a population around the current parameters,
|
|
26
|
+
evaluates candidate solutions, applies selection, crossover, mutation,
|
|
27
|
+
and adopts the best candidate when it improves the loss.
|
|
28
|
+
|
|
29
|
+
Lower loss is assumed to be better.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
params,
|
|
35
|
+
population_size: int = 50,
|
|
36
|
+
mutation_rate: float = 0.1,
|
|
37
|
+
step_size: float = 0.1,
|
|
38
|
+
random_state: int | None = None,
|
|
39
|
+
):
|
|
40
|
+
if population_size < 2:
|
|
41
|
+
raise ValueError("population_size must be at least 2.")
|
|
42
|
+
if mutation_rate < 0 or mutation_rate > 1:
|
|
43
|
+
raise ValueError("mutation_rate must be in the interval [0, 1].")
|
|
44
|
+
if step_size <= 0:
|
|
45
|
+
raise ValueError("step_size must be positive.")
|
|
46
|
+
|
|
47
|
+
defaults = {
|
|
48
|
+
"population_size": population_size,
|
|
49
|
+
"mutation_rate": mutation_rate,
|
|
50
|
+
"step_size": step_size,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
super().__init__(params, defaults)
|
|
54
|
+
|
|
55
|
+
self.population_size = population_size
|
|
56
|
+
self.mutation_rate = mutation_rate
|
|
57
|
+
self.step_size = step_size
|
|
58
|
+
|
|
59
|
+
self._generator = torch.Generator()
|
|
60
|
+
if random_state is not None:
|
|
61
|
+
self._generator.manual_seed(random_state)
|
|
62
|
+
|
|
63
|
+
self._initialized = False
|
|
64
|
+
self._current_loss: float | None = None
|
|
65
|
+
self._best_params: list[torch.Tensor] | None = None
|
|
66
|
+
|
|
67
|
+
def step(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
|
|
68
|
+
if closure is None:
|
|
69
|
+
raise ValueError("GA requires a closure that returns the loss.")
|
|
70
|
+
|
|
71
|
+
if not self._initialized:
|
|
72
|
+
loss_tensor = self._evaluate_loss(closure)
|
|
73
|
+
loss = float(loss_tensor.detach().item())
|
|
74
|
+
|
|
75
|
+
self._initialized = True
|
|
76
|
+
self._current_loss = loss
|
|
77
|
+
self._update_best_loss(loss)
|
|
78
|
+
self._best_params = self._clone_params()
|
|
79
|
+
|
|
80
|
+
return loss_tensor
|
|
81
|
+
|
|
82
|
+
current_params = self._clone_params()
|
|
83
|
+
current_loss = self._current_loss
|
|
84
|
+
|
|
85
|
+
population = self._initialize_population(current_params)
|
|
86
|
+
population_losses = self._evaluate_population(population, closure)
|
|
87
|
+
|
|
88
|
+
selected = self._select_population(population, population_losses)
|
|
89
|
+
children = self._crossover(selected)
|
|
90
|
+
children = self._mutate(children)
|
|
91
|
+
|
|
92
|
+
candidate_population = selected + children
|
|
93
|
+
candidate_losses = self._evaluate_population(candidate_population, closure)
|
|
94
|
+
|
|
95
|
+
best_idx = min(
|
|
96
|
+
range(len(candidate_losses)),
|
|
97
|
+
key=lambda idx: candidate_losses[idx],
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
best_candidate = candidate_population[best_idx]
|
|
101
|
+
best_candidate_loss = candidate_losses[best_idx]
|
|
102
|
+
|
|
103
|
+
self.proposed_steps += len(candidate_population)
|
|
104
|
+
|
|
105
|
+
if best_candidate_loss <= current_loss:
|
|
106
|
+
self._restore_params(best_candidate)
|
|
107
|
+
self.accepted_steps += 1
|
|
108
|
+
self._current_loss = best_candidate_loss
|
|
109
|
+
|
|
110
|
+
if self.best_loss is None or best_candidate_loss < self.best_loss:
|
|
111
|
+
self.best_loss = best_candidate_loss
|
|
112
|
+
self._best_params = self._clone_params()
|
|
113
|
+
|
|
114
|
+
return torch.tensor(best_candidate_loss)
|
|
115
|
+
|
|
116
|
+
self._restore_params(current_params)
|
|
117
|
+
self.rejected_steps += 1
|
|
118
|
+
|
|
119
|
+
return torch.tensor(current_loss)
|
|
120
|
+
|
|
121
|
+
def _evaluate_loss(
|
|
122
|
+
self,
|
|
123
|
+
closure: Callable[[], torch.Tensor],
|
|
124
|
+
) -> torch.Tensor:
|
|
125
|
+
with torch.enable_grad():
|
|
126
|
+
loss_tensor = closure()
|
|
127
|
+
|
|
128
|
+
self._record_eval(float(loss_tensor.detach().item()))
|
|
129
|
+
return loss_tensor
|
|
130
|
+
|
|
131
|
+
@torch.no_grad()
|
|
132
|
+
def _initialize_population(
|
|
133
|
+
self,
|
|
134
|
+
base_params: list[torch.Tensor],
|
|
135
|
+
) -> list[list[torch.Tensor]]:
|
|
136
|
+
population = [base_params]
|
|
137
|
+
|
|
138
|
+
for _ in range(self.population_size - 1):
|
|
139
|
+
individual = []
|
|
140
|
+
|
|
141
|
+
for param in base_params:
|
|
142
|
+
noise = torch.randn(
|
|
143
|
+
param.shape,
|
|
144
|
+
generator=self._generator,
|
|
145
|
+
device=param.device,
|
|
146
|
+
dtype=param.dtype,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
individual.append(param + self.step_size * noise)
|
|
150
|
+
|
|
151
|
+
population.append(individual)
|
|
152
|
+
|
|
153
|
+
return population
|
|
154
|
+
|
|
155
|
+
def _evaluate_population(
|
|
156
|
+
self,
|
|
157
|
+
population: list[list[torch.Tensor]],
|
|
158
|
+
closure: Callable[[], torch.Tensor],
|
|
159
|
+
) -> list[float]:
|
|
160
|
+
original = self._clone_params()
|
|
161
|
+
losses = []
|
|
162
|
+
|
|
163
|
+
for individual in population:
|
|
164
|
+
self._restore_params(individual)
|
|
165
|
+
|
|
166
|
+
with torch.enable_grad():
|
|
167
|
+
loss_tensor = closure()
|
|
168
|
+
|
|
169
|
+
loss = float(loss_tensor.detach().item())
|
|
170
|
+
self._record_eval()
|
|
171
|
+
losses.append(loss)
|
|
172
|
+
|
|
173
|
+
self._restore_params(original)
|
|
174
|
+
return losses
|
|
175
|
+
|
|
176
|
+
def _select_population(
|
|
177
|
+
self,
|
|
178
|
+
population: list[list[torch.Tensor]],
|
|
179
|
+
losses: list[float],
|
|
180
|
+
) -> list[list[torch.Tensor]]:
|
|
181
|
+
keep_count = max(2, self.population_size // 2)
|
|
182
|
+
|
|
183
|
+
ranked_indices = sorted(
|
|
184
|
+
range(len(losses)),
|
|
185
|
+
key=lambda idx: losses[idx],
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
return [
|
|
189
|
+
self._clone_individual(population[idx])
|
|
190
|
+
for idx in ranked_indices[:keep_count]
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
@torch.no_grad()
|
|
194
|
+
def _crossover(
|
|
195
|
+
self,
|
|
196
|
+
parents: list[list[torch.Tensor]],
|
|
197
|
+
) -> list[list[torch.Tensor]]:
|
|
198
|
+
child_count = self.population_size - len(parents)
|
|
199
|
+
children = []
|
|
200
|
+
|
|
201
|
+
for _ in range(child_count):
|
|
202
|
+
idx1 = torch.randint(
|
|
203
|
+
low=0,
|
|
204
|
+
high=len(parents),
|
|
205
|
+
size=(1,),
|
|
206
|
+
generator=self._generator,
|
|
207
|
+
).item()
|
|
208
|
+
|
|
209
|
+
idx2 = torch.randint(
|
|
210
|
+
low=0,
|
|
211
|
+
high=len(parents),
|
|
212
|
+
size=(1,),
|
|
213
|
+
generator=self._generator,
|
|
214
|
+
).item()
|
|
215
|
+
|
|
216
|
+
parent1 = parents[idx1]
|
|
217
|
+
parent2 = parents[idx2]
|
|
218
|
+
|
|
219
|
+
child = []
|
|
220
|
+
|
|
221
|
+
for p1, p2 in zip(parent1, parent2):
|
|
222
|
+
mask = (
|
|
223
|
+
torch.rand(
|
|
224
|
+
p1.shape,
|
|
225
|
+
generator=self._generator,
|
|
226
|
+
device=p1.device,
|
|
227
|
+
dtype=p1.dtype,
|
|
228
|
+
)
|
|
229
|
+
< 0.5
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
child_param = torch.where(mask, p1, p2)
|
|
233
|
+
child.append(child_param)
|
|
234
|
+
|
|
235
|
+
children.append(child)
|
|
236
|
+
|
|
237
|
+
return children
|
|
238
|
+
|
|
239
|
+
@torch.no_grad()
|
|
240
|
+
def _mutate(
|
|
241
|
+
self,
|
|
242
|
+
population: list[list[torch.Tensor]],
|
|
243
|
+
) -> list[list[torch.Tensor]]:
|
|
244
|
+
mutated = []
|
|
245
|
+
|
|
246
|
+
for individual in population:
|
|
247
|
+
new_individual = []
|
|
248
|
+
|
|
249
|
+
for param in individual:
|
|
250
|
+
mutation_mask = (
|
|
251
|
+
torch.rand(
|
|
252
|
+
param.shape,
|
|
253
|
+
generator=self._generator,
|
|
254
|
+
device=param.device,
|
|
255
|
+
dtype=param.dtype,
|
|
256
|
+
)
|
|
257
|
+
< self.mutation_rate
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
noise = torch.randn(
|
|
261
|
+
param.shape,
|
|
262
|
+
generator=self._generator,
|
|
263
|
+
device=param.device,
|
|
264
|
+
dtype=param.dtype,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
new_param = param + mutation_mask * self.step_size * noise
|
|
268
|
+
new_individual.append(new_param)
|
|
269
|
+
|
|
270
|
+
mutated.append(new_individual)
|
|
271
|
+
|
|
272
|
+
return mutated
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def _clone_individual(
|
|
276
|
+
individual: list[torch.Tensor],
|
|
277
|
+
) -> list[torch.Tensor]:
|
|
278
|
+
return [param.detach().clone() for param in individual]
|
|
279
|
+
|
|
280
|
+
@torch.no_grad()
|
|
281
|
+
def restore_best(self) -> None:
|
|
282
|
+
"""Restore the best parameters observed so far."""
|
|
283
|
+
if self._best_params is not None:
|
|
284
|
+
self._restore_params(self._best_params)
|
pyperch/optim/rhc.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Randomized Optimization methods for PyPerch.
|
|
3
|
+
|
|
4
|
+
Based on the original PyPerch optimizers by Jakub Owczarek
|
|
5
|
+
(BSD 3-Clause License).
|
|
6
|
+
|
|
7
|
+
These were also inspired by ABAGAIL’s randomized optimization algorithms - https://github.com/pushkar/ABAGAIL.
|
|
8
|
+
|
|
9
|
+
Substantial refactoring and redesign by John Mansfield (2026).
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from .base import RandomizedOptimizer
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RHC(RandomizedOptimizer):
|
|
23
|
+
"""Randomized Hill Climbing optimizer for arbitrary PyTorch models.
|
|
24
|
+
|
|
25
|
+
The closure should return a scalar loss tensor. Lower loss is assumed to be
|
|
26
|
+
better. Gradients are not required.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
params,
|
|
32
|
+
step_size: float = 0.1,
|
|
33
|
+
restarts: int = 0,
|
|
34
|
+
restart_interval: int | None = None,
|
|
35
|
+
random_state: int | None = None,
|
|
36
|
+
):
|
|
37
|
+
if step_size <= 0:
|
|
38
|
+
raise ValueError("step_size must be positive.")
|
|
39
|
+
if restarts < 0:
|
|
40
|
+
raise ValueError("restarts must be >= 0.")
|
|
41
|
+
if restart_interval is not None and restart_interval <= 0:
|
|
42
|
+
raise ValueError("restart_interval must be positive when provided.")
|
|
43
|
+
|
|
44
|
+
defaults = {"step_size": step_size}
|
|
45
|
+
super().__init__(params, defaults)
|
|
46
|
+
|
|
47
|
+
self.step_size = step_size
|
|
48
|
+
self.restarts = restarts
|
|
49
|
+
self.restart_interval = restart_interval
|
|
50
|
+
self.completed_restarts = 0
|
|
51
|
+
|
|
52
|
+
self._generator = torch.Generator()
|
|
53
|
+
if random_state is not None:
|
|
54
|
+
self._generator.manual_seed(random_state)
|
|
55
|
+
|
|
56
|
+
self._initialized = False
|
|
57
|
+
self._current_loss: float | None = None
|
|
58
|
+
self._best_params: list[torch.Tensor] | None = None
|
|
59
|
+
|
|
60
|
+
def step(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
|
|
61
|
+
"""Evaluate one candidate move and keep it if loss does not increase."""
|
|
62
|
+
if closure is None:
|
|
63
|
+
raise ValueError("RHC requires a closure that returns the loss.")
|
|
64
|
+
|
|
65
|
+
if not self._initialized or self._current_loss is None:
|
|
66
|
+
loss_tensor = self._evaluate(closure)
|
|
67
|
+
loss = float(loss_tensor.detach().item())
|
|
68
|
+
self._initialized = True
|
|
69
|
+
self._current_loss = loss
|
|
70
|
+
self._save_best_if_needed(loss)
|
|
71
|
+
return loss_tensor
|
|
72
|
+
|
|
73
|
+
old_params = self._clone_params()
|
|
74
|
+
old_loss = self._current_loss
|
|
75
|
+
|
|
76
|
+
self._propose_step()
|
|
77
|
+
self.proposed_steps += 1
|
|
78
|
+
|
|
79
|
+
candidate_loss_tensor = self._evaluate(closure)
|
|
80
|
+
candidate_loss = float(candidate_loss_tensor.detach().item())
|
|
81
|
+
|
|
82
|
+
if candidate_loss <= old_loss:
|
|
83
|
+
self._current_loss = candidate_loss
|
|
84
|
+
self.accepted_steps += 1
|
|
85
|
+
self._save_best_if_needed(candidate_loss)
|
|
86
|
+
return candidate_loss_tensor
|
|
87
|
+
|
|
88
|
+
self._restore_params(old_params)
|
|
89
|
+
self.rejected_steps += 1
|
|
90
|
+
self._maybe_restart()
|
|
91
|
+
return torch.tensor(
|
|
92
|
+
old_loss,
|
|
93
|
+
dtype=candidate_loss_tensor.dtype,
|
|
94
|
+
device=candidate_loss_tensor.device,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def _evaluate(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
|
|
98
|
+
"""Run the closure and count one objective evaluation."""
|
|
99
|
+
with torch.enable_grad():
|
|
100
|
+
loss_tensor = closure()
|
|
101
|
+
loss = float(loss_tensor.detach().item())
|
|
102
|
+
self._record_eval(loss)
|
|
103
|
+
return loss_tensor
|
|
104
|
+
|
|
105
|
+
@torch.no_grad()
|
|
106
|
+
def _propose_step(self) -> None:
|
|
107
|
+
"""Add Gaussian noise to each trainable parameter."""
|
|
108
|
+
for group in self.param_groups:
|
|
109
|
+
step_size = group["step_size"]
|
|
110
|
+
for p in group["params"]:
|
|
111
|
+
if not p.requires_grad:
|
|
112
|
+
continue
|
|
113
|
+
noise = torch.randn(
|
|
114
|
+
p.shape,
|
|
115
|
+
generator=self._generator,
|
|
116
|
+
device=p.device,
|
|
117
|
+
dtype=p.dtype,
|
|
118
|
+
)
|
|
119
|
+
p.add_(step_size * noise)
|
|
120
|
+
|
|
121
|
+
@torch.no_grad()
|
|
122
|
+
def _maybe_restart(self) -> None:
|
|
123
|
+
"""Randomly reset parameters when the restart schedule is reached."""
|
|
124
|
+
if self.restart_interval is None:
|
|
125
|
+
return
|
|
126
|
+
if self.completed_restarts >= self.restarts:
|
|
127
|
+
return
|
|
128
|
+
if self.proposed_steps % self.restart_interval != 0:
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
for p in self._parameters():
|
|
132
|
+
noise = torch.randn(
|
|
133
|
+
p.shape,
|
|
134
|
+
generator=self._generator,
|
|
135
|
+
device=p.device,
|
|
136
|
+
dtype=p.dtype,
|
|
137
|
+
)
|
|
138
|
+
p.copy_(noise)
|
|
139
|
+
|
|
140
|
+
self.completed_restarts += 1
|
|
141
|
+
self._current_loss = None
|
|
142
|
+
|
|
143
|
+
@torch.no_grad()
|
|
144
|
+
def _save_best_if_needed(self, loss: float) -> None:
|
|
145
|
+
"""Save the current parameters when they improve the best loss."""
|
|
146
|
+
if self.best_loss is None or loss <= self.best_loss:
|
|
147
|
+
self.best_loss = loss
|
|
148
|
+
self._best_params = self._clone_params()
|
|
149
|
+
|
|
150
|
+
@torch.no_grad()
|
|
151
|
+
def restore_best(self) -> None:
|
|
152
|
+
"""Restore the best parameter values observed so far."""
|
|
153
|
+
if self._best_params is not None:
|
|
154
|
+
self._restore_params(self._best_params)
|
pyperch/optim/sa.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Simulated annealing optimizer.
|
|
2
|
+
|
|
3
|
+
Randomized Optimization methods for PyPerch.
|
|
4
|
+
|
|
5
|
+
Based on the original PyPerch optimizers by Jakub Owczarek
|
|
6
|
+
(BSD 3-Clause License).
|
|
7
|
+
|
|
8
|
+
These were also inspired by ABAGAIL’s randomized optimization algorithms - https://github.com/pushkar/ABAGAIL.
|
|
9
|
+
|
|
10
|
+
Substantial refactoring and redesign by John Mansfield (2026).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import math
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
from .base import RandomizedOptimizer
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SA(RandomizedOptimizer):
|
|
24
|
+
"""Simulated annealing optimizer for arbitrary PyTorch models.
|
|
25
|
+
|
|
26
|
+
This optimizer uses random perturbations instead of gradients.
|
|
27
|
+
|
|
28
|
+
Expected usage:
|
|
29
|
+
|
|
30
|
+
optimizer = SA(model.parameters(), step_size=0.1)
|
|
31
|
+
|
|
32
|
+
def closure():
|
|
33
|
+
output = model(X)
|
|
34
|
+
loss = loss_fn(output, y)
|
|
35
|
+
return loss
|
|
36
|
+
|
|
37
|
+
loss = optimizer.step(closure)
|
|
38
|
+
|
|
39
|
+
Lower loss is assumed to be better.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
params,
|
|
45
|
+
step_size: float = 0.1,
|
|
46
|
+
temperature: float = 1.0,
|
|
47
|
+
min_temperature: float = 0.1,
|
|
48
|
+
cooling: float = 0.95,
|
|
49
|
+
random_state: int | None = None,
|
|
50
|
+
):
|
|
51
|
+
if step_size <= 0:
|
|
52
|
+
raise ValueError("step_size must be positive.")
|
|
53
|
+
if temperature <= 0:
|
|
54
|
+
raise ValueError("temperature must be positive.")
|
|
55
|
+
if min_temperature <= 0:
|
|
56
|
+
raise ValueError("min_temperature must be positive.")
|
|
57
|
+
if cooling <= 0 or cooling > 1:
|
|
58
|
+
raise ValueError("cooling must be in the interval (0, 1].")
|
|
59
|
+
|
|
60
|
+
defaults = {
|
|
61
|
+
"step_size": step_size,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
super().__init__(params, defaults)
|
|
65
|
+
|
|
66
|
+
self.step_size = step_size
|
|
67
|
+
self.temperature = temperature
|
|
68
|
+
self.min_temperature = min_temperature
|
|
69
|
+
self.cooling = cooling
|
|
70
|
+
|
|
71
|
+
self._generator = torch.Generator()
|
|
72
|
+
if random_state is not None:
|
|
73
|
+
self._generator.manual_seed(random_state)
|
|
74
|
+
|
|
75
|
+
self._initialized = False
|
|
76
|
+
self._current_loss: float | None = None
|
|
77
|
+
self._best_params: list[torch.Tensor] | None = None
|
|
78
|
+
|
|
79
|
+
def step(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
|
|
80
|
+
if closure is None:
|
|
81
|
+
raise ValueError("SA requires a closure that returns the loss.")
|
|
82
|
+
|
|
83
|
+
if not self._initialized:
|
|
84
|
+
with torch.enable_grad():
|
|
85
|
+
loss_tensor = closure()
|
|
86
|
+
|
|
87
|
+
loss = float(loss_tensor.detach().item())
|
|
88
|
+
|
|
89
|
+
self.function_evals += 1
|
|
90
|
+
self._initialized = True
|
|
91
|
+
self._current_loss = loss
|
|
92
|
+
self._update_best_loss(loss)
|
|
93
|
+
self._best_params = self._clone_params()
|
|
94
|
+
|
|
95
|
+
return loss_tensor
|
|
96
|
+
|
|
97
|
+
trainable = self._parameters()
|
|
98
|
+
|
|
99
|
+
if not trainable:
|
|
100
|
+
with torch.enable_grad():
|
|
101
|
+
loss_tensor = closure()
|
|
102
|
+
|
|
103
|
+
loss = float(loss_tensor.detach().item())
|
|
104
|
+
self.function_evals += 1
|
|
105
|
+
self._current_loss = loss
|
|
106
|
+
self._update_best_loss(loss)
|
|
107
|
+
|
|
108
|
+
return loss_tensor
|
|
109
|
+
|
|
110
|
+
param_index = torch.randint(
|
|
111
|
+
low=0,
|
|
112
|
+
high=len(trainable),
|
|
113
|
+
size=(1,),
|
|
114
|
+
generator=self._generator,
|
|
115
|
+
).item()
|
|
116
|
+
|
|
117
|
+
param = trainable[param_index]
|
|
118
|
+
|
|
119
|
+
old_param = param.detach().clone()
|
|
120
|
+
|
|
121
|
+
with torch.no_grad():
|
|
122
|
+
noise = (
|
|
123
|
+
torch.rand(
|
|
124
|
+
param.shape,
|
|
125
|
+
generator=self._generator,
|
|
126
|
+
device=param.device,
|
|
127
|
+
dtype=param.dtype,
|
|
128
|
+
)
|
|
129
|
+
- 0.5
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
param.add_(self.step_size * noise)
|
|
133
|
+
|
|
134
|
+
self.proposed_steps += 1
|
|
135
|
+
|
|
136
|
+
with torch.enable_grad():
|
|
137
|
+
candidate_loss_tensor = closure()
|
|
138
|
+
|
|
139
|
+
candidate_loss = float(candidate_loss_tensor.detach().item())
|
|
140
|
+
self.function_evals += 1
|
|
141
|
+
|
|
142
|
+
delta = candidate_loss - self._current_loss
|
|
143
|
+
|
|
144
|
+
accept = False
|
|
145
|
+
|
|
146
|
+
if delta <= 0:
|
|
147
|
+
accept = True
|
|
148
|
+
else:
|
|
149
|
+
acceptance_probability = math.exp(-delta / max(self.temperature, 1e-12))
|
|
150
|
+
|
|
151
|
+
random_value = torch.rand(
|
|
152
|
+
size=(1,),
|
|
153
|
+
generator=self._generator,
|
|
154
|
+
).item()
|
|
155
|
+
|
|
156
|
+
accept = random_value < acceptance_probability
|
|
157
|
+
|
|
158
|
+
if accept:
|
|
159
|
+
self.accepted_steps += 1
|
|
160
|
+
self._current_loss = candidate_loss
|
|
161
|
+
|
|
162
|
+
if self.best_loss is None or candidate_loss < self.best_loss:
|
|
163
|
+
self.best_loss = candidate_loss
|
|
164
|
+
self._best_params = self._clone_params()
|
|
165
|
+
|
|
166
|
+
result = candidate_loss_tensor
|
|
167
|
+
|
|
168
|
+
else:
|
|
169
|
+
with torch.no_grad():
|
|
170
|
+
param.copy_(old_param)
|
|
171
|
+
|
|
172
|
+
self.rejected_steps += 1
|
|
173
|
+
|
|
174
|
+
result = torch.tensor(
|
|
175
|
+
self._current_loss,
|
|
176
|
+
dtype=candidate_loss_tensor.dtype,
|
|
177
|
+
device=candidate_loss_tensor.device,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
self.temperature = max(
|
|
181
|
+
self.temperature * self.cooling,
|
|
182
|
+
self.min_temperature,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
return result
|
|
186
|
+
|
|
187
|
+
@torch.no_grad()
|
|
188
|
+
def restore_best(self) -> None:
|
|
189
|
+
"""Restore the best parameters observed so far."""
|
|
190
|
+
if self._best_params is not None:
|
|
191
|
+
self._restore_params(self._best_params)
|
pyperch/search/grid.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from itertools import product
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def grid_search(
|
|
8
|
+
param_grid: dict, objective: Callable[[dict], float]
|
|
9
|
+
) -> tuple[dict, float]:
|
|
10
|
+
"""Evaluate every parameter combination and return the best result."""
|
|
11
|
+
keys = list(param_grid.keys())
|
|
12
|
+
best_params = None
|
|
13
|
+
best_score = None
|
|
14
|
+
|
|
15
|
+
for values in product(*(param_grid[key] for key in keys)):
|
|
16
|
+
params = dict(zip(keys, values))
|
|
17
|
+
score = objective(params)
|
|
18
|
+
if best_score is None or score < best_score:
|
|
19
|
+
best_params = params
|
|
20
|
+
best_score = score
|
|
21
|
+
|
|
22
|
+
return best_params, best_score
|
pyperch/search/optuna.py
ADDED
pyperch/search/random.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def random_search(
|
|
8
|
+
param_space: dict,
|
|
9
|
+
objective: Callable[[dict], float],
|
|
10
|
+
n_iter: int = 10,
|
|
11
|
+
seed: int | None = None,
|
|
12
|
+
) -> tuple[dict, float]:
|
|
13
|
+
"""Sample parameter combinations and return the best result."""
|
|
14
|
+
rng = random.Random(seed)
|
|
15
|
+
best_params = None
|
|
16
|
+
best_score = None
|
|
17
|
+
|
|
18
|
+
for _ in range(n_iter):
|
|
19
|
+
params = {key: rng.choice(values) for key, values in param_space.items()}
|
|
20
|
+
score = objective(params)
|
|
21
|
+
if best_score is None or score < best_score:
|
|
22
|
+
best_params = params
|
|
23
|
+
best_score = score
|
|
24
|
+
|
|
25
|
+
return best_params, best_score
|
pyperch/utils/random.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class OptimizerSnapshot:
|
|
6
|
+
"""Serializable snapshot of optimizer counters."""
|
|
7
|
+
|
|
8
|
+
function_evals: int
|
|
9
|
+
proposed_steps: int
|
|
10
|
+
accepted_steps: int
|
|
11
|
+
rejected_steps: int
|
|
12
|
+
best_loss: float | None
|
|
13
|
+
|
|
14
|
+
@classmethod
|
|
15
|
+
def from_optimizer(cls, optimizer):
|
|
16
|
+
"""Create a snapshot from an optimizer with standard counters."""
|
|
17
|
+
return cls(
|
|
18
|
+
function_evals=optimizer.function_evals,
|
|
19
|
+
proposed_steps=optimizer.proposed_steps,
|
|
20
|
+
accepted_steps=optimizer.accepted_steps,
|
|
21
|
+
rejected_steps=optimizer.rejected_steps,
|
|
22
|
+
best_loss=optimizer.best_loss,
|
|
23
|
+
)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyperch
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: PyTorch-native randomized optimization algorithms.
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: pytorch,optimization,machine-learning,randomized-optimization
|
|
7
|
+
Author: John Mansfield
|
|
8
|
+
Requires-Python: >=3.10,<=3.13
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Dist: numpy (>=1.24)
|
|
16
|
+
Requires-Dist: torch (>=2.1,<3.0)
|
|
17
|
+
Project-URL: Homepage, https://github.com/jlm429/pyperch
|
|
18
|
+
Project-URL: Repository, https://github.com/jlm429/pyperch
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# pyperch
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+

|
|
25
|
+

|
|
26
|
+

|
|
27
|
+

|
|
28
|
+
[](https://dl.circleci.com/status-badge/redirect/circleci/WH9eaoZnQRJ8SGFDrvqQAd/5meq6x5R3uDA3KSuHARdVk/tree/master)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
A lightweight, modular library for neural network weight optimization using randomized search algorithms built directly on top of PyTorch. Pyperch is a research and teaching-oriented library for training neural networks using randomized optimization methods (RHC, SA, GA), gradient-based methods, and hybrid combinations.
|
|
32
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
pyperch/__init__.py,sha256=sDsvlQ-HyYg3YcvdS0MQOpga_KFKXHq6hwOnshOIMfc,139
|
|
2
|
+
pyperch/optim/__init__.py,sha256=aJVVr9_B3ih9S45lXy2F2rWl--EMyT9aDtSqVfzXzbo,170
|
|
3
|
+
pyperch/optim/base.py,sha256=y7mAHbkW457FjbqNo9np0Eey2UQZGhJR8tJ7cKBp_HU,1816
|
|
4
|
+
pyperch/optim/ga.py,sha256=m-NVQNYFuXIZOqpfxtROO58Gh39DXEY66wkmymprIQg,8393
|
|
5
|
+
pyperch/optim/rhc.py,sha256=FbSOKaIGcWizA_bOZIpVLEwyYiNY4rNSP176C4k9qv4,5114
|
|
6
|
+
pyperch/optim/sa.py,sha256=EAxcK6sh9AjvRxN5oKlCP5uMn6IfsdQlEWgR29Tmasw,5270
|
|
7
|
+
pyperch/search/__init__.py,sha256=hj5o_EKEr3_FqEhvLPenkqtFR0QCO2bzpGth3wqbtWo,108
|
|
8
|
+
pyperch/search/grid.py,sha256=hGOFzkwb7HEZ6jEIbnivizfw7VRRwEWPidM0I_I9V1g,651
|
|
9
|
+
pyperch/search/optuna.py,sha256=-Lh3XZD4jwpkLjXon_wZ_KfdFfL-qaiVRw8_5nUVKes,261
|
|
10
|
+
pyperch/search/random.py,sha256=XF_QBRnUmbbwovkdC3ECBVPhJuF5D-ZTMPm1pcYdMYQ,688
|
|
11
|
+
pyperch/utils/__init__.py,sha256=Obvq21lNQ2Pl2NmYRVaVvFU4qRAUFqbiM0BhEhDJOvg,114
|
|
12
|
+
pyperch/utils/random.py,sha256=3JI2xyIgfOdyUnxwpWwa76J2vtMxX7pDuVaMSTBrBpg,160
|
|
13
|
+
pyperch/utils/tracking.py,sha256=GzGPhsYbM_k0F-V9p4mM-lvMk3uIwtRuR2p9jT1gmys,669
|
|
14
|
+
pyperch-0.3.0.dist-info/METADATA,sha256=HzvRWD3009CAfdPa-_zts_rnbFUXef2l_kZcCxOPxl0,1698
|
|
15
|
+
pyperch-0.3.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
16
|
+
pyperch-0.3.0.dist-info/RECORD,,
|