x-evolution 0.1.32__tar.gz → 0.1.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {x_evolution-0.1.32 → x_evolution-0.1.34}/.gitignore +3 -0
- {x_evolution-0.1.32 → x_evolution-0.1.34}/PKG-INFO +24 -2
- {x_evolution-0.1.32 → x_evolution-0.1.34}/README.md +22 -0
- {x_evolution-0.1.32 → x_evolution-0.1.34}/pyproject.toml +1 -1
- x_evolution-0.1.34/x_evolution/update_rules.py +554 -0
- {x_evolution-0.1.32 → x_evolution-0.1.34}/x_evolution/x_evolution.py +82 -30
- {x_evolution-0.1.32 → x_evolution-0.1.34}/LICENSE +0 -0
- {x_evolution-0.1.32 → x_evolution-0.1.34}/x_evolution/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: x-evolution
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.34
|
|
4
4
|
Summary: x-evolution
|
|
5
5
|
Project-URL: Homepage, https://pypi.org/project/x-evolution/
|
|
6
6
|
Project-URL: Repository, https://github.com/lucidrains/x-evolution
|
|
@@ -122,6 +122,28 @@ For a quick demo on LunarLander with 8 processes
|
|
|
122
122
|
$ uv run torchrun --nproc_per_node=8 train_lunar.py --cpu True --noise_population_size 100 --fitness_to_weighted_factor centered_rank
|
|
123
123
|
```
|
|
124
124
|
|
|
125
|
+
## Sim-to-Real Bipedal Locomotion (Microduck)
|
|
126
|
+
|
|
127
|
+
You can train a walking gait for the [Microduck](https://github.com/pollen-robotics/reachy2_microduck) biped robot from scratch using `EvoStrategy` and `torchrun`.
|
|
128
|
+
|
|
129
|
+
With a population size of 128 distributed across 8 CPU processes, a forward walking gait emerges around **150 generations** (~3-4 minutes):
|
|
130
|
+
|
|
131
|
+
```shell
|
|
132
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
To render evaluation videos or resume from the saved best checkpoint:
|
|
136
|
+
|
|
137
|
+
```shell
|
|
138
|
+
# render periodic evaluation videos to videos/
|
|
139
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py --eval_video True --eval_every 50
|
|
140
|
+
|
|
141
|
+
# resume training from best checkpoint
|
|
142
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py --resume True
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Special thanks to [Pollen Robotics 🤗](https://www.pollen-robotics.com/) for their open sourcing of such a polished sim2real env!
|
|
146
|
+
|
|
125
147
|
## Citations
|
|
126
148
|
|
|
127
149
|
```bibtex
|
|
@@ -71,6 +71,28 @@ For a quick demo on LunarLander with 8 processes
|
|
|
71
71
|
$ uv run torchrun --nproc_per_node=8 train_lunar.py --cpu True --noise_population_size 100 --fitness_to_weighted_factor centered_rank
|
|
72
72
|
```
|
|
73
73
|
|
|
74
|
+
## Sim-to-Real Bipedal Locomotion (Microduck)
|
|
75
|
+
|
|
76
|
+
You can train a walking gait for the [Microduck](https://github.com/pollen-robotics/reachy2_microduck) biped robot from scratch using `EvoStrategy` and `torchrun`.
|
|
77
|
+
|
|
78
|
+
With a population size of 128 distributed across 8 CPU processes, a forward walking gait emerges around **150 generations** (~3-4 minutes):
|
|
79
|
+
|
|
80
|
+
```shell
|
|
81
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
To render evaluation videos or resume from the saved best checkpoint:
|
|
85
|
+
|
|
86
|
+
```shell
|
|
87
|
+
# render periodic evaluation videos to videos/
|
|
88
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py --eval_video True --eval_every 50
|
|
89
|
+
|
|
90
|
+
# resume training from best checkpoint
|
|
91
|
+
$ uv run torchrun --nproc_per_node=8 train_microduck.py --resume True
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Special thanks to [Pollen Robotics 🤗](https://www.pollen-robotics.com/) for their open sourcing of such a polished sim2real env!
|
|
95
|
+
|
|
74
96
|
## Citations
|
|
75
97
|
|
|
76
98
|
```bibtex
|
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from math import exp, log, sqrt
|
|
4
|
+
from functools import partial
|
|
5
|
+
from typing import Callable, Literal
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import tensor, Tensor
|
|
9
|
+
from torch.nn import Module, Parameter, ParameterList
|
|
10
|
+
from torch.optim import SGD, Optimizer
|
|
11
|
+
from torch.optim.lr_scheduler import LRScheduler
|
|
12
|
+
|
|
13
|
+
import torch.nn.functional as F
|
|
14
|
+
|
|
15
|
+
from beartype import beartype
|
|
16
|
+
|
|
17
|
+
from einops import pack
|
|
18
|
+
from torch_einops_utils import pack_with_inverse
|
|
19
|
+
|
|
20
|
+
from x_mlps_pytorch.noisable import Noisable, with_seed
|
|
21
|
+
|
|
22
|
+
# constants
|
|
23
|
+
|
|
24
|
+
MAX_SEED_VALUE = int(2 ** 32)
|
|
25
|
+
|
|
26
|
+
# helper functions
|
|
27
|
+
|
|
28
|
+
def exists(v):
|
|
29
|
+
return v is not None
|
|
30
|
+
|
|
31
|
+
def default(v, d):
|
|
32
|
+
return v if exists(v) else d
|
|
33
|
+
|
|
34
|
+
def accum_grad_(t, value):
|
|
35
|
+
if not exists(t.grad):
|
|
36
|
+
t.grad = value.clone()
|
|
37
|
+
else:
|
|
38
|
+
t.grad.add_(value)
|
|
39
|
+
|
|
40
|
+
# fitness weighting factors
|
|
41
|
+
|
|
42
|
+
def normalize(t, eps = 1e-6):
|
|
43
|
+
return F.layer_norm(t, t.shape, eps = eps)
|
|
44
|
+
|
|
45
|
+
def centered_rank_normalize(fitnesses):
|
|
46
|
+
dtype, device, n = fitnesses.dtype, fitnesses.device, fitnesses.numel()
|
|
47
|
+
flat_fitnesses, inverse = pack_with_inverse(fitnesses, '*')
|
|
48
|
+
ranks = flat_fitnesses.argsort().argsort().to(dtype = dtype)
|
|
49
|
+
ranks = (ranks / (n - 1)) - 0.5
|
|
50
|
+
return inverse(ranks)
|
|
51
|
+
|
|
52
|
+
FITNESS_WEIGHT_FACTORS = dict(
|
|
53
|
+
normalize = normalize,
|
|
54
|
+
centered_rank = centered_rank_normalize
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
FitnessWeightFactorType = Literal['normalize', 'centered_rank']
|
|
58
|
+
|
|
59
|
+
# base update rule
|
|
60
|
+
# turns the population of (fitness, seed) pairs into a param update, given the noise
|
|
61
|
+
# can be reproduced deterministically from the seeds via the `Noisable` wrapper
|
|
62
|
+
|
|
63
|
+
class UpdateRule(Module):
|
|
64
|
+
|
|
65
|
+
@beartype
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
*,
|
|
69
|
+
model: Module,
|
|
70
|
+
noisable_model: Noisable,
|
|
71
|
+
named_parameters: dict[str, Parameter],
|
|
72
|
+
param_names: list[str],
|
|
73
|
+
param_shapes: dict[str, torch.Size],
|
|
74
|
+
param_dtypes: dict[str, torch.dtype],
|
|
75
|
+
noise_population_size: int,
|
|
76
|
+
mirror_sampling: bool,
|
|
77
|
+
noise_scale: float,
|
|
78
|
+
device
|
|
79
|
+
):
|
|
80
|
+
super().__init__()
|
|
81
|
+
|
|
82
|
+
self.model = model
|
|
83
|
+
self.noisable_model = noisable_model
|
|
84
|
+
|
|
85
|
+
self.named_parameters = named_parameters
|
|
86
|
+
self.param_names = list(param_names)
|
|
87
|
+
self.param_shapes = param_shapes
|
|
88
|
+
self.param_dtypes = param_dtypes
|
|
89
|
+
|
|
90
|
+
self.noise_population_size = noise_population_size
|
|
91
|
+
self.mirror_sampling = mirror_sampling
|
|
92
|
+
self.noise_scale = noise_scale
|
|
93
|
+
|
|
94
|
+
self.device = device
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def num_params(self):
|
|
98
|
+
return len(self.param_names)
|
|
99
|
+
|
|
100
|
+
def noise_scale_for(self, param_name):
|
|
101
|
+
# scale used to perturb a given param during rollouts
|
|
102
|
+
|
|
103
|
+
raise NotImplementedError
|
|
104
|
+
|
|
105
|
+
@torch.inference_mode()
|
|
106
|
+
def evolve(
|
|
107
|
+
self,
|
|
108
|
+
fitnesses: list[float] | Tensor,
|
|
109
|
+
seeds_for_population: list[int] | Tensor,
|
|
110
|
+
scoped_names: set[str] | None = None
|
|
111
|
+
):
|
|
112
|
+
# update model params given fitnesses and the seeds reproducing the perturbation noise
|
|
113
|
+
|
|
114
|
+
raise NotImplementedError
|
|
115
|
+
|
|
116
|
+
def to_device(self, device):
|
|
117
|
+
self.to(device)
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def extra_log(self):
|
|
121
|
+
# extra state to log at end of each generation
|
|
122
|
+
|
|
123
|
+
return ''
|
|
124
|
+
|
|
125
|
+
# plain es - the weighted average of the noise, scaled by normalized fitnesses, is the gradient estimate
|
|
126
|
+
# optionally with learned per-parameter sigmas
|
|
127
|
+
|
|
128
|
+
class PlainES(UpdateRule):
|
|
129
|
+
|
|
130
|
+
@beartype
|
|
131
|
+
def __init__(
|
|
132
|
+
self,
|
|
133
|
+
*,
|
|
134
|
+
model: Module,
|
|
135
|
+
noisable_model: Noisable,
|
|
136
|
+
named_parameters: dict[str, Parameter],
|
|
137
|
+
param_names: list[str],
|
|
138
|
+
param_shapes: dict[str, torch.Size],
|
|
139
|
+
param_dtypes: dict[str, torch.dtype],
|
|
140
|
+
noise_population_size: int,
|
|
141
|
+
mirror_sampling: bool,
|
|
142
|
+
noise_scale = 1e-2, # noise scaling during rollouts, can be customized per parameter name
|
|
143
|
+
device,
|
|
144
|
+
learning_rate = 1e-3,
|
|
145
|
+
learned_noise_scale = False,
|
|
146
|
+
noise_scale_learning_rate = 1e-5,
|
|
147
|
+
noise_scale_clamp_range: tuple[float, float] = (1e-3, 1e-1),
|
|
148
|
+
use_optimizer = True,
|
|
149
|
+
optimizer_klass: type[Optimizer] | Callable = partial(SGD, nesterov = True, momentum = 0.1, weight_decay = 1e-2),
|
|
150
|
+
optimizer_kwargs: dict = dict(),
|
|
151
|
+
use_sigma_optimizer = True,
|
|
152
|
+
sigma_optimizer_klass: type[Optimizer] | Callable = partial(SGD, nesterov = True, momentum = 0.1),
|
|
153
|
+
sigma_optimizer_kwargs: dict = dict(),
|
|
154
|
+
use_scheduler = False,
|
|
155
|
+
scheduler_klass: type[LRScheduler] | None = None,
|
|
156
|
+
scheduler_kwargs: dict = dict(),
|
|
157
|
+
use_sigma_scheduler = False,
|
|
158
|
+
sigma_scheduler_klass: type[LRScheduler] | None = None,
|
|
159
|
+
sigma_scheduler_kwargs: dict = dict(),
|
|
160
|
+
fitness_to_weighted_factor: Callable[[Tensor], Tensor] | FitnessWeightFactorType = 'normalize'
|
|
161
|
+
):
|
|
162
|
+
super().__init__(
|
|
163
|
+
model = model,
|
|
164
|
+
noisable_model = noisable_model,
|
|
165
|
+
named_parameters = named_parameters,
|
|
166
|
+
param_names = param_names,
|
|
167
|
+
param_shapes = param_shapes,
|
|
168
|
+
param_dtypes = param_dtypes,
|
|
169
|
+
noise_population_size = noise_population_size,
|
|
170
|
+
mirror_sampling = mirror_sampling,
|
|
171
|
+
noise_scale = noise_scale,
|
|
172
|
+
device = device
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
self.learning_rate = learning_rate
|
|
176
|
+
|
|
177
|
+
# transforms fitnesses to weights for the weighted average of the noise
|
|
178
|
+
|
|
179
|
+
if isinstance(fitness_to_weighted_factor, str):
|
|
180
|
+
if fitness_to_weighted_factor not in FITNESS_WEIGHT_FACTORS:
|
|
181
|
+
raise ValueError(f'fitness_to_weighted_factor must be one of {list(FITNESS_WEIGHT_FACTORS.keys())}')
|
|
182
|
+
fitness_to_weighted_factor = FITNESS_WEIGHT_FACTORS[fitness_to_weighted_factor]
|
|
183
|
+
|
|
184
|
+
self.fitness_to_weighted_factor = fitness_to_weighted_factor
|
|
185
|
+
|
|
186
|
+
# noise scale, fixed or learned
|
|
187
|
+
|
|
188
|
+
self.learned_noise_scale = learned_noise_scale
|
|
189
|
+
|
|
190
|
+
if learned_noise_scale:
|
|
191
|
+
self.sigmas = ParameterList([Parameter(torch.ones_like(param) * noise_scale) for _, param in named_parameters.items()])
|
|
192
|
+
self.param_name_to_sigma_index = {name: i for i, name in enumerate(named_parameters.keys())}
|
|
193
|
+
|
|
194
|
+
min_noise_scale, max_noise_scale = noise_scale_clamp_range
|
|
195
|
+
assert min_noise_scale > 0. and min_noise_scale < max_noise_scale
|
|
196
|
+
|
|
197
|
+
self.sigma_clamp_ = lambda t: t.clamp_(*noise_scale_clamp_range)
|
|
198
|
+
|
|
199
|
+
self.noise_scale_learning_rate = noise_scale_learning_rate
|
|
200
|
+
|
|
201
|
+
# maybe use optimizer, allow for Adam
|
|
202
|
+
|
|
203
|
+
self.use_optimizer = use_optimizer
|
|
204
|
+
|
|
205
|
+
if use_optimizer:
|
|
206
|
+
optim_params = [named_parameters[name] for name in param_names]
|
|
207
|
+
self.optimizer = optimizer_klass(optim_params, lr = learning_rate, **optimizer_kwargs)
|
|
208
|
+
|
|
209
|
+
self.use_sigma_optimizer = use_sigma_optimizer
|
|
210
|
+
|
|
211
|
+
if learned_noise_scale and use_sigma_optimizer:
|
|
212
|
+
self.sigma_optimizer = sigma_optimizer_klass(self.sigmas, lr = noise_scale_learning_rate, **sigma_optimizer_kwargs)
|
|
213
|
+
|
|
214
|
+
self.use_scheduler = use_scheduler
|
|
215
|
+
|
|
216
|
+
if use_scheduler and exists(scheduler_klass) and use_optimizer:
|
|
217
|
+
self.scheduler = scheduler_klass(self.optimizer, **scheduler_kwargs)
|
|
218
|
+
|
|
219
|
+
self.use_sigma_scheduler = use_sigma_scheduler
|
|
220
|
+
|
|
221
|
+
if use_sigma_scheduler and exists(sigma_scheduler_klass) and use_sigma_optimizer:
|
|
222
|
+
self.sigma_scheduler = sigma_scheduler_klass(self.sigma_optimizer, **sigma_scheduler_kwargs)
|
|
223
|
+
|
|
224
|
+
def noise_scale_for(self, param_name):
|
|
225
|
+
if not self.learned_noise_scale:
|
|
226
|
+
return self.noise_scale
|
|
227
|
+
|
|
228
|
+
sigma_index = self.param_name_to_sigma_index[param_name]
|
|
229
|
+
return self.sigmas[sigma_index]
|
|
230
|
+
|
|
231
|
+
@torch.inference_mode()
|
|
232
|
+
def evolve(
|
|
233
|
+
self,
|
|
234
|
+
fitnesses: list[float] | Tensor,
|
|
235
|
+
seeds_for_population: list[int] | Tensor,
|
|
236
|
+
scoped_names: set[str] | None = None
|
|
237
|
+
):
|
|
238
|
+
use_optimizer = self.use_optimizer
|
|
239
|
+
model = self.noisable_model
|
|
240
|
+
|
|
241
|
+
if isinstance(fitnesses, list):
|
|
242
|
+
fitnesses = tensor(fitnesses, dtype = torch.float32)
|
|
243
|
+
|
|
244
|
+
if isinstance(seeds_for_population, list):
|
|
245
|
+
seeds_for_population = tensor(seeds_for_population)
|
|
246
|
+
|
|
247
|
+
fitnesses = fitnesses.to(self.device)
|
|
248
|
+
seeds_for_population = seeds_for_population.to(self.device)
|
|
249
|
+
|
|
250
|
+
# maybe normalize the fitness with z-score
|
|
251
|
+
|
|
252
|
+
fitnesses = self.fitness_to_weighted_factor(fitnesses)
|
|
253
|
+
|
|
254
|
+
if self.mirror_sampling:
|
|
255
|
+
fitness_pos, fitness_neg = fitnesses.unbind(dim = -1)
|
|
256
|
+
weights = fitness_pos - fitness_neg
|
|
257
|
+
else:
|
|
258
|
+
weights = fitnesses
|
|
259
|
+
|
|
260
|
+
weights /= self.noise_population_size * (2. if self.mirror_sampling else 1.)
|
|
261
|
+
|
|
262
|
+
# update one seed at a time, for large models
|
|
263
|
+
|
|
264
|
+
iterator = zip(seeds_for_population.tolist(), weights.tolist())
|
|
265
|
+
|
|
266
|
+
for individual_seed, weight in iterator:
|
|
267
|
+
|
|
268
|
+
individual_param_seeds = with_seed(individual_seed)(torch.randint)(0, MAX_SEED_VALUE, (self.num_params,))
|
|
269
|
+
|
|
270
|
+
# setup noise config
|
|
271
|
+
|
|
272
|
+
noise_config = dict(zip(self.param_names, individual_param_seeds.tolist()))
|
|
273
|
+
|
|
274
|
+
if exists(scoped_names):
|
|
275
|
+
noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
|
|
276
|
+
|
|
277
|
+
noise_config_with_weight = dict()
|
|
278
|
+
|
|
279
|
+
# loop through each noise
|
|
280
|
+
|
|
281
|
+
for param_name, seed in noise_config.items():
|
|
282
|
+
|
|
283
|
+
shape = self.param_shapes[param_name]
|
|
284
|
+
dtype = self.param_dtypes[param_name]
|
|
285
|
+
|
|
286
|
+
# reproduce the noise from the seed
|
|
287
|
+
|
|
288
|
+
noise = with_seed(seed)(model.create_noise_fn)(shape, dtype = dtype)
|
|
289
|
+
noise = noise.to(self.device)
|
|
290
|
+
|
|
291
|
+
# scale the weight
|
|
292
|
+
|
|
293
|
+
noise_scale = self.noise_scale_for(param_name)
|
|
294
|
+
|
|
295
|
+
scaled_weight = weight / noise_scale
|
|
296
|
+
|
|
297
|
+
if not use_optimizer:
|
|
298
|
+
scaled_weight = scaled_weight * self.learning_rate
|
|
299
|
+
|
|
300
|
+
# set the weighted noise
|
|
301
|
+
|
|
302
|
+
noise_config_with_weight[param_name] = (noise, scaled_weight)
|
|
303
|
+
|
|
304
|
+
# maybe learned sigma
|
|
305
|
+
|
|
306
|
+
if self.learned_noise_scale:
|
|
307
|
+
one_grad_sigma = weight * (noise ** 2 - 1) / noise_scale
|
|
308
|
+
|
|
309
|
+
sigma_index = self.param_name_to_sigma_index[param_name]
|
|
310
|
+
sigma = self.sigmas[sigma_index]
|
|
311
|
+
|
|
312
|
+
if self.use_sigma_optimizer:
|
|
313
|
+
accum_grad_(sigma, -one_grad_sigma)
|
|
314
|
+
else:
|
|
315
|
+
sigma.add_(one_grad_sigma * self.noise_scale_learning_rate)
|
|
316
|
+
|
|
317
|
+
# now update params for one seed
|
|
318
|
+
|
|
319
|
+
model.add_noise_(noise_config_with_weight, negate = use_optimizer, add_to_grad = use_optimizer)
|
|
320
|
+
|
|
321
|
+
if use_optimizer:
|
|
322
|
+
self.optimizer.step()
|
|
323
|
+
self.optimizer.zero_grad()
|
|
324
|
+
|
|
325
|
+
if self.use_scheduler and exists(self.scheduler):
|
|
326
|
+
self.scheduler.step()
|
|
327
|
+
|
|
328
|
+
if self.learned_noise_scale:
|
|
329
|
+
if self.use_sigma_optimizer:
|
|
330
|
+
self.sigma_optimizer.step()
|
|
331
|
+
self.sigma_optimizer.zero_grad()
|
|
332
|
+
|
|
333
|
+
if self.use_sigma_scheduler and exists(self.sigma_scheduler):
|
|
334
|
+
self.sigma_scheduler.step()
|
|
335
|
+
|
|
336
|
+
for sigma in self.sigmas:
|
|
337
|
+
self.sigma_clamp_(sigma)
|
|
338
|
+
|
|
339
|
+
def extra_log(self):
|
|
340
|
+
if not self.learned_noise_scale:
|
|
341
|
+
return ''
|
|
342
|
+
|
|
343
|
+
packed_sigma, _ = pack(list(self.sigmas), '*')
|
|
344
|
+
return f'average sigma: {packed_sigma.mean().item():.3f}'
|
|
345
|
+
|
|
346
|
+
# sep cma-es
|
|
347
|
+
|
|
348
|
+
class SepCMAES(UpdateRule):
|
|
349
|
+
|
|
350
|
+
@beartype
|
|
351
|
+
def __init__(
|
|
352
|
+
self,
|
|
353
|
+
*,
|
|
354
|
+
model: Module,
|
|
355
|
+
noisable_model: Noisable,
|
|
356
|
+
named_parameters: dict[str, Parameter],
|
|
357
|
+
param_names: list[str],
|
|
358
|
+
param_shapes: dict[str, torch.Size],
|
|
359
|
+
param_dtypes: dict[str, torch.dtype],
|
|
360
|
+
noise_population_size: int,
|
|
361
|
+
mirror_sampling: bool,
|
|
362
|
+
noise_scale = 1e-2, # initial global step size sigma
|
|
363
|
+
device,
|
|
364
|
+
cma_mu: int | None = None,
|
|
365
|
+
**plain_es_compat_kwargs
|
|
366
|
+
):
|
|
367
|
+
# validate that only known plain es hyperparameters are being ignored
|
|
368
|
+
|
|
369
|
+
unsupported_kwargs = set(plain_es_compat_kwargs) - set(PLAIN_ES_DEFAULT_KWARGS)
|
|
370
|
+
|
|
371
|
+
if unsupported_kwargs:
|
|
372
|
+
raise TypeError(f'SepCMAES.__init__() got unexpected keyword arguments: {unsupported_kwargs}')
|
|
373
|
+
|
|
374
|
+
super().__init__(
|
|
375
|
+
model = model,
|
|
376
|
+
noisable_model = noisable_model,
|
|
377
|
+
named_parameters = named_parameters,
|
|
378
|
+
param_names = param_names,
|
|
379
|
+
param_shapes = param_shapes,
|
|
380
|
+
param_dtypes = param_dtypes,
|
|
381
|
+
noise_population_size = noise_population_size,
|
|
382
|
+
mirror_sampling = mirror_sampling,
|
|
383
|
+
noise_scale = noise_scale,
|
|
384
|
+
device = device
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
pop_size = self.noise_population_size * (2 if self.mirror_sampling else 1)
|
|
388
|
+
mu = default(cma_mu, pop_size // 2)
|
|
389
|
+
self.cma_mu = mu
|
|
390
|
+
|
|
391
|
+
dim = sum(self.param_shapes[name].numel() for name in self.param_names)
|
|
392
|
+
|
|
393
|
+
# cma strategy constants
|
|
394
|
+
|
|
395
|
+
weights = log(mu + 0.5) - torch.log(torch.arange(1, mu + 1, dtype = torch.float32))
|
|
396
|
+
weights /= weights.sum()
|
|
397
|
+
mueff = 1 / (weights ** 2).sum().item()
|
|
398
|
+
|
|
399
|
+
self.register_buffer('cma_weights', weights)
|
|
400
|
+
|
|
401
|
+
cc = (4 + mueff / dim) / (dim + 4 + 2 * mueff / dim)
|
|
402
|
+
c_sigma = (mueff + 2) / (dim + mueff + 5)
|
|
403
|
+
c1 = 2 / ((dim + 1.3) ** 2 + mueff)
|
|
404
|
+
cmu_coeff = min(1 - c1, 2 * (mueff - 2 + 1 / mueff) / ((dim + 2) ** 2 + mueff))
|
|
405
|
+
|
|
406
|
+
self.cma_c1_sep = min(1, c1 * (dim + 1.5) / 3)
|
|
407
|
+
self.cma_cmu_sep = min(1 - self.cma_c1_sep, cmu_coeff * (dim + 1.5) / 3)
|
|
408
|
+
self.cma_cc = cc
|
|
409
|
+
self.cma_c_sigma = c_sigma
|
|
410
|
+
self.cma_damps = 1 + 2 * max(0, sqrt((mueff - 1) / (dim + 1)) - 1) + c_sigma
|
|
411
|
+
self.cma_chiN = sqrt(dim) * (1 - 1 / (4 * dim) + 1 / (21 * dim ** 2))
|
|
412
|
+
self.cma_mueff = mueff
|
|
413
|
+
|
|
414
|
+
self.cma_sigma = noise_scale
|
|
415
|
+
|
|
416
|
+
self.param_name_to_cma_index = {name: i for i, name in enumerate(self.param_names)}
|
|
417
|
+
|
|
418
|
+
self.cma_pc = ParameterList([Parameter(torch.zeros_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
|
|
419
|
+
self.cma_ps = ParameterList([Parameter(torch.zeros_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
|
|
420
|
+
self.cma_C = ParameterList([Parameter(torch.ones_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
|
|
421
|
+
|
|
422
|
+
def noise_scale_for(self, param_name):
|
|
423
|
+
idx = self.param_name_to_cma_index[param_name]
|
|
424
|
+
return self.cma_sigma * torch.sqrt(self.cma_C[idx])
|
|
425
|
+
|
|
426
|
+
@torch.inference_mode()
|
|
427
|
+
def evolve(
|
|
428
|
+
self,
|
|
429
|
+
fitnesses: list[float] | Tensor,
|
|
430
|
+
seeds_for_population: list[int] | Tensor,
|
|
431
|
+
scoped_names: set[str] | None = None
|
|
432
|
+
):
|
|
433
|
+
model = self.noisable_model
|
|
434
|
+
|
|
435
|
+
if isinstance(fitnesses, list):
|
|
436
|
+
fitnesses = tensor(fitnesses, dtype = torch.float32)
|
|
437
|
+
|
|
438
|
+
if isinstance(seeds_for_population, list):
|
|
439
|
+
seeds_for_population = tensor(seeds_for_population)
|
|
440
|
+
|
|
441
|
+
fitnesses = fitnesses.to(self.device)
|
|
442
|
+
seeds_for_population = seeds_for_population.to(self.device)
|
|
443
|
+
|
|
444
|
+
flat_fitnesses = fitnesses.view(-1) if self.mirror_sampling else fitnesses
|
|
445
|
+
sorted_indices = flat_fitnesses.argsort(descending = True)
|
|
446
|
+
top_mu_indices = sorted_indices[:self.cma_mu]
|
|
447
|
+
|
|
448
|
+
# weighted recombination of the best mu perturbations
|
|
449
|
+
|
|
450
|
+
z_step = {name: torch.zeros(self.param_shapes[name], dtype = self.param_dtypes[name], device = self.device) for name in self.param_names}
|
|
451
|
+
z_sq_step = {name: torch.zeros(self.param_shapes[name], dtype = self.param_dtypes[name], device = self.device) for name in self.param_names}
|
|
452
|
+
|
|
453
|
+
for rank_idx, flat_idx in enumerate(top_mu_indices.tolist()):
|
|
454
|
+
weight = self.cma_weights[rank_idx].item()
|
|
455
|
+
|
|
456
|
+
seed_idx = flat_idx // 2 if self.mirror_sampling else flat_idx
|
|
457
|
+
is_negated = (flat_idx % 2 == 1) if self.mirror_sampling else False
|
|
458
|
+
|
|
459
|
+
individual_seed = seeds_for_population[seed_idx].item()
|
|
460
|
+
individual_param_seeds = with_seed(individual_seed)(torch.randint)(0, MAX_SEED_VALUE, (self.num_params,))
|
|
461
|
+
|
|
462
|
+
noise_config = dict(zip(self.param_names, individual_param_seeds.tolist()))
|
|
463
|
+
|
|
464
|
+
if exists(scoped_names):
|
|
465
|
+
noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
|
|
466
|
+
|
|
467
|
+
for param_name, seed in noise_config.items():
|
|
468
|
+
shape = self.param_shapes[param_name]
|
|
469
|
+
dtype = self.param_dtypes[param_name]
|
|
470
|
+
noise = with_seed(seed)(model.create_noise_fn)(shape, dtype = dtype).to(self.device) # reproduce the noise from the seed
|
|
471
|
+
|
|
472
|
+
if is_negated:
|
|
473
|
+
noise = -noise
|
|
474
|
+
|
|
475
|
+
z_step[param_name].add_(noise * weight)
|
|
476
|
+
z_sq_step[param_name].add_((noise ** 2) * weight)
|
|
477
|
+
|
|
478
|
+
ps_norm_sq = 0.
|
|
479
|
+
|
|
480
|
+
# evolution paths, per coordinate
|
|
481
|
+
|
|
482
|
+
ps_scale = (self.cma_c_sigma * (2 - self.cma_c_sigma) * self.cma_mueff) ** 0.5
|
|
483
|
+
pc_scale = (self.cma_cc * (2 - self.cma_cc) * self.cma_mueff) ** 0.5
|
|
484
|
+
|
|
485
|
+
for name in self.param_names:
|
|
486
|
+
idx = self.param_name_to_cma_index[name]
|
|
487
|
+
z = z_step[name]
|
|
488
|
+
C = self.cma_C[idx]
|
|
489
|
+
|
|
490
|
+
self.cma_ps[idx].mul_(1 - self.cma_c_sigma).add_(z, alpha = ps_scale)
|
|
491
|
+
ps_norm_sq += self.cma_ps[idx].square().sum().item()
|
|
492
|
+
|
|
493
|
+
self.cma_pc[idx].mul_(1 - self.cma_cc).add_(torch.sqrt(C) * z, alpha = pc_scale)
|
|
494
|
+
|
|
495
|
+
# adapt the global step size sigma
|
|
496
|
+
|
|
497
|
+
ps_norm = ps_norm_sq ** 0.5
|
|
498
|
+
old_sigma = self.cma_sigma
|
|
499
|
+
self.cma_sigma *= exp((self.cma_c_sigma / self.cma_damps) * (ps_norm / self.cma_chiN - 1))
|
|
500
|
+
|
|
501
|
+
model_params = dict(self.model.named_parameters())
|
|
502
|
+
|
|
503
|
+
for name in self.param_names:
|
|
504
|
+
idx = self.param_name_to_cma_index[name]
|
|
505
|
+
C = self.cma_C[idx]
|
|
506
|
+
pc = self.cma_pc[idx]
|
|
507
|
+
z = z_step[name]
|
|
508
|
+
z_sq = z_sq_step[name]
|
|
509
|
+
|
|
510
|
+
old_C = C.clone()
|
|
511
|
+
|
|
512
|
+
# adapt the covariance diagonal, then update params
|
|
513
|
+
|
|
514
|
+
C.mul_(1 - self.cma_c1_sep - self.cma_cmu_sep)
|
|
515
|
+
C.add_(pc.square(), alpha = self.cma_c1_sep)
|
|
516
|
+
C.add_(old_C * z_sq, alpha = self.cma_cmu_sep)
|
|
517
|
+
|
|
518
|
+
update = old_sigma * torch.sqrt(old_C) * z
|
|
519
|
+
model_params[name].data.add_(update)
|
|
520
|
+
|
|
521
|
+
def extra_log(self):
|
|
522
|
+
return f'cma sigma: {self.cma_sigma:.3f}'
|
|
523
|
+
|
|
524
|
+
# defaults for classic plain es hyperparameters
|
|
525
|
+
# kept in sync with `EvoStrategy` for backward compat
|
|
526
|
+
|
|
527
|
+
PLAIN_ES_DEFAULT_KWARGS = dict(
|
|
528
|
+
learning_rate = 1e-3,
|
|
529
|
+
learned_noise_scale = False,
|
|
530
|
+
noise_scale_learning_rate = 1e-5,
|
|
531
|
+
noise_scale_clamp_range = (1e-3, 1e-1),
|
|
532
|
+
use_optimizer = True,
|
|
533
|
+
optimizer_klass = partial(SGD, nesterov = True, momentum = 0.1, weight_decay = 1e-2),
|
|
534
|
+
optimizer_kwargs = dict(),
|
|
535
|
+
use_sigma_optimizer = True,
|
|
536
|
+
sigma_optimizer_klass = partial(SGD, nesterov = True, momentum = 0.1),
|
|
537
|
+
sigma_optimizer_kwargs = dict(),
|
|
538
|
+
use_scheduler = False,
|
|
539
|
+
scheduler_klass = None,
|
|
540
|
+
scheduler_kwargs = dict(),
|
|
541
|
+
use_sigma_scheduler = False,
|
|
542
|
+
sigma_scheduler_klass = None,
|
|
543
|
+
sigma_scheduler_kwargs = dict(),
|
|
544
|
+
fitness_to_weighted_factor = 'normalize'
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
# registry of update rules, extend as new algorithms are added
|
|
548
|
+
|
|
549
|
+
UPDATE_RULES = dict(
|
|
550
|
+
es = PlainES,
|
|
551
|
+
plain_es = PlainES,
|
|
552
|
+
sep_cma_es = SepCMAES,
|
|
553
|
+
sep_cma = SepCMAES
|
|
554
|
+
)
|
|
@@ -72,6 +72,8 @@ FITNESS_WEIGHT_FACTORS = dict(
|
|
|
72
72
|
|
|
73
73
|
FitnessWeightFactorType = Literal['normalize', 'centered_rank']
|
|
74
74
|
|
|
75
|
+
ParametersToOptimize = list[str] | Module | list[Module] | list[Parameter]
|
|
76
|
+
|
|
75
77
|
# class
|
|
76
78
|
|
|
77
79
|
class EvoStrategy(Module):
|
|
@@ -86,7 +88,7 @@ class EvoStrategy(Module):
|
|
|
86
88
|
noise_population_size = 30,
|
|
87
89
|
learning_rate = 1e-3,
|
|
88
90
|
mirror_sampling = True,
|
|
89
|
-
params_to_optimize:
|
|
91
|
+
params_to_optimize: dict[str, ParametersToOptimize] | ParametersToOptimize | None = None,
|
|
90
92
|
noise_low_rank: int | None = None,
|
|
91
93
|
rollout_fixed_seed = True,
|
|
92
94
|
noise_scale = 1e-2, # the noise scaling during rollouts with environment, todo - figure out right value and make sure it can also be customized per parameter name through a dict
|
|
@@ -115,6 +117,7 @@ class EvoStrategy(Module):
|
|
|
115
117
|
accelerate_kwargs: dict = dict(),
|
|
116
118
|
reject_generation_fitnesses_if: Callable[[Tensor], bool] | None = None,
|
|
117
119
|
on_result: Callable | None = None,
|
|
120
|
+
on_generation: Callable | None = None,
|
|
118
121
|
vectorized = False,
|
|
119
122
|
vector_size: int | None = None,
|
|
120
123
|
sync_on_init = True
|
|
@@ -125,6 +128,7 @@ class EvoStrategy(Module):
|
|
|
125
128
|
self.vectorized = vectorized
|
|
126
129
|
self.vector_size = vector_size
|
|
127
130
|
self.on_result = on_result
|
|
131
|
+
self.on_generation = on_generation
|
|
128
132
|
|
|
129
133
|
if not exists(accelerator):
|
|
130
134
|
accelerator = Accelerator(cpu = cpu, **accelerate_kwargs)
|
|
@@ -136,11 +140,12 @@ class EvoStrategy(Module):
|
|
|
136
140
|
self.environment = environment
|
|
137
141
|
|
|
138
142
|
if accelerator.is_main_process:
|
|
139
|
-
|
|
143
|
+
pre_main_callback = getattr(environment, 'pre_main_callback', None)
|
|
144
|
+
if callable(pre_main_callback):
|
|
140
145
|
self.print('pre_main_callback detected on environment passed in and is invoked')
|
|
141
|
-
|
|
146
|
+
pre_main_callback()
|
|
142
147
|
|
|
143
|
-
|
|
148
|
+
self.wait_for_everyone()
|
|
144
149
|
|
|
145
150
|
# take care of model and parameters
|
|
146
151
|
|
|
@@ -160,24 +165,31 @@ class EvoStrategy(Module):
|
|
|
160
165
|
named_parameters_dict = dict(model.named_parameters())
|
|
161
166
|
|
|
162
167
|
param_to_name_index = {param: name for name, param in named_parameters_dict.items()}
|
|
168
|
+
self._param_to_name_index = param_to_name_index
|
|
163
169
|
|
|
164
170
|
param_names = set(named_parameters_dict.keys())
|
|
165
171
|
|
|
166
172
|
# default to all parameters to optimize with evo strategy
|
|
167
173
|
|
|
168
|
-
|
|
174
|
+
self.scopes = None
|
|
169
175
|
|
|
170
|
-
|
|
171
|
-
|
|
176
|
+
if isinstance(params_to_optimize, dict):
|
|
177
|
+
self.scopes = dict()
|
|
178
|
+
all_scoped_params = set()
|
|
172
179
|
|
|
173
|
-
|
|
174
|
-
|
|
180
|
+
for scope_name, scope_params in params_to_optimize.items():
|
|
181
|
+
resolved_scope_params = self._resolve_params_to_names(scope_params)
|
|
182
|
+
assert all([name in param_names for name in resolved_scope_params]), f'some parameters in scope {scope_name} are not found in the model'
|
|
183
|
+
self.scopes[scope_name] = resolved_scope_params
|
|
184
|
+
all_scoped_params |= resolved_scope_params
|
|
175
185
|
|
|
176
|
-
|
|
177
|
-
|
|
186
|
+
params_to_optimize = list(all_scoped_params)
|
|
187
|
+
|
|
188
|
+
params_to_optimize = default(params_to_optimize, param_names)
|
|
178
189
|
|
|
179
|
-
|
|
180
|
-
|
|
190
|
+
# convert to names using the helper method
|
|
191
|
+
|
|
192
|
+
params_to_optimize = list(self._resolve_params_to_names(params_to_optimize))
|
|
181
193
|
|
|
182
194
|
# validate
|
|
183
195
|
|
|
@@ -280,9 +292,19 @@ class EvoStrategy(Module):
|
|
|
280
292
|
def device(self):
|
|
281
293
|
return self.accelerate.device
|
|
282
294
|
|
|
295
|
+
@property
|
|
296
|
+
def is_distributed(self):
|
|
297
|
+
return self.accelerate.num_processes > 1
|
|
298
|
+
|
|
299
|
+
def wait_for_everyone(self):
|
|
300
|
+
if not (self.is_distributed and dist.is_available() and dist.is_initialized()):
|
|
301
|
+
return
|
|
302
|
+
|
|
303
|
+
dist.barrier()
|
|
304
|
+
|
|
283
305
|
@torch.no_grad()
|
|
284
306
|
def sync_model_params_and_buffers_(self):
|
|
285
|
-
if not self.
|
|
307
|
+
if not (self.is_distributed and dist.is_available() and dist.is_initialized()):
|
|
286
308
|
return
|
|
287
309
|
|
|
288
310
|
for param in self.model.parameters():
|
|
@@ -291,6 +313,21 @@ class EvoStrategy(Module):
|
|
|
291
313
|
for buffer in self.model.buffers():
|
|
292
314
|
dist.broadcast(buffer, src = 0)
|
|
293
315
|
|
|
316
|
+
def _resolve_params_to_names(
|
|
317
|
+
self,
|
|
318
|
+
params: ParametersToOptimize
|
|
319
|
+
):
|
|
320
|
+
|
|
321
|
+
if isinstance(params, Module):
|
|
322
|
+
params = list(params.parameters())
|
|
323
|
+
elif is_bearable(params, list[Module]):
|
|
324
|
+
params = list(ModuleList(params).parameters())
|
|
325
|
+
|
|
326
|
+
if is_bearable(params, list[Parameter]):
|
|
327
|
+
params = [self._param_to_name_index[param] for param in set(params)]
|
|
328
|
+
|
|
329
|
+
return set(params)
|
|
330
|
+
|
|
294
331
|
def print(self, *args, **kwargs):
|
|
295
332
|
if not self.verbose:
|
|
296
333
|
return
|
|
@@ -308,7 +345,8 @@ class EvoStrategy(Module):
|
|
|
308
345
|
def evolve_(
|
|
309
346
|
self,
|
|
310
347
|
fitnesses: list[float] | Tensor,
|
|
311
|
-
seeds_for_population: list[int] | Tensor
|
|
348
|
+
seeds_for_population: list[int] | Tensor,
|
|
349
|
+
scoped_names: set[str] | None = None
|
|
312
350
|
):
|
|
313
351
|
use_optimizer = self.use_optimizer
|
|
314
352
|
model = self.noisable_model
|
|
@@ -349,6 +387,9 @@ class EvoStrategy(Module):
|
|
|
349
387
|
|
|
350
388
|
noise_config = dict(zip(self.param_names_to_optimize, individual_param_seeds.tolist()))
|
|
351
389
|
|
|
390
|
+
if exists(scoped_names):
|
|
391
|
+
noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
|
|
392
|
+
|
|
352
393
|
noise_config_with_weight = dict()
|
|
353
394
|
|
|
354
395
|
# loop through each noise
|
|
@@ -418,7 +459,7 @@ class EvoStrategy(Module):
|
|
|
418
459
|
filepath = self.checkpoint_folder / f'{filename}.pt'
|
|
419
460
|
torch.save(self.model.state_dict(), str(filepath))
|
|
420
461
|
|
|
421
|
-
self.
|
|
462
|
+
self.wait_for_everyone()
|
|
422
463
|
|
|
423
464
|
@torch.inference_mode()
|
|
424
465
|
def forward(
|
|
@@ -427,10 +468,21 @@ class EvoStrategy(Module):
|
|
|
427
468
|
num_generations = None,
|
|
428
469
|
disable_distributed = False,
|
|
429
470
|
rollback_model_at_end = False,
|
|
430
|
-
verbose = None
|
|
471
|
+
verbose = None,
|
|
472
|
+
scope: str | None = None,
|
|
473
|
+
params_to_optimize: ParametersToOptimize | None = None
|
|
431
474
|
):
|
|
432
475
|
verbose = default(verbose, self.verbose)
|
|
433
476
|
|
|
477
|
+
scoped_names = None
|
|
478
|
+
|
|
479
|
+
if exists(scope):
|
|
480
|
+
assert exists(self.scopes) and scope in self.scopes, f'scope "{scope}" not found in passed parameter dictionary'
|
|
481
|
+
scoped_names = self.scopes[scope]
|
|
482
|
+
|
|
483
|
+
if exists(params_to_optimize):
|
|
484
|
+
scoped_names = self._resolve_params_to_names(params_to_optimize)
|
|
485
|
+
|
|
434
486
|
model = self.noisable_model.to(self.device)
|
|
435
487
|
|
|
436
488
|
# maybe save model for rolling back (for meta-evo)
|
|
@@ -494,17 +546,6 @@ class EvoStrategy(Module):
|
|
|
494
546
|
|
|
495
547
|
fitnesses = []
|
|
496
548
|
|
|
497
|
-
# function for fitness
|
|
498
|
-
|
|
499
|
-
def rollout_for_fitness():
|
|
500
|
-
fitness = self.environment(model)
|
|
501
|
-
|
|
502
|
-
if is_tensor(fitness):
|
|
503
|
-
assert fitness.numel() == 1
|
|
504
|
-
fitness = fitness.item()
|
|
505
|
-
|
|
506
|
-
return fitness
|
|
507
|
-
|
|
508
549
|
# seeds
|
|
509
550
|
|
|
510
551
|
maybe_rollout_seed = maybe_get_synced_seed() if self.rollout_fixed_seed else None
|
|
@@ -525,6 +566,9 @@ class EvoStrategy(Module):
|
|
|
525
566
|
individual_param_seeds = with_seed(individual_seed.item())(randint)(0, MAX_SEED_VALUE, (self.num_params,))
|
|
526
567
|
noise_config = dict(zip(self.param_names_to_optimize, individual_param_seeds.tolist()))
|
|
527
568
|
|
|
569
|
+
if exists(scoped_names):
|
|
570
|
+
noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
|
|
571
|
+
|
|
528
572
|
noise_config_with_scale = dict()
|
|
529
573
|
for param_name, seed in noise_config.items():
|
|
530
574
|
noise_scale = self._get_noise_scale(param_name)
|
|
@@ -590,7 +634,12 @@ class EvoStrategy(Module):
|
|
|
590
634
|
|
|
591
635
|
# pass fitnesses to evolve function
|
|
592
636
|
|
|
593
|
-
self.evolve_(fitnesses, seeds_for_population)
|
|
637
|
+
self.evolve_(fitnesses, seeds_for_population, scoped_names=scoped_names)
|
|
638
|
+
|
|
639
|
+
# on generation callback
|
|
640
|
+
|
|
641
|
+
if exists(self.on_generation) and self.accelerate.is_main_process:
|
|
642
|
+
self.on_generation(generation, fitnesses, self.model)
|
|
594
643
|
|
|
595
644
|
# log
|
|
596
645
|
|
|
@@ -609,7 +658,7 @@ class EvoStrategy(Module):
|
|
|
609
658
|
exists(self.checkpoint_every) and
|
|
610
659
|
divisible_by(generation, self.checkpoint_every)
|
|
611
660
|
):
|
|
612
|
-
self.checkpoint(f'{filename}.{generation}
|
|
661
|
+
self.checkpoint(f'{filename}.{generation}')
|
|
613
662
|
|
|
614
663
|
# increment generation
|
|
615
664
|
|
|
@@ -631,4 +680,7 @@ class EvoStrategy(Module):
|
|
|
631
680
|
# return fitnesses across generations
|
|
632
681
|
# for meta-evolutionary (nesting EvoStrategy within the environment of another and optimizing some meta-network)
|
|
633
682
|
|
|
683
|
+
if len(fitnesses_across_generations) == 0:
|
|
684
|
+
return tensor([], device = self.device)
|
|
685
|
+
|
|
634
686
|
return stack(fitnesses_across_generations)
|
|
File without changes
|
|
File without changes
|