x-evolution 0.1.32__tar.gz → 0.1.34__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,8 @@
1
1
  recordings/
2
2
  recordings_humanoid/
3
+ microduck_rl/
4
+ checkpoints-microduck/
5
+ videos/
3
6
 
4
7
  .DS_Store
5
8
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: x-evolution
3
- Version: 0.1.32
3
+ Version: 0.1.34
4
4
  Summary: x-evolution
5
5
  Project-URL: Homepage, https://pypi.org/project/x-evolution/
6
6
  Project-URL: Repository, https://github.com/lucidrains/x-evolution
@@ -122,6 +122,28 @@ For a quick demo on LunarLander with 8 processes
122
122
  $ uv run torchrun --nproc_per_node=8 train_lunar.py --cpu True --noise_population_size 100 --fitness_to_weighted_factor centered_rank
123
123
  ```
124
124
 
125
+ ## Sim-to-Real Bipedal Locomotion (Microduck)
126
+
127
+ You can train a walking gait for the [Microduck](https://github.com/pollen-robotics/reachy2_microduck) biped robot from scratch using `EvoStrategy` and `torchrun`.
128
+
129
+ With a population size of 128 distributed across 8 CPU processes, a forward walking gait emerges around **150 generations** (~3-4 minutes):
130
+
131
+ ```shell
132
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py
133
+ ```
134
+
135
+ To render evaluation videos or resume from the saved best checkpoint:
136
+
137
+ ```shell
138
+ # render periodic evaluation videos to videos/
139
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py --eval_video True --eval_every 50
140
+
141
+ # resume training from best checkpoint
142
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py --resume True
143
+ ```
144
+
145
+ Special thanks to [Pollen Robotics 🤗](https://www.pollen-robotics.com/) for their open sourcing of such a polished sim2real env!
146
+
125
147
  ## Citations
126
148
 
127
149
  ```bibtex
@@ -71,6 +71,28 @@ For a quick demo on LunarLander with 8 processes
71
71
  $ uv run torchrun --nproc_per_node=8 train_lunar.py --cpu True --noise_population_size 100 --fitness_to_weighted_factor centered_rank
72
72
  ```
73
73
 
74
+ ## Sim-to-Real Bipedal Locomotion (Microduck)
75
+
76
+ You can train a walking gait for the [Microduck](https://github.com/pollen-robotics/reachy2_microduck) biped robot from scratch using `EvoStrategy` and `torchrun`.
77
+
78
+ With a population size of 128 distributed across 8 CPU processes, a forward walking gait emerges around **150 generations** (~3-4 minutes):
79
+
80
+ ```shell
81
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py
82
+ ```
83
+
84
+ To render evaluation videos or resume from the saved best checkpoint:
85
+
86
+ ```shell
87
+ # render periodic evaluation videos to videos/
88
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py --eval_video True --eval_every 50
89
+
90
+ # resume training from best checkpoint
91
+ $ uv run torchrun --nproc_per_node=8 train_microduck.py --resume True
92
+ ```
93
+
94
+ Special thanks to [Pollen Robotics 🤗](https://www.pollen-robotics.com/) for their open sourcing of such a polished sim2real env!
95
+
74
96
  ## Citations
75
97
 
76
98
  ```bibtex
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "x-evolution"
3
- version = "0.1.32"
3
+ version = "0.1.34"
4
4
  description = "x-evolution"
5
5
  authors = [
6
6
  { name = "Phil Wang", email = "lucidrains@gmail.com" }
@@ -0,0 +1,554 @@
1
+ from __future__ import annotations
2
+
3
+ from math import exp, log, sqrt
4
+ from functools import partial
5
+ from typing import Callable, Literal
6
+
7
+ import torch
8
+ from torch import tensor, Tensor
9
+ from torch.nn import Module, Parameter, ParameterList
10
+ from torch.optim import SGD, Optimizer
11
+ from torch.optim.lr_scheduler import LRScheduler
12
+
13
+ import torch.nn.functional as F
14
+
15
+ from beartype import beartype
16
+
17
+ from einops import pack
18
+ from torch_einops_utils import pack_with_inverse
19
+
20
+ from x_mlps_pytorch.noisable import Noisable, with_seed
21
+
22
+ # constants
23
+
24
+ MAX_SEED_VALUE = int(2 ** 32)
25
+
26
+ # helper functions
27
+
28
+ def exists(v):
29
+ return v is not None
30
+
31
+ def default(v, d):
32
+ return v if exists(v) else d
33
+
34
+ def accum_grad_(t, value):
35
+ if not exists(t.grad):
36
+ t.grad = value.clone()
37
+ else:
38
+ t.grad.add_(value)
39
+
40
+ # fitness weighting factors
41
+
42
+ def normalize(t, eps = 1e-6):
43
+ return F.layer_norm(t, t.shape, eps = eps)
44
+
45
+ def centered_rank_normalize(fitnesses):
46
+ dtype, device, n = fitnesses.dtype, fitnesses.device, fitnesses.numel()
47
+ flat_fitnesses, inverse = pack_with_inverse(fitnesses, '*')
48
+ ranks = flat_fitnesses.argsort().argsort().to(dtype = dtype)
49
+ ranks = (ranks / (n - 1)) - 0.5
50
+ return inverse(ranks)
51
+
52
+ FITNESS_WEIGHT_FACTORS = dict(
53
+ normalize = normalize,
54
+ centered_rank = centered_rank_normalize
55
+ )
56
+
57
+ FitnessWeightFactorType = Literal['normalize', 'centered_rank']
58
+
59
+ # base update rule
60
+ # turns the population of (fitness, seed) pairs into a param update, given the noise
61
+ # can be reproduced deterministically from the seeds via the `Noisable` wrapper
62
+
63
+ class UpdateRule(Module):
64
+
65
+ @beartype
66
+ def __init__(
67
+ self,
68
+ *,
69
+ model: Module,
70
+ noisable_model: Noisable,
71
+ named_parameters: dict[str, Parameter],
72
+ param_names: list[str],
73
+ param_shapes: dict[str, torch.Size],
74
+ param_dtypes: dict[str, torch.dtype],
75
+ noise_population_size: int,
76
+ mirror_sampling: bool,
77
+ noise_scale: float,
78
+ device
79
+ ):
80
+ super().__init__()
81
+
82
+ self.model = model
83
+ self.noisable_model = noisable_model
84
+
85
+ self.named_parameters = named_parameters
86
+ self.param_names = list(param_names)
87
+ self.param_shapes = param_shapes
88
+ self.param_dtypes = param_dtypes
89
+
90
+ self.noise_population_size = noise_population_size
91
+ self.mirror_sampling = mirror_sampling
92
+ self.noise_scale = noise_scale
93
+
94
+ self.device = device
95
+
96
+ @property
97
+ def num_params(self):
98
+ return len(self.param_names)
99
+
100
+ def noise_scale_for(self, param_name):
101
+ # scale used to perturb a given param during rollouts
102
+
103
+ raise NotImplementedError
104
+
105
+ @torch.inference_mode()
106
+ def evolve(
107
+ self,
108
+ fitnesses: list[float] | Tensor,
109
+ seeds_for_population: list[int] | Tensor,
110
+ scoped_names: set[str] | None = None
111
+ ):
112
+ # update model params given fitnesses and the seeds reproducing the perturbation noise
113
+
114
+ raise NotImplementedError
115
+
116
+ def to_device(self, device):
117
+ self.to(device)
118
+ return self
119
+
120
+ def extra_log(self):
121
+ # extra state to log at end of each generation
122
+
123
+ return ''
124
+
125
+ # plain es - the weighted average of the noise, scaled by normalized fitnesses, is the gradient estimate
126
+ # optionally with learned per-parameter sigmas
127
+
128
+ class PlainES(UpdateRule):
129
+
130
+ @beartype
131
+ def __init__(
132
+ self,
133
+ *,
134
+ model: Module,
135
+ noisable_model: Noisable,
136
+ named_parameters: dict[str, Parameter],
137
+ param_names: list[str],
138
+ param_shapes: dict[str, torch.Size],
139
+ param_dtypes: dict[str, torch.dtype],
140
+ noise_population_size: int,
141
+ mirror_sampling: bool,
142
+ noise_scale = 1e-2, # noise scaling during rollouts, can be customized per parameter name
143
+ device,
144
+ learning_rate = 1e-3,
145
+ learned_noise_scale = False,
146
+ noise_scale_learning_rate = 1e-5,
147
+ noise_scale_clamp_range: tuple[float, float] = (1e-3, 1e-1),
148
+ use_optimizer = True,
149
+ optimizer_klass: type[Optimizer] | Callable = partial(SGD, nesterov = True, momentum = 0.1, weight_decay = 1e-2),
150
+ optimizer_kwargs: dict = dict(),
151
+ use_sigma_optimizer = True,
152
+ sigma_optimizer_klass: type[Optimizer] | Callable = partial(SGD, nesterov = True, momentum = 0.1),
153
+ sigma_optimizer_kwargs: dict = dict(),
154
+ use_scheduler = False,
155
+ scheduler_klass: type[LRScheduler] | None = None,
156
+ scheduler_kwargs: dict = dict(),
157
+ use_sigma_scheduler = False,
158
+ sigma_scheduler_klass: type[LRScheduler] | None = None,
159
+ sigma_scheduler_kwargs: dict = dict(),
160
+ fitness_to_weighted_factor: Callable[[Tensor], Tensor] | FitnessWeightFactorType = 'normalize'
161
+ ):
162
+ super().__init__(
163
+ model = model,
164
+ noisable_model = noisable_model,
165
+ named_parameters = named_parameters,
166
+ param_names = param_names,
167
+ param_shapes = param_shapes,
168
+ param_dtypes = param_dtypes,
169
+ noise_population_size = noise_population_size,
170
+ mirror_sampling = mirror_sampling,
171
+ noise_scale = noise_scale,
172
+ device = device
173
+ )
174
+
175
+ self.learning_rate = learning_rate
176
+
177
+ # transforms fitnesses to weights for the weighted average of the noise
178
+
179
+ if isinstance(fitness_to_weighted_factor, str):
180
+ if fitness_to_weighted_factor not in FITNESS_WEIGHT_FACTORS:
181
+ raise ValueError(f'fitness_to_weighted_factor must be one of {list(FITNESS_WEIGHT_FACTORS.keys())}')
182
+ fitness_to_weighted_factor = FITNESS_WEIGHT_FACTORS[fitness_to_weighted_factor]
183
+
184
+ self.fitness_to_weighted_factor = fitness_to_weighted_factor
185
+
186
+ # noise scale, fixed or learned
187
+
188
+ self.learned_noise_scale = learned_noise_scale
189
+
190
+ if learned_noise_scale:
191
+ self.sigmas = ParameterList([Parameter(torch.ones_like(param) * noise_scale) for _, param in named_parameters.items()])
192
+ self.param_name_to_sigma_index = {name: i for i, name in enumerate(named_parameters.keys())}
193
+
194
+ min_noise_scale, max_noise_scale = noise_scale_clamp_range
195
+ assert min_noise_scale > 0. and min_noise_scale < max_noise_scale
196
+
197
+ self.sigma_clamp_ = lambda t: t.clamp_(*noise_scale_clamp_range)
198
+
199
+ self.noise_scale_learning_rate = noise_scale_learning_rate
200
+
201
+ # maybe use optimizer, allow for Adam
202
+
203
+ self.use_optimizer = use_optimizer
204
+
205
+ if use_optimizer:
206
+ optim_params = [named_parameters[name] for name in param_names]
207
+ self.optimizer = optimizer_klass(optim_params, lr = learning_rate, **optimizer_kwargs)
208
+
209
+ self.use_sigma_optimizer = use_sigma_optimizer
210
+
211
+ if learned_noise_scale and use_sigma_optimizer:
212
+ self.sigma_optimizer = sigma_optimizer_klass(self.sigmas, lr = noise_scale_learning_rate, **sigma_optimizer_kwargs)
213
+
214
+ self.use_scheduler = use_scheduler
215
+
216
+ if use_scheduler and exists(scheduler_klass) and use_optimizer:
217
+ self.scheduler = scheduler_klass(self.optimizer, **scheduler_kwargs)
218
+
219
+ self.use_sigma_scheduler = use_sigma_scheduler
220
+
221
+ if use_sigma_scheduler and exists(sigma_scheduler_klass) and use_sigma_optimizer:
222
+ self.sigma_scheduler = sigma_scheduler_klass(self.sigma_optimizer, **sigma_scheduler_kwargs)
223
+
224
+ def noise_scale_for(self, param_name):
225
+ if not self.learned_noise_scale:
226
+ return self.noise_scale
227
+
228
+ sigma_index = self.param_name_to_sigma_index[param_name]
229
+ return self.sigmas[sigma_index]
230
+
231
+ @torch.inference_mode()
232
+ def evolve(
233
+ self,
234
+ fitnesses: list[float] | Tensor,
235
+ seeds_for_population: list[int] | Tensor,
236
+ scoped_names: set[str] | None = None
237
+ ):
238
+ use_optimizer = self.use_optimizer
239
+ model = self.noisable_model
240
+
241
+ if isinstance(fitnesses, list):
242
+ fitnesses = tensor(fitnesses, dtype = torch.float32)
243
+
244
+ if isinstance(seeds_for_population, list):
245
+ seeds_for_population = tensor(seeds_for_population)
246
+
247
+ fitnesses = fitnesses.to(self.device)
248
+ seeds_for_population = seeds_for_population.to(self.device)
249
+
250
+ # maybe normalize the fitness with z-score
251
+
252
+ fitnesses = self.fitness_to_weighted_factor(fitnesses)
253
+
254
+ if self.mirror_sampling:
255
+ fitness_pos, fitness_neg = fitnesses.unbind(dim = -1)
256
+ weights = fitness_pos - fitness_neg
257
+ else:
258
+ weights = fitnesses
259
+
260
+ weights /= self.noise_population_size * (2. if self.mirror_sampling else 1.)
261
+
262
+ # update one seed at a time, for large models
263
+
264
+ iterator = zip(seeds_for_population.tolist(), weights.tolist())
265
+
266
+ for individual_seed, weight in iterator:
267
+
268
+ individual_param_seeds = with_seed(individual_seed)(torch.randint)(0, MAX_SEED_VALUE, (self.num_params,))
269
+
270
+ # setup noise config
271
+
272
+ noise_config = dict(zip(self.param_names, individual_param_seeds.tolist()))
273
+
274
+ if exists(scoped_names):
275
+ noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
276
+
277
+ noise_config_with_weight = dict()
278
+
279
+ # loop through each noise
280
+
281
+ for param_name, seed in noise_config.items():
282
+
283
+ shape = self.param_shapes[param_name]
284
+ dtype = self.param_dtypes[param_name]
285
+
286
+ # reproduce the noise from the seed
287
+
288
+ noise = with_seed(seed)(model.create_noise_fn)(shape, dtype = dtype)
289
+ noise = noise.to(self.device)
290
+
291
+ # scale the weight
292
+
293
+ noise_scale = self.noise_scale_for(param_name)
294
+
295
+ scaled_weight = weight / noise_scale
296
+
297
+ if not use_optimizer:
298
+ scaled_weight = scaled_weight * self.learning_rate
299
+
300
+ # set the weighted noise
301
+
302
+ noise_config_with_weight[param_name] = (noise, scaled_weight)
303
+
304
+ # maybe learned sigma
305
+
306
+ if self.learned_noise_scale:
307
+ one_grad_sigma = weight * (noise ** 2 - 1) / noise_scale
308
+
309
+ sigma_index = self.param_name_to_sigma_index[param_name]
310
+ sigma = self.sigmas[sigma_index]
311
+
312
+ if self.use_sigma_optimizer:
313
+ accum_grad_(sigma, -one_grad_sigma)
314
+ else:
315
+ sigma.add_(one_grad_sigma * self.noise_scale_learning_rate)
316
+
317
+ # now update params for one seed
318
+
319
+ model.add_noise_(noise_config_with_weight, negate = use_optimizer, add_to_grad = use_optimizer)
320
+
321
+ if use_optimizer:
322
+ self.optimizer.step()
323
+ self.optimizer.zero_grad()
324
+
325
+ if self.use_scheduler and exists(self.scheduler):
326
+ self.scheduler.step()
327
+
328
+ if self.learned_noise_scale:
329
+ if self.use_sigma_optimizer:
330
+ self.sigma_optimizer.step()
331
+ self.sigma_optimizer.zero_grad()
332
+
333
+ if self.use_sigma_scheduler and exists(self.sigma_scheduler):
334
+ self.sigma_scheduler.step()
335
+
336
+ for sigma in self.sigmas:
337
+ self.sigma_clamp_(sigma)
338
+
339
+ def extra_log(self):
340
+ if not self.learned_noise_scale:
341
+ return ''
342
+
343
+ packed_sigma, _ = pack(list(self.sigmas), '*')
344
+ return f'average sigma: {packed_sigma.mean().item():.3f}'
345
+
346
+ # sep cma-es
347
+
348
+ class SepCMAES(UpdateRule):
349
+
350
+ @beartype
351
+ def __init__(
352
+ self,
353
+ *,
354
+ model: Module,
355
+ noisable_model: Noisable,
356
+ named_parameters: dict[str, Parameter],
357
+ param_names: list[str],
358
+ param_shapes: dict[str, torch.Size],
359
+ param_dtypes: dict[str, torch.dtype],
360
+ noise_population_size: int,
361
+ mirror_sampling: bool,
362
+ noise_scale = 1e-2, # initial global step size sigma
363
+ device,
364
+ cma_mu: int | None = None,
365
+ **plain_es_compat_kwargs
366
+ ):
367
+ # validate that only known plain es hyperparameters are being ignored
368
+
369
+ unsupported_kwargs = set(plain_es_compat_kwargs) - set(PLAIN_ES_DEFAULT_KWARGS)
370
+
371
+ if unsupported_kwargs:
372
+ raise TypeError(f'SepCMAES.__init__() got unexpected keyword arguments: {unsupported_kwargs}')
373
+
374
+ super().__init__(
375
+ model = model,
376
+ noisable_model = noisable_model,
377
+ named_parameters = named_parameters,
378
+ param_names = param_names,
379
+ param_shapes = param_shapes,
380
+ param_dtypes = param_dtypes,
381
+ noise_population_size = noise_population_size,
382
+ mirror_sampling = mirror_sampling,
383
+ noise_scale = noise_scale,
384
+ device = device
385
+ )
386
+
387
+ pop_size = self.noise_population_size * (2 if self.mirror_sampling else 1)
388
+ mu = default(cma_mu, pop_size // 2)
389
+ self.cma_mu = mu
390
+
391
+ dim = sum(self.param_shapes[name].numel() for name in self.param_names)
392
+
393
+ # cma strategy constants
394
+
395
+ weights = log(mu + 0.5) - torch.log(torch.arange(1, mu + 1, dtype = torch.float32))
396
+ weights /= weights.sum()
397
+ mueff = 1 / (weights ** 2).sum().item()
398
+
399
+ self.register_buffer('cma_weights', weights)
400
+
401
+ cc = (4 + mueff / dim) / (dim + 4 + 2 * mueff / dim)
402
+ c_sigma = (mueff + 2) / (dim + mueff + 5)
403
+ c1 = 2 / ((dim + 1.3) ** 2 + mueff)
404
+ cmu_coeff = min(1 - c1, 2 * (mueff - 2 + 1 / mueff) / ((dim + 2) ** 2 + mueff))
405
+
406
+ self.cma_c1_sep = min(1, c1 * (dim + 1.5) / 3)
407
+ self.cma_cmu_sep = min(1 - self.cma_c1_sep, cmu_coeff * (dim + 1.5) / 3)
408
+ self.cma_cc = cc
409
+ self.cma_c_sigma = c_sigma
410
+ self.cma_damps = 1 + 2 * max(0, sqrt((mueff - 1) / (dim + 1)) - 1) + c_sigma
411
+ self.cma_chiN = sqrt(dim) * (1 - 1 / (4 * dim) + 1 / (21 * dim ** 2))
412
+ self.cma_mueff = mueff
413
+
414
+ self.cma_sigma = noise_scale
415
+
416
+ self.param_name_to_cma_index = {name: i for i, name in enumerate(self.param_names)}
417
+
418
+ self.cma_pc = ParameterList([Parameter(torch.zeros_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
419
+ self.cma_ps = ParameterList([Parameter(torch.zeros_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
420
+ self.cma_C = ParameterList([Parameter(torch.ones_like(named_parameters[name]), requires_grad = False) for name in self.param_names])
421
+
422
+ def noise_scale_for(self, param_name):
423
+ idx = self.param_name_to_cma_index[param_name]
424
+ return self.cma_sigma * torch.sqrt(self.cma_C[idx])
425
+
426
+ @torch.inference_mode()
427
+ def evolve(
428
+ self,
429
+ fitnesses: list[float] | Tensor,
430
+ seeds_for_population: list[int] | Tensor,
431
+ scoped_names: set[str] | None = None
432
+ ):
433
+ model = self.noisable_model
434
+
435
+ if isinstance(fitnesses, list):
436
+ fitnesses = tensor(fitnesses, dtype = torch.float32)
437
+
438
+ if isinstance(seeds_for_population, list):
439
+ seeds_for_population = tensor(seeds_for_population)
440
+
441
+ fitnesses = fitnesses.to(self.device)
442
+ seeds_for_population = seeds_for_population.to(self.device)
443
+
444
+ flat_fitnesses = fitnesses.view(-1) if self.mirror_sampling else fitnesses
445
+ sorted_indices = flat_fitnesses.argsort(descending = True)
446
+ top_mu_indices = sorted_indices[:self.cma_mu]
447
+
448
+ # weighted recombination of the best mu perturbations
449
+
450
+ z_step = {name: torch.zeros(self.param_shapes[name], dtype = self.param_dtypes[name], device = self.device) for name in self.param_names}
451
+ z_sq_step = {name: torch.zeros(self.param_shapes[name], dtype = self.param_dtypes[name], device = self.device) for name in self.param_names}
452
+
453
+ for rank_idx, flat_idx in enumerate(top_mu_indices.tolist()):
454
+ weight = self.cma_weights[rank_idx].item()
455
+
456
+ seed_idx = flat_idx // 2 if self.mirror_sampling else flat_idx
457
+ is_negated = (flat_idx % 2 == 1) if self.mirror_sampling else False
458
+
459
+ individual_seed = seeds_for_population[seed_idx].item()
460
+ individual_param_seeds = with_seed(individual_seed)(torch.randint)(0, MAX_SEED_VALUE, (self.num_params,))
461
+
462
+ noise_config = dict(zip(self.param_names, individual_param_seeds.tolist()))
463
+
464
+ if exists(scoped_names):
465
+ noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
466
+
467
+ for param_name, seed in noise_config.items():
468
+ shape = self.param_shapes[param_name]
469
+ dtype = self.param_dtypes[param_name]
470
+ noise = with_seed(seed)(model.create_noise_fn)(shape, dtype = dtype).to(self.device) # reproduce the noise from the seed
471
+
472
+ if is_negated:
473
+ noise = -noise
474
+
475
+ z_step[param_name].add_(noise * weight)
476
+ z_sq_step[param_name].add_((noise ** 2) * weight)
477
+
478
+ ps_norm_sq = 0.
479
+
480
+ # evolution paths, per coordinate
481
+
482
+ ps_scale = (self.cma_c_sigma * (2 - self.cma_c_sigma) * self.cma_mueff) ** 0.5
483
+ pc_scale = (self.cma_cc * (2 - self.cma_cc) * self.cma_mueff) ** 0.5
484
+
485
+ for name in self.param_names:
486
+ idx = self.param_name_to_cma_index[name]
487
+ z = z_step[name]
488
+ C = self.cma_C[idx]
489
+
490
+ self.cma_ps[idx].mul_(1 - self.cma_c_sigma).add_(z, alpha = ps_scale)
491
+ ps_norm_sq += self.cma_ps[idx].square().sum().item()
492
+
493
+ self.cma_pc[idx].mul_(1 - self.cma_cc).add_(torch.sqrt(C) * z, alpha = pc_scale)
494
+
495
+ # adapt the global step size sigma
496
+
497
+ ps_norm = ps_norm_sq ** 0.5
498
+ old_sigma = self.cma_sigma
499
+ self.cma_sigma *= exp((self.cma_c_sigma / self.cma_damps) * (ps_norm / self.cma_chiN - 1))
500
+
501
+ model_params = dict(self.model.named_parameters())
502
+
503
+ for name in self.param_names:
504
+ idx = self.param_name_to_cma_index[name]
505
+ C = self.cma_C[idx]
506
+ pc = self.cma_pc[idx]
507
+ z = z_step[name]
508
+ z_sq = z_sq_step[name]
509
+
510
+ old_C = C.clone()
511
+
512
+ # adapt the covariance diagonal, then update params
513
+
514
+ C.mul_(1 - self.cma_c1_sep - self.cma_cmu_sep)
515
+ C.add_(pc.square(), alpha = self.cma_c1_sep)
516
+ C.add_(old_C * z_sq, alpha = self.cma_cmu_sep)
517
+
518
+ update = old_sigma * torch.sqrt(old_C) * z
519
+ model_params[name].data.add_(update)
520
+
521
+ def extra_log(self):
522
+ return f'cma sigma: {self.cma_sigma:.3f}'
523
+
524
+ # defaults for classic plain es hyperparameters
525
+ # kept in sync with `EvoStrategy` for backward compat
526
+
527
+ PLAIN_ES_DEFAULT_KWARGS = dict(
528
+ learning_rate = 1e-3,
529
+ learned_noise_scale = False,
530
+ noise_scale_learning_rate = 1e-5,
531
+ noise_scale_clamp_range = (1e-3, 1e-1),
532
+ use_optimizer = True,
533
+ optimizer_klass = partial(SGD, nesterov = True, momentum = 0.1, weight_decay = 1e-2),
534
+ optimizer_kwargs = dict(),
535
+ use_sigma_optimizer = True,
536
+ sigma_optimizer_klass = partial(SGD, nesterov = True, momentum = 0.1),
537
+ sigma_optimizer_kwargs = dict(),
538
+ use_scheduler = False,
539
+ scheduler_klass = None,
540
+ scheduler_kwargs = dict(),
541
+ use_sigma_scheduler = False,
542
+ sigma_scheduler_klass = None,
543
+ sigma_scheduler_kwargs = dict(),
544
+ fitness_to_weighted_factor = 'normalize'
545
+ )
546
+
547
+ # registry of update rules, extend as new algorithms are added
548
+
549
+ UPDATE_RULES = dict(
550
+ es = PlainES,
551
+ plain_es = PlainES,
552
+ sep_cma_es = SepCMAES,
553
+ sep_cma = SepCMAES
554
+ )
@@ -72,6 +72,8 @@ FITNESS_WEIGHT_FACTORS = dict(
72
72
 
73
73
  FitnessWeightFactorType = Literal['normalize', 'centered_rank']
74
74
 
75
+ ParametersToOptimize = list[str] | Module | list[Module] | list[Parameter]
76
+
75
77
  # class
76
78
 
77
79
  class EvoStrategy(Module):
@@ -86,7 +88,7 @@ class EvoStrategy(Module):
86
88
  noise_population_size = 30,
87
89
  learning_rate = 1e-3,
88
90
  mirror_sampling = True,
89
- params_to_optimize: list[str] | Module | list[Module] | list[Parameter] | None = None,
91
+ params_to_optimize: dict[str, ParametersToOptimize] | ParametersToOptimize | None = None,
90
92
  noise_low_rank: int | None = None,
91
93
  rollout_fixed_seed = True,
92
94
  noise_scale = 1e-2, # the noise scaling during rollouts with environment, todo - figure out right value and make sure it can also be customized per parameter name through a dict
@@ -115,6 +117,7 @@ class EvoStrategy(Module):
115
117
  accelerate_kwargs: dict = dict(),
116
118
  reject_generation_fitnesses_if: Callable[[Tensor], bool] | None = None,
117
119
  on_result: Callable | None = None,
120
+ on_generation: Callable | None = None,
118
121
  vectorized = False,
119
122
  vector_size: int | None = None,
120
123
  sync_on_init = True
@@ -125,6 +128,7 @@ class EvoStrategy(Module):
125
128
  self.vectorized = vectorized
126
129
  self.vector_size = vector_size
127
130
  self.on_result = on_result
131
+ self.on_generation = on_generation
128
132
 
129
133
  if not exists(accelerator):
130
134
  accelerator = Accelerator(cpu = cpu, **accelerate_kwargs)
@@ -136,11 +140,12 @@ class EvoStrategy(Module):
136
140
  self.environment = environment
137
141
 
138
142
  if accelerator.is_main_process:
139
- if hasattr(environment, 'pre_main_callback') and callable(environment.pre_main_callback):
143
+ pre_main_callback = getattr(environment, 'pre_main_callback', None)
144
+ if callable(pre_main_callback):
140
145
  self.print('pre_main_callback detected on environment passed in and is invoked')
141
- environment.pre_main_callback()
146
+ pre_main_callback()
142
147
 
143
- accelerator.wait_for_everyone()
148
+ self.wait_for_everyone()
144
149
 
145
150
  # take care of model and parameters
146
151
 
@@ -160,24 +165,31 @@ class EvoStrategy(Module):
160
165
  named_parameters_dict = dict(model.named_parameters())
161
166
 
162
167
  param_to_name_index = {param: name for name, param in named_parameters_dict.items()}
168
+ self._param_to_name_index = param_to_name_index
163
169
 
164
170
  param_names = set(named_parameters_dict.keys())
165
171
 
166
172
  # default to all parameters to optimize with evo strategy
167
173
 
168
- params_to_optimize = default(params_to_optimize, param_names)
174
+ self.scopes = None
169
175
 
170
- # if Modules given, convert to Parameters
171
- # then convert Parameters to names
176
+ if isinstance(params_to_optimize, dict):
177
+ self.scopes = dict()
178
+ all_scoped_params = set()
172
179
 
173
- if isinstance(params_to_optimize, Module):
174
- params_to_optimize = list(params_to_optimize.parameters())
180
+ for scope_name, scope_params in params_to_optimize.items():
181
+ resolved_scope_params = self._resolve_params_to_names(scope_params)
182
+ assert all([name in param_names for name in resolved_scope_params]), f'some parameters in scope {scope_name} are not found in the model'
183
+ self.scopes[scope_name] = resolved_scope_params
184
+ all_scoped_params |= resolved_scope_params
175
185
 
176
- if is_bearable(params_to_optimize, list[Module]):
177
- params_to_optimize = list(ModuleList(params_to_optimize).parameters())
186
+ params_to_optimize = list(all_scoped_params)
187
+
188
+ params_to_optimize = default(params_to_optimize, param_names)
178
189
 
179
- if is_bearable(params_to_optimize, list[Parameter]):
180
- params_to_optimize = [param_to_name_index[param] for param in set(params_to_optimize)]
190
+ # convert to names using the helper method
191
+
192
+ params_to_optimize = list(self._resolve_params_to_names(params_to_optimize))
181
193
 
182
194
  # validate
183
195
 
@@ -280,9 +292,19 @@ class EvoStrategy(Module):
280
292
  def device(self):
281
293
  return self.accelerate.device
282
294
 
295
+ @property
296
+ def is_distributed(self):
297
+ return self.accelerate.num_processes > 1
298
+
299
+ def wait_for_everyone(self):
300
+ if not (self.is_distributed and dist.is_available() and dist.is_initialized()):
301
+ return
302
+
303
+ dist.barrier()
304
+
283
305
  @torch.no_grad()
284
306
  def sync_model_params_and_buffers_(self):
285
- if not self.accelerate.num_processes > 1:
307
+ if not (self.is_distributed and dist.is_available() and dist.is_initialized()):
286
308
  return
287
309
 
288
310
  for param in self.model.parameters():
@@ -291,6 +313,21 @@ class EvoStrategy(Module):
291
313
  for buffer in self.model.buffers():
292
314
  dist.broadcast(buffer, src = 0)
293
315
 
316
+ def _resolve_params_to_names(
317
+ self,
318
+ params: ParametersToOptimize
319
+ ):
320
+
321
+ if isinstance(params, Module):
322
+ params = list(params.parameters())
323
+ elif is_bearable(params, list[Module]):
324
+ params = list(ModuleList(params).parameters())
325
+
326
+ if is_bearable(params, list[Parameter]):
327
+ params = [self._param_to_name_index[param] for param in set(params)]
328
+
329
+ return set(params)
330
+
294
331
  def print(self, *args, **kwargs):
295
332
  if not self.verbose:
296
333
  return
@@ -308,7 +345,8 @@ class EvoStrategy(Module):
308
345
  def evolve_(
309
346
  self,
310
347
  fitnesses: list[float] | Tensor,
311
- seeds_for_population: list[int] | Tensor
348
+ seeds_for_population: list[int] | Tensor,
349
+ scoped_names: set[str] | None = None
312
350
  ):
313
351
  use_optimizer = self.use_optimizer
314
352
  model = self.noisable_model
@@ -349,6 +387,9 @@ class EvoStrategy(Module):
349
387
 
350
388
  noise_config = dict(zip(self.param_names_to_optimize, individual_param_seeds.tolist()))
351
389
 
390
+ if exists(scoped_names):
391
+ noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
392
+
352
393
  noise_config_with_weight = dict()
353
394
 
354
395
  # loop through each noise
@@ -418,7 +459,7 @@ class EvoStrategy(Module):
418
459
  filepath = self.checkpoint_folder / f'{filename}.pt'
419
460
  torch.save(self.model.state_dict(), str(filepath))
420
461
 
421
- self.accelerate.wait_for_everyone()
462
+ self.wait_for_everyone()
422
463
 
423
464
  @torch.inference_mode()
424
465
  def forward(
@@ -427,10 +468,21 @@ class EvoStrategy(Module):
427
468
  num_generations = None,
428
469
  disable_distributed = False,
429
470
  rollback_model_at_end = False,
430
- verbose = None
471
+ verbose = None,
472
+ scope: str | None = None,
473
+ params_to_optimize: ParametersToOptimize | None = None
431
474
  ):
432
475
  verbose = default(verbose, self.verbose)
433
476
 
477
+ scoped_names = None
478
+
479
+ if exists(scope):
480
+ assert exists(self.scopes) and scope in self.scopes, f'scope "{scope}" not found in passed parameter dictionary'
481
+ scoped_names = self.scopes[scope]
482
+
483
+ if exists(params_to_optimize):
484
+ scoped_names = self._resolve_params_to_names(params_to_optimize)
485
+
434
486
  model = self.noisable_model.to(self.device)
435
487
 
436
488
  # maybe save model for rolling back (for meta-evo)
@@ -494,17 +546,6 @@ class EvoStrategy(Module):
494
546
 
495
547
  fitnesses = []
496
548
 
497
- # function for fitness
498
-
499
- def rollout_for_fitness():
500
- fitness = self.environment(model)
501
-
502
- if is_tensor(fitness):
503
- assert fitness.numel() == 1
504
- fitness = fitness.item()
505
-
506
- return fitness
507
-
508
549
  # seeds
509
550
 
510
551
  maybe_rollout_seed = maybe_get_synced_seed() if self.rollout_fixed_seed else None
@@ -525,6 +566,9 @@ class EvoStrategy(Module):
525
566
  individual_param_seeds = with_seed(individual_seed.item())(randint)(0, MAX_SEED_VALUE, (self.num_params,))
526
567
  noise_config = dict(zip(self.param_names_to_optimize, individual_param_seeds.tolist()))
527
568
 
569
+ if exists(scoped_names):
570
+ noise_config = {k: v for k, v in noise_config.items() if k in scoped_names}
571
+
528
572
  noise_config_with_scale = dict()
529
573
  for param_name, seed in noise_config.items():
530
574
  noise_scale = self._get_noise_scale(param_name)
@@ -590,7 +634,12 @@ class EvoStrategy(Module):
590
634
 
591
635
  # pass fitnesses to evolve function
592
636
 
593
- self.evolve_(fitnesses, seeds_for_population)
637
+ self.evolve_(fitnesses, seeds_for_population, scoped_names=scoped_names)
638
+
639
+ # on generation callback
640
+
641
+ if exists(self.on_generation) and self.accelerate.is_main_process:
642
+ self.on_generation(generation, fitnesses, self.model)
594
643
 
595
644
  # log
596
645
 
@@ -609,7 +658,7 @@ class EvoStrategy(Module):
609
658
  exists(self.checkpoint_every) and
610
659
  divisible_by(generation, self.checkpoint_every)
611
660
  ):
612
- self.checkpoint(f'{filename}.{generation}.pt')
661
+ self.checkpoint(f'{filename}.{generation}')
613
662
 
614
663
  # increment generation
615
664
 
@@ -631,4 +680,7 @@ class EvoStrategy(Module):
631
680
  # return fitnesses across generations
632
681
  # for meta-evolutionary (nesting EvoStrategy within the environment of another and optimizing some meta-network)
633
682
 
683
+ if len(fitnesses_across_generations) == 0:
684
+ return tensor([], device = self.device)
685
+
634
686
  return stack(fitnesses_across_generations)
File without changes