x-evolution 0.1.26__tar.gz → 0.1.29__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: x-evolution
3
- Version: 0.1.26
3
+ Version: 0.1.29
4
4
  Summary: x-evolution
5
5
  Project-URL: Homepage, https://pypi.org/project/x-evolution/
6
6
  Project-URL: Repository, https://github.com/lucidrains/x-evolution
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "x-evolution"
3
- version = "0.1.26"
3
+ version = "0.1.29"
4
4
  description = "x-evolution"
5
5
  authors = [
6
6
  { name = "Phil Wang", email = "lucidrains@gmail.com" }
@@ -0,0 +1,211 @@
1
+ # /// script
2
+ # dependencies = [
3
+ # "fire",
4
+ # "gymnasium[mujoco]>=1.0.0",
5
+ # "gymnasium-robotics",
6
+ # "moviepy",
7
+ # "tqdm",
8
+ # "x-evolution>=0.1.27",
9
+ # "x-mlps-pytorch"
10
+ # ]
11
+ # ///
12
+
13
+ import fire
14
+ from shutil import rmtree
15
+ import gymnasium as gym
16
+ import gymnasium_robotics
17
+ import numpy as np
18
+ from tqdm import tqdm
19
+
20
+ import torch
21
+ from torch import nn
22
+ from torch.nn import Module, GRU, Linear
23
+ import torch.nn.functional as F
24
+
25
+ # functions
26
+
27
+ def exists(v):
28
+ return v is not None
29
+
30
+ def softclamp(t, value):
31
+ return (t / value).tanh() * value
32
+
33
+ class AntMazeEnvironment(Module):
34
+ def __init__(
35
+ self,
36
+ env_id = 'AntMaze_UMazeDense-v5',
37
+ video_folder = './recordings_ant_maze',
38
+ render_every_eps = 100,
39
+ max_steps = 1000,
40
+ repeats = 1,
41
+ verbose = False
42
+ ):
43
+ super().__init__()
44
+
45
+ self.verbose = verbose
46
+
47
+ env = gym.make(env_id, render_mode = 'rgb_array')
48
+
49
+ self.env = env
50
+ self.max_steps = max_steps
51
+ self.repeats = repeats
52
+ self.video_folder = video_folder
53
+ self.render_every_eps = render_every_eps
54
+
55
+ def pre_main_callback(self):
56
+ rmtree(self.video_folder, ignore_errors = True)
57
+
58
+ self.env = gym.wrappers.RecordVideo(
59
+ env = self.env,
60
+ video_folder = self.video_folder,
61
+ name_prefix = 'recording',
62
+ episode_trigger = lambda eps_num: (eps_num % self.render_every_eps) == 0,
63
+ disable_logger = True
64
+ )
65
+
66
+ def forward(self, model):
67
+
68
+ device = next(model.parameters()).device
69
+
70
+ seed = torch.randint(0, int(1e6), ())
71
+
72
+ num_envs = 1
73
+ cum_reward = torch.zeros(num_envs, device = device)
74
+
75
+ for _ in range(self.repeats):
76
+ obs, _ = self.env.reset(seed = seed.item())
77
+
78
+ step = 0
79
+ hiddens = None
80
+
81
+ dones = torch.zeros(num_envs, device = device, dtype = torch.bool)
82
+
83
+ while step < self.max_steps and not dones.all():
84
+
85
+ state = obs['observation']
86
+ goal = obs['desired_goal']
87
+
88
+ state_torch = torch.from_numpy(state).float().to(device)
89
+ goal_torch = torch.from_numpy(goal).float().to(device)
90
+
91
+ state_goal = torch.cat((state_torch, goal_torch), dim = -1)
92
+
93
+ action_logits, hiddens = model(state_goal, hiddens)
94
+
95
+ mean, log_var = action_logits.chunk(2, dim = -1)
96
+
97
+ std = (0.5 * softclamp(log_var, 5.)).exp()
98
+ sampled = mean + torch.randn_like(mean) * std
99
+ action = sampled.tanh()
100
+
101
+ next_obs, reward, truncated, terminated, info = self.env.step(action.detach().cpu().numpy())
102
+
103
+ reward_np = np.array(reward) if not isinstance(reward, np.ndarray) else reward
104
+ total_reward_base = torch.from_numpy(reward_np).float().to(device)
105
+
106
+ exploration_bonus = std.mean(dim = -1) * 0.01
107
+ penalize_extreme_actions = (mean.abs() > 1.).float().mean(dim = -1) * 0.01
108
+
109
+ total_reward = total_reward_base + exploration_bonus - penalize_extreme_actions
110
+
111
+ mask = (~dones).float()
112
+ cum_reward += total_reward * mask
113
+
114
+ dones_np = np.array(truncated | terminated) if not isinstance(truncated | terminated, np.ndarray) else (truncated | terminated)
115
+ dones |= torch.from_numpy(dones_np).to(device)
116
+
117
+ step += 1
118
+ obs = next_obs
119
+
120
+ return cum_reward.item() / self.repeats
121
+
122
+ # model
123
+
124
+ from x_mlps_pytorch.residual_normed_mlp import ResidualNormedMLP
125
+
126
+ class Model(Module):
127
+
128
+ def __init__(self, state_dim = 27 + 2, action_dim = 8):
129
+ super().__init__()
130
+
131
+ self.deep_mlp = ResidualNormedMLP(
132
+ dim_in = state_dim,
133
+ dim = 256,
134
+ depth = 8,
135
+ residual_every = 2
136
+ )
137
+
138
+ self.gru = GRU(256, 256, batch_first = True)
139
+
140
+ self.to_pred = Linear(256, action_dim * 2, bias = False)
141
+
142
+ def forward(self, state, hiddens = None):
143
+
144
+ x = self.deep_mlp(state)
145
+
146
+ is_batched = x.ndim > 1
147
+
148
+ if not is_batched:
149
+ x = x.unsqueeze(0)
150
+
151
+ x = x.unsqueeze(-2)
152
+ gru_out, hiddens = self.gru(x, hiddens)
153
+ x = x + gru_out
154
+ x = x.squeeze(-2)
155
+
156
+ if not is_batched:
157
+ x = x.squeeze(0)
158
+
159
+ return self.to_pred(x), hiddens
160
+
161
+ # evo strategy
162
+
163
+ from x_evolution import EvoStrategy
164
+ from torch.optim.lr_scheduler import CosineAnnealingLR
165
+
166
+ def main(
167
+ env_id = 'AntMaze_UMazeDense-v5',
168
+ num_generations = 50_000,
169
+ max_steps = 250,
170
+ noise_population_size = 64,
171
+ verbose = True
172
+ ):
173
+ # Determine dims from temp env
174
+ temp_env = gym.make(env_id)
175
+ obs_space = temp_env.observation_space
176
+ action_space = temp_env.action_space
177
+
178
+ state_dim = obs_space['observation'].shape[0] + obs_space['desired_goal'].shape[0]
179
+ action_dim = action_space.shape[0]
180
+ temp_env.close()
181
+
182
+ model = Model(state_dim = state_dim, action_dim = action_dim)
183
+
184
+ evo_strat = EvoStrategy(
185
+ model,
186
+ environment = AntMazeEnvironment(
187
+ env_id = env_id,
188
+ max_steps = max_steps,
189
+ repeats = 1,
190
+ render_every_eps = 200,
191
+ verbose = verbose
192
+ ),
193
+ num_generations = num_generations,
194
+ noise_population_size = noise_population_size,
195
+ noise_low_rank = 1,
196
+ noise_scale = 1e-2,
197
+ noise_scale_clamp_range = (5e-3, 2e-2),
198
+ learned_noise_scale = True,
199
+ use_sigma_optimizer = True,
200
+ learning_rate = 1e-3,
201
+ noise_scale_learning_rate = 1e-4,
202
+ use_scheduler = True,
203
+ scheduler_klass = CosineAnnealingLR,
204
+ scheduler_kwargs = dict(T_max = num_generations),
205
+ verbose = verbose
206
+ )
207
+
208
+ evo_strat()
209
+
210
+ if __name__ == '__main__':
211
+ fire.Fire(main)
@@ -11,8 +11,11 @@ from torch.nn import Module, ModuleList, Parameter, ParameterList
11
11
  from torch.optim import SGD, Adam, Optimizer
12
12
  from torch.optim.lr_scheduler import LRScheduler
13
13
 
14
+ import torch.distributed as dist
14
15
  import torch.nn.functional as F
15
16
 
17
+ from tqdm import tqdm
18
+
16
19
  from beartype import beartype
17
20
  from beartype.door import is_bearable
18
21
 
@@ -95,7 +98,8 @@ class EvoStrategy(Module):
95
98
  accelerate_kwargs: dict = dict(),
96
99
  reject_generation_fitnesses_if: Callable[[Tensor], bool] | None = None,
97
100
  vectorized = False,
98
- vector_size: int | None = None
101
+ vector_size: int | None = None,
102
+ sync_on_init = True
99
103
  ):
100
104
  super().__init__()
101
105
  self.verbose = verbose
@@ -127,12 +131,10 @@ class EvoStrategy(Module):
127
131
  self.model = model
128
132
  self.noisable_model = Noisable(model, low_rank = noise_low_rank)
129
133
 
130
- # use prepare and run through environment once to sync params
131
-
132
- wrapped_model = accelerator.prepare(model)
134
+ # maybe sync model params and buffers
133
135
 
134
- with torch.no_grad():
135
- environment(wrapped_model)
136
+ if sync_on_init:
137
+ self.sync_model_params_and_buffers_()
136
138
 
137
139
  # get param dictionary
138
140
 
@@ -254,6 +256,17 @@ class EvoStrategy(Module):
254
256
  def device(self):
255
257
  return self.accelerate.device
256
258
 
259
+ @torch.no_grad()
260
+ def sync_model_params_and_buffers_(self):
261
+ if not self.accelerate.num_processes > 1:
262
+ return
263
+
264
+ for param in self.model.parameters():
265
+ dist.broadcast(param, src = 0)
266
+
267
+ for buffer in self.model.buffers():
268
+ dist.broadcast(buffer, src = 0)
269
+
257
270
  def print(self, *args, **kwargs):
258
271
  if not self.verbose:
259
272
  return
@@ -474,7 +487,11 @@ class EvoStrategy(Module):
474
487
 
475
488
  # now loop through the entire population of noise
476
489
 
477
- for noise_index, individual_seed in zip(noise_indices, seeds_for_machine):
490
+ is_main = self.accelerate.is_main_process
491
+
492
+ pbar = tqdm(zip(noise_indices, seeds_for_machine), total = len(seeds_for_machine), desc = f'Generation {generation}', disable = not (self.verbose and is_main))
493
+
494
+ for noise_index, individual_seed in pbar:
478
495
 
479
496
  if noise_index >= pop_size:
480
497
  fitnesses.append([0., 0.] if self.mirror_sampling else 0.)
@@ -505,6 +522,7 @@ class EvoStrategy(Module):
505
522
 
506
523
  if not self.mirror_sampling:
507
524
  fitnesses.append(fitness)
525
+ pbar.set_postfix(reward = f'{fitness:.3f}')
508
526
  continue
509
527
 
510
528
  # handle mirror sampling
@@ -512,6 +530,9 @@ class EvoStrategy(Module):
512
530
  fitness_mirrored = get_fitness(negate = True)
513
531
 
514
532
  fitnesses.append([fitness, fitness_mirrored])
533
+ pbar.set_postfix(reward = f'{(fitness + fitness_mirrored) / 2:.3f}')
534
+
535
+ pbar.close()
515
536
 
516
537
  # normalize the fitness and weighted sum of all the noise is the update
517
538
 
File without changes
File without changes
File without changes
File without changes