x-evolution 0.1.26__tar.gz → 0.1.29__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {x_evolution-0.1.26 → x_evolution-0.1.29}/PKG-INFO +1 -1
- {x_evolution-0.1.26 → x_evolution-0.1.29}/pyproject.toml +1 -1
- x_evolution-0.1.29/train_ant_maze.py +211 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/x_evolution/x_evolution.py +28 -7
- {x_evolution-0.1.26 → x_evolution-0.1.29}/.github/workflows/python-publish.yml +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/.github/workflows/test.yml +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/.gitignore +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/LICENSE +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/README.md +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/scripts/install-humanoid.sh +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/tests/test_evolution.py +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/train_humanoid.py +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/train_lunar.py +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/train_mnist.py +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/train_xor.py +0 -0
- {x_evolution-0.1.26 → x_evolution-0.1.29}/x_evolution/__init__.py +0 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# dependencies = [
|
|
3
|
+
# "fire",
|
|
4
|
+
# "gymnasium[mujoco]>=1.0.0",
|
|
5
|
+
# "gymnasium-robotics",
|
|
6
|
+
# "moviepy",
|
|
7
|
+
# "tqdm",
|
|
8
|
+
# "x-evolution>=0.1.27",
|
|
9
|
+
# "x-mlps-pytorch"
|
|
10
|
+
# ]
|
|
11
|
+
# ///
|
|
12
|
+
|
|
13
|
+
import fire
|
|
14
|
+
from shutil import rmtree
|
|
15
|
+
import gymnasium as gym
|
|
16
|
+
import gymnasium_robotics
|
|
17
|
+
import numpy as np
|
|
18
|
+
from tqdm import tqdm
|
|
19
|
+
|
|
20
|
+
import torch
|
|
21
|
+
from torch import nn
|
|
22
|
+
from torch.nn import Module, GRU, Linear
|
|
23
|
+
import torch.nn.functional as F
|
|
24
|
+
|
|
25
|
+
# functions
|
|
26
|
+
|
|
27
|
+
def exists(v):
|
|
28
|
+
return v is not None
|
|
29
|
+
|
|
30
|
+
def softclamp(t, value):
|
|
31
|
+
return (t / value).tanh() * value
|
|
32
|
+
|
|
33
|
+
class AntMazeEnvironment(Module):
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
env_id = 'AntMaze_UMazeDense-v5',
|
|
37
|
+
video_folder = './recordings_ant_maze',
|
|
38
|
+
render_every_eps = 100,
|
|
39
|
+
max_steps = 1000,
|
|
40
|
+
repeats = 1,
|
|
41
|
+
verbose = False
|
|
42
|
+
):
|
|
43
|
+
super().__init__()
|
|
44
|
+
|
|
45
|
+
self.verbose = verbose
|
|
46
|
+
|
|
47
|
+
env = gym.make(env_id, render_mode = 'rgb_array')
|
|
48
|
+
|
|
49
|
+
self.env = env
|
|
50
|
+
self.max_steps = max_steps
|
|
51
|
+
self.repeats = repeats
|
|
52
|
+
self.video_folder = video_folder
|
|
53
|
+
self.render_every_eps = render_every_eps
|
|
54
|
+
|
|
55
|
+
def pre_main_callback(self):
|
|
56
|
+
rmtree(self.video_folder, ignore_errors = True)
|
|
57
|
+
|
|
58
|
+
self.env = gym.wrappers.RecordVideo(
|
|
59
|
+
env = self.env,
|
|
60
|
+
video_folder = self.video_folder,
|
|
61
|
+
name_prefix = 'recording',
|
|
62
|
+
episode_trigger = lambda eps_num: (eps_num % self.render_every_eps) == 0,
|
|
63
|
+
disable_logger = True
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def forward(self, model):
|
|
67
|
+
|
|
68
|
+
device = next(model.parameters()).device
|
|
69
|
+
|
|
70
|
+
seed = torch.randint(0, int(1e6), ())
|
|
71
|
+
|
|
72
|
+
num_envs = 1
|
|
73
|
+
cum_reward = torch.zeros(num_envs, device = device)
|
|
74
|
+
|
|
75
|
+
for _ in range(self.repeats):
|
|
76
|
+
obs, _ = self.env.reset(seed = seed.item())
|
|
77
|
+
|
|
78
|
+
step = 0
|
|
79
|
+
hiddens = None
|
|
80
|
+
|
|
81
|
+
dones = torch.zeros(num_envs, device = device, dtype = torch.bool)
|
|
82
|
+
|
|
83
|
+
while step < self.max_steps and not dones.all():
|
|
84
|
+
|
|
85
|
+
state = obs['observation']
|
|
86
|
+
goal = obs['desired_goal']
|
|
87
|
+
|
|
88
|
+
state_torch = torch.from_numpy(state).float().to(device)
|
|
89
|
+
goal_torch = torch.from_numpy(goal).float().to(device)
|
|
90
|
+
|
|
91
|
+
state_goal = torch.cat((state_torch, goal_torch), dim = -1)
|
|
92
|
+
|
|
93
|
+
action_logits, hiddens = model(state_goal, hiddens)
|
|
94
|
+
|
|
95
|
+
mean, log_var = action_logits.chunk(2, dim = -1)
|
|
96
|
+
|
|
97
|
+
std = (0.5 * softclamp(log_var, 5.)).exp()
|
|
98
|
+
sampled = mean + torch.randn_like(mean) * std
|
|
99
|
+
action = sampled.tanh()
|
|
100
|
+
|
|
101
|
+
next_obs, reward, truncated, terminated, info = self.env.step(action.detach().cpu().numpy())
|
|
102
|
+
|
|
103
|
+
reward_np = np.array(reward) if not isinstance(reward, np.ndarray) else reward
|
|
104
|
+
total_reward_base = torch.from_numpy(reward_np).float().to(device)
|
|
105
|
+
|
|
106
|
+
exploration_bonus = std.mean(dim = -1) * 0.01
|
|
107
|
+
penalize_extreme_actions = (mean.abs() > 1.).float().mean(dim = -1) * 0.01
|
|
108
|
+
|
|
109
|
+
total_reward = total_reward_base + exploration_bonus - penalize_extreme_actions
|
|
110
|
+
|
|
111
|
+
mask = (~dones).float()
|
|
112
|
+
cum_reward += total_reward * mask
|
|
113
|
+
|
|
114
|
+
dones_np = np.array(truncated | terminated) if not isinstance(truncated | terminated, np.ndarray) else (truncated | terminated)
|
|
115
|
+
dones |= torch.from_numpy(dones_np).to(device)
|
|
116
|
+
|
|
117
|
+
step += 1
|
|
118
|
+
obs = next_obs
|
|
119
|
+
|
|
120
|
+
return cum_reward.item() / self.repeats
|
|
121
|
+
|
|
122
|
+
# model
|
|
123
|
+
|
|
124
|
+
from x_mlps_pytorch.residual_normed_mlp import ResidualNormedMLP
|
|
125
|
+
|
|
126
|
+
class Model(Module):
|
|
127
|
+
|
|
128
|
+
def __init__(self, state_dim = 27 + 2, action_dim = 8):
|
|
129
|
+
super().__init__()
|
|
130
|
+
|
|
131
|
+
self.deep_mlp = ResidualNormedMLP(
|
|
132
|
+
dim_in = state_dim,
|
|
133
|
+
dim = 256,
|
|
134
|
+
depth = 8,
|
|
135
|
+
residual_every = 2
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
self.gru = GRU(256, 256, batch_first = True)
|
|
139
|
+
|
|
140
|
+
self.to_pred = Linear(256, action_dim * 2, bias = False)
|
|
141
|
+
|
|
142
|
+
def forward(self, state, hiddens = None):
|
|
143
|
+
|
|
144
|
+
x = self.deep_mlp(state)
|
|
145
|
+
|
|
146
|
+
is_batched = x.ndim > 1
|
|
147
|
+
|
|
148
|
+
if not is_batched:
|
|
149
|
+
x = x.unsqueeze(0)
|
|
150
|
+
|
|
151
|
+
x = x.unsqueeze(-2)
|
|
152
|
+
gru_out, hiddens = self.gru(x, hiddens)
|
|
153
|
+
x = x + gru_out
|
|
154
|
+
x = x.squeeze(-2)
|
|
155
|
+
|
|
156
|
+
if not is_batched:
|
|
157
|
+
x = x.squeeze(0)
|
|
158
|
+
|
|
159
|
+
return self.to_pred(x), hiddens
|
|
160
|
+
|
|
161
|
+
# evo strategy
|
|
162
|
+
|
|
163
|
+
from x_evolution import EvoStrategy
|
|
164
|
+
from torch.optim.lr_scheduler import CosineAnnealingLR
|
|
165
|
+
|
|
166
|
+
def main(
|
|
167
|
+
env_id = 'AntMaze_UMazeDense-v5',
|
|
168
|
+
num_generations = 50_000,
|
|
169
|
+
max_steps = 250,
|
|
170
|
+
noise_population_size = 64,
|
|
171
|
+
verbose = True
|
|
172
|
+
):
|
|
173
|
+
# Determine dims from temp env
|
|
174
|
+
temp_env = gym.make(env_id)
|
|
175
|
+
obs_space = temp_env.observation_space
|
|
176
|
+
action_space = temp_env.action_space
|
|
177
|
+
|
|
178
|
+
state_dim = obs_space['observation'].shape[0] + obs_space['desired_goal'].shape[0]
|
|
179
|
+
action_dim = action_space.shape[0]
|
|
180
|
+
temp_env.close()
|
|
181
|
+
|
|
182
|
+
model = Model(state_dim = state_dim, action_dim = action_dim)
|
|
183
|
+
|
|
184
|
+
evo_strat = EvoStrategy(
|
|
185
|
+
model,
|
|
186
|
+
environment = AntMazeEnvironment(
|
|
187
|
+
env_id = env_id,
|
|
188
|
+
max_steps = max_steps,
|
|
189
|
+
repeats = 1,
|
|
190
|
+
render_every_eps = 200,
|
|
191
|
+
verbose = verbose
|
|
192
|
+
),
|
|
193
|
+
num_generations = num_generations,
|
|
194
|
+
noise_population_size = noise_population_size,
|
|
195
|
+
noise_low_rank = 1,
|
|
196
|
+
noise_scale = 1e-2,
|
|
197
|
+
noise_scale_clamp_range = (5e-3, 2e-2),
|
|
198
|
+
learned_noise_scale = True,
|
|
199
|
+
use_sigma_optimizer = True,
|
|
200
|
+
learning_rate = 1e-3,
|
|
201
|
+
noise_scale_learning_rate = 1e-4,
|
|
202
|
+
use_scheduler = True,
|
|
203
|
+
scheduler_klass = CosineAnnealingLR,
|
|
204
|
+
scheduler_kwargs = dict(T_max = num_generations),
|
|
205
|
+
verbose = verbose
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
evo_strat()
|
|
209
|
+
|
|
210
|
+
if __name__ == '__main__':
|
|
211
|
+
fire.Fire(main)
|
|
@@ -11,8 +11,11 @@ from torch.nn import Module, ModuleList, Parameter, ParameterList
|
|
|
11
11
|
from torch.optim import SGD, Adam, Optimizer
|
|
12
12
|
from torch.optim.lr_scheduler import LRScheduler
|
|
13
13
|
|
|
14
|
+
import torch.distributed as dist
|
|
14
15
|
import torch.nn.functional as F
|
|
15
16
|
|
|
17
|
+
from tqdm import tqdm
|
|
18
|
+
|
|
16
19
|
from beartype import beartype
|
|
17
20
|
from beartype.door import is_bearable
|
|
18
21
|
|
|
@@ -95,7 +98,8 @@ class EvoStrategy(Module):
|
|
|
95
98
|
accelerate_kwargs: dict = dict(),
|
|
96
99
|
reject_generation_fitnesses_if: Callable[[Tensor], bool] | None = None,
|
|
97
100
|
vectorized = False,
|
|
98
|
-
vector_size: int | None = None
|
|
101
|
+
vector_size: int | None = None,
|
|
102
|
+
sync_on_init = True
|
|
99
103
|
):
|
|
100
104
|
super().__init__()
|
|
101
105
|
self.verbose = verbose
|
|
@@ -127,12 +131,10 @@ class EvoStrategy(Module):
|
|
|
127
131
|
self.model = model
|
|
128
132
|
self.noisable_model = Noisable(model, low_rank = noise_low_rank)
|
|
129
133
|
|
|
130
|
-
#
|
|
131
|
-
|
|
132
|
-
wrapped_model = accelerator.prepare(model)
|
|
134
|
+
# maybe sync model params and buffers
|
|
133
135
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
+
if sync_on_init:
|
|
137
|
+
self.sync_model_params_and_buffers_()
|
|
136
138
|
|
|
137
139
|
# get param dictionary
|
|
138
140
|
|
|
@@ -254,6 +256,17 @@ class EvoStrategy(Module):
|
|
|
254
256
|
def device(self):
|
|
255
257
|
return self.accelerate.device
|
|
256
258
|
|
|
259
|
+
@torch.no_grad()
|
|
260
|
+
def sync_model_params_and_buffers_(self):
|
|
261
|
+
if not self.accelerate.num_processes > 1:
|
|
262
|
+
return
|
|
263
|
+
|
|
264
|
+
for param in self.model.parameters():
|
|
265
|
+
dist.broadcast(param, src = 0)
|
|
266
|
+
|
|
267
|
+
for buffer in self.model.buffers():
|
|
268
|
+
dist.broadcast(buffer, src = 0)
|
|
269
|
+
|
|
257
270
|
def print(self, *args, **kwargs):
|
|
258
271
|
if not self.verbose:
|
|
259
272
|
return
|
|
@@ -474,7 +487,11 @@ class EvoStrategy(Module):
|
|
|
474
487
|
|
|
475
488
|
# now loop through the entire population of noise
|
|
476
489
|
|
|
477
|
-
|
|
490
|
+
is_main = self.accelerate.is_main_process
|
|
491
|
+
|
|
492
|
+
pbar = tqdm(zip(noise_indices, seeds_for_machine), total = len(seeds_for_machine), desc = f'Generation {generation}', disable = not (self.verbose and is_main))
|
|
493
|
+
|
|
494
|
+
for noise_index, individual_seed in pbar:
|
|
478
495
|
|
|
479
496
|
if noise_index >= pop_size:
|
|
480
497
|
fitnesses.append([0., 0.] if self.mirror_sampling else 0.)
|
|
@@ -505,6 +522,7 @@ class EvoStrategy(Module):
|
|
|
505
522
|
|
|
506
523
|
if not self.mirror_sampling:
|
|
507
524
|
fitnesses.append(fitness)
|
|
525
|
+
pbar.set_postfix(reward = f'{fitness:.3f}')
|
|
508
526
|
continue
|
|
509
527
|
|
|
510
528
|
# handle mirror sampling
|
|
@@ -512,6 +530,9 @@ class EvoStrategy(Module):
|
|
|
512
530
|
fitness_mirrored = get_fitness(negate = True)
|
|
513
531
|
|
|
514
532
|
fitnesses.append([fitness, fitness_mirrored])
|
|
533
|
+
pbar.set_postfix(reward = f'{(fitness + fitness_mirrored) / 2:.3f}')
|
|
534
|
+
|
|
535
|
+
pbar.close()
|
|
515
536
|
|
|
516
537
|
# normalize the fitness and weighted sum of all the noise is the update
|
|
517
538
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|