evolutionary-policy-optimization 0.0.40__tar.gz → 0.0.42__tar.gz

{evolutionary_policy_optimization-0.0.40 → evolutionary_policy_optimization-0.0.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evolutionary-policy-optimization
-Version: 0.0.40
+Version: 0.0.42
 Summary: EPO - Pytorch
 Project-URL: Homepage, https://pypi.org/project/evolutionary-policy-optimization/
 Project-URL: Repository, https://github.com/lucidrains/evolutionary-policy-optimization
@@ -162,4 +162,25 @@ agent.load('./agent.pt')
 }
 ```
 
+```bibtex
+@inproceedings{Khadka2018EvolutionGuidedPG,
+    title   = {Evolution-Guided Policy Gradient in Reinforcement Learning},
+    author  = {Shauharda Khadka and Kagan Tumer},
+    booktitle = {Neural Information Processing Systems},
+    year    = {2018},
+    url     = {https://api.semanticscholar.org/CorpusID:53096951}
+}
+```
+
+```bibtex
+@article{Fortunato2017NoisyNF,
+    title   = {Noisy Networks for Exploration},
+    author  = {Meire Fortunato and Mohammad Gheshlaghi Azar and Bilal Piot and Jacob Menick and Ian Osband and Alex Graves and Vlad Mnih and R{\'e}mi Munos and Demis Hassabis and Olivier Pietquin and Charles Blundell and Shane Legg},
+    journal = {ArXiv},
+    year    = {2017},
+    volume  = {abs/1706.10295},
+    url     = {https://api.semanticscholar.org/CorpusID:5176587}
+}
+```
+
 *Evolution is cleverer than you are.* - Leslie Orgel

{evolutionary_policy_optimization-0.0.40 → evolutionary_policy_optimization-0.0.42}/README.md

@@ -110,4 +110,25 @@ agent.load('./agent.pt')
 }
 ```
 
+```bibtex
+@inproceedings{Khadka2018EvolutionGuidedPG,
+    title   = {Evolution-Guided Policy Gradient in Reinforcement Learning},
+    author  = {Shauharda Khadka and Kagan Tumer},
+    booktitle = {Neural Information Processing Systems},
+    year    = {2018},
+    url     = {https://api.semanticscholar.org/CorpusID:53096951}
+}
+```
+
+```bibtex
+@article{Fortunato2017NoisyNF,
+    title   = {Noisy Networks for Exploration},
+    author  = {Meire Fortunato and Mohammad Gheshlaghi Azar and Bilal Piot and Jacob Menick and Ian Osband and Alex Graves and Vlad Mnih and R{\'e}mi Munos and Demis Hassabis and Olivier Pietquin and Charles Blundell and Shane Legg},
+    journal = {ArXiv},
+    year    = {2017},
+    volume  = {abs/1706.10295},
+    url     = {https://api.semanticscholar.org/CorpusID:5176587}
+}
+```
+
 *Evolution is cleverer than you are.* - Leslie Orgel

{evolutionary_policy_optimization-0.0.40 → evolutionary_policy_optimization-0.0.42}/evolutionary_policy_optimization/epo.py

@@ -798,6 +798,10 @@ class Agent(Module):
 
                 latents = self.latent_gene_pool(latent_id = latent_gene_ids)
 
+                orig_latents = latents
+                latents = latents.detach()
+                latents.requires_grad_()
+
                 # learn actor
 
                 logits = self.actor(states, latents)
@@ -822,6 +826,14 @@ class Agent(Module):
                 self.critic_optim.step()
                 self.critic_optim.zero_grad()
 
+                # maybe update latents, if not frozen
+
+                if not self.latent_gene_pool.frozen_latents:
+                    orig_latents.backward(latents.grad)
+
+                    self.latent_optim.step()
+                    self.latent_optim.zero_grad()
+
         # apply evolution
 
         self.latent_gene_pool.genetic_algorithm_step(fitness_scores)
@@ -863,25 +875,32 @@ def create_agent(
     actor_num_actions,
     actor_dim_hiddens: int | tuple[int, ...],
     critic_dim_hiddens: int | tuple[int, ...],
+    latent_gene_pool_kwargs: dict = dict(),
+    actor_kwargs: dict = dict(),
+    critic_kwargs: dict = dict(),
 ) -> Agent:
 
     latent_gene_pool = LatentGenePool(
         num_latents = num_latents,
-        dim_latent = dim_latent
+        dim_latent = dim_latent,
+        **latent_gene_pool_kwargs
     )
 
     actor = Actor(
         num_actions = actor_num_actions,
         dim_state = dim_state,
         dim_latent = dim_latent,
-        dim_hiddens = actor_dim_hiddens
+        dim_hiddens = actor_dim_hiddens,
+        **actor_kwargs
     )
 
     critic = Critic(
         dim_state = dim_state,
         dim_latent = dim_latent,
-        dim_hiddens = critic_dim_hiddens
-    )                
+        dim_hiddens = critic_dim_hiddens,
+        **critic_kwargs
+    )
+
     return Agent(actor = actor, critic = critic, latent_gene_pool = latent_gene_pool)
 
 # EPO - which is just PPO with natural selection of a population of latent variables conditioning the agent
@@ -951,7 +970,7 @@ class EPO(Module):
 
                 done = tensor(False)
 
-                while time < self.max_episode_length:
+                while time < self.max_episode_length and not done:
 
                     # sample action
 

{evolutionary_policy_optimization-0.0.40 → evolutionary_policy_optimization-0.0.42}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evolutionary-policy-optimization"
-version = "0.0.40"
+version = "0.0.42"
 description = "EPO - Pytorch"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{evolutionary_policy_optimization-0.0.40 → evolutionary_policy_optimization-0.0.42}/tests/test_epo.py

@@ -73,7 +73,10 @@ def test_create_agent(
     agent.save('./agent.pt', overwrite = True)
     agent.load('./agent.pt')
 
-def test_e2e_with_mock_env():
+@pytest.mark.parametrize('frozen_latents', (False, True))
+def test_e2e_with_mock_env(
+    frozen_latents
+):
     from evolutionary_policy_optimization import create_agent, EPO, Env
 
     agent = create_agent(
@@ -82,7 +85,10 @@ def test_e2e_with_mock_env():
         dim_latent = 32,
         actor_num_actions = 5,
         actor_dim_hiddens = (256, 128),
-        critic_dim_hiddens = (256, 128, 64)
+        critic_dim_hiddens = (256, 128, 64),
+        latent_gene_pool_kwargs = dict(
+            frozen_latents = frozen_latents
+        )
     )
 
     epo = EPO(