evolutionary-policy-optimization 0.2.14__py3-none-any.whl → 0.2.15__py3-none-any.whl

evolutionary_policy_optimization/epo.py

@@ -1063,6 +1063,7 @@ class Agent(Module):
         critic_loss_kwargs: dict = dict(
             eps_clip = 0.4
         ),
+        use_spo = False, # Simple Policy Optimization - Xie et al. https://arxiv.org/abs/2401.16025v9
         use_improved_critic_loss = True,
         shrink_and_perturb_every = None,
         shrink_and_perturb_kwargs: dict = dict(),
@@ -1126,6 +1127,7 @@ class Agent(Module):
         self.actor_loss = partial(actor_loss, **actor_loss_kwargs)
         self.critic_loss_kwargs = critic_loss_kwargs
 
+        self.use_spo = use_spo
         self.use_improved_critic_loss = use_improved_critic_loss
 
         # fitness score related
@@ -1399,7 +1401,7 @@ class Agent(Module):
 
                 logits = self.actor(states, latents)
 
-                actor_loss = self.actor_loss(logits, log_probs, actions, advantages)
+                actor_loss = self.actor_loss(logits, log_probs, actions, advantages, use_spo = self.use_spo)
 
                 actor_loss.backward()
 
@@ -1498,7 +1500,8 @@ def actor_loss(
     eps_clip = 0.2,
     entropy_weight = .01,
     eps = 1e-5,
-    norm_advantages = True
+    norm_advantages = True,
+    use_spo = False
 ):
     batch = logits.shape[0]
 
@@ -1506,14 +1509,22 @@ def actor_loss(
 
     ratio = (log_probs - old_log_probs).exp()
 
-    # classic clipped surrogate loss from ppo
-
-    clipped_ratio = ratio.clamp(min = 1. - eps_clip, max = 1. + eps_clip)
-
     if norm_advantages:
         advantages = F.layer_norm(advantages, (batch,), eps = eps)
 
-    actor_loss = -torch.min(clipped_ratio * advantages, ratio * advantages)
+    if use_spo:
+        # simple policy optimization - line 14 Algorithm 1 https://arxiv.org/abs/2401.16025v9
+
+        actor_loss = - (
+            ratio * advantages -
+            advantages.abs() / (2 * eps_clip) * (ratio - 1.).square()
+        )
+    else:
+        # classic clipped surrogate loss from ppo
+
+        clipped_ratio = ratio.clamp(min = 1. - eps_clip, max = 1. + eps_clip)
+
+        actor_loss = -torch.min(clipped_ratio * advantages, ratio * advantages)
 
     # add entropy loss for exploration
 

{evolutionary_policy_optimization-0.2.14.dist-info → evolutionary_policy_optimization-0.2.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evolutionary-policy-optimization
-Version: 0.2.14
+Version: 0.2.15
 Summary: EPO - Pytorch
 Project-URL: Homepage, https://pypi.org/project/evolutionary-policy-optimization/
 Project-URL: Repository, https://github.com/lucidrains/evolutionary-policy-optimization
@@ -300,4 +300,16 @@ That's it
 }
 ```
 
+```bibtex
+@misc{xie2025simplepolicyoptimization,
+    title   = {Simple Policy Optimization}, 
+    author  = {Zhengpeng Xie and Qiang Zhang and Fan Yang and Marco Hutter and Renjing Xu},
+    year    = {2025},
+    eprint  = {2401.16025},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG},
+    url = {https://arxiv.org/abs/2401.16025}, 
+}
+```
+
 *Evolution is cleverer than you are.* - Leslie Orgel

{evolutionary_policy_optimization-0.2.14.dist-info → evolutionary_policy_optimization-0.2.15.dist-info}/RECORD

@@ -1,10 +1,10 @@
 evolutionary_policy_optimization/__init__.py,sha256=NyiYDYU7DlpmOTM7xiBQET3r1WwX0ebrgMCBLSQrW3c,288
 evolutionary_policy_optimization/distributed.py,sha256=clN8Bvhb6JIQy2F8FTF312B0RN3iYoPkKjZreBtAwks,2366
 evolutionary_policy_optimization/env_wrappers.py,sha256=bDL06o9_b1iW6k3fw2xifnOnYlzs643tdW6Yv2gsIdw,803
-evolutionary_policy_optimization/epo.py,sha256=dp-SGhg9_NGO8Rezf8K2xlWYCpOXRKB2JaRy5UGP2uo,53142
+evolutionary_policy_optimization/epo.py,sha256=oA8Ft5VRq5XorCZa0Hrxh6rTDEQg7DzSVfR0MwPg5J4,53591
 evolutionary_policy_optimization/experimental.py,sha256=QZG0__wwFqHN_LJK7e-mHxlIL1mwjlvG6o6bcOpeAKg,6166
 evolutionary_policy_optimization/mock_env.py,sha256=TLyyRm6tOD0Kdn9QqJJQriaSnsR-YmNQHo4OohmZFG4,1410
-evolutionary_policy_optimization-0.2.14.dist-info/METADATA,sha256=opgQguJ4n6CgOYToRoNu9RTBJpM6oiamSOKJsHVV0FE,9972
-evolutionary_policy_optimization-0.2.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-evolutionary_policy_optimization-0.2.14.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
-evolutionary_policy_optimization-0.2.14.dist-info/RECORD,,
+evolutionary_policy_optimization-0.2.15.dist-info/METADATA,sha256=7w5tZm9yIeNTHm2d5npon5Nd5ekUpb9ahkf6VJkSZj0,10319
+evolutionary_policy_optimization-0.2.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+evolutionary_policy_optimization-0.2.15.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
+evolutionary_policy_optimization-0.2.15.dist-info/RECORD,,