evolutionary-policy-optimization 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

evolutionary_policy_optimization/epo.py

@@ -364,11 +364,38 @@ class Critic(Module):
             hl_gauss_loss = hl_gauss_loss_kwargs
         )
 
+        self.use_regression = use_regression
+
+        hl_gauss_loss = self.to_pred.hl_gauss_loss
+
+        self.maybe_bins_to_value = hl_gauss_loss if not use_regression else identity
+        self.maybe_value_to_bins = hl_gauss_loss.transform_to_logprobs if not use_regression else identity
+        self.loss_fn = hl_gauss_loss if not use_regression else F.mse_loss
+
+    def forward_for_loss(
+        self,
+        state,
+        latent,
+        old_values,
+        target,
+        eps_clip = 0.4
+    ):
+        logits = self.forward(state, latent, return_logits = True)
+
+        value = self.maybe_bins_to_value(logits)
+
+        clipped_value = old_values + (value - old_values).clamp(1. - eps_clip, 1. + eps_clip)
+
+        loss = self.loss_fn(value, target, reduction = 'none')
+        clipped_loss = self.loss_fn(clipped_value, target, reduction = 'none')
+
+        return torch.max(loss, clipped_loss).mean()
+
     def forward(
         self,
         state,
         latent,
-        target = None
+        return_logits = False
     ):
 
         hidden = self.init_layer(state)
@@ -377,7 +404,8 @@ class Critic(Module):
 
         hidden = self.final_act(hidden)
 
-        return self.to_pred(hidden, target = target)
+        pred_kwargs = dict(return_logits = return_logits) if not self.use_regression else dict()
+        return self.to_pred(hidden, **pred_kwargs)
 
 # criteria for running genetic algorithm
 
@@ -740,6 +768,9 @@ class Agent(Module):
             entropy_weight = .01,
             norm_advantages = True
         ),
+        critic_loss_kwargs: dict = dict(
+            eps_clip = 0.4
+        ),
         ema_kwargs: dict = dict(),
         actor_optim_kwargs: dict = dict(),
         critic_optim_kwargs: dict = dict(),
@@ -778,9 +809,13 @@ class Agent(Module):
 
         # gae function
 
-        self.actor_loss = partial(actor_loss, **actor_loss_kwargs)
         self.calc_gae = partial(calc_generalized_advantage_estimate, **calc_gae_kwargs)
 
+        # actor critic loss related
+
+        self.actor_loss = partial(actor_loss, **actor_loss_kwargs)
+        self.critic_loss_kwargs = critic_loss_kwargs
+
         # fitness score related
 
         self.get_fitness_scores = get_fitness_scores
@@ -900,7 +935,6 @@ class Agent(Module):
         if not exists(latent) and exists(latent_id):
             latent = maybe_unwrap(self.latent_gene_pool)(latent_id = latent_id)
 
-        print(self.device, state.device, next(self.actor.parameters()).device)
         logits = maybe_unwrap(self.actor)(state, latent)
 
         if not sample:
@@ -1044,10 +1078,12 @@ class Agent(Module):
 
                 # learn critic with maybe classification loss
 
-                critic_loss = self.critic(
+                critic_loss = self.critic.forward_for_loss(
                     states,
                     latents,
-                    target = advantages + old_values
+                    old_values = old_values,
+                    target = advantages + old_values,
+                    **self.critic_loss_kwargs
                 )
 
                 critic_loss.backward()

{evolutionary_policy_optimization-0.1.4.dist-info → evolutionary_policy_optimization-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evolutionary-policy-optimization
-Version: 0.1.4
+Version: 0.1.6
 Summary: EPO - Pytorch
 Project-URL: Homepage, https://pypi.org/project/evolutionary-policy-optimization/
 Project-URL: Repository, https://github.com/lucidrains/evolutionary-policy-optimization
@@ -185,4 +185,15 @@ agent.load('./agent.pt')
 }
 ```
 
+```bibtex
+@article{Banerjee2022BoostingEI,
+    title   = {Boosting Exploration in Actor-Critic Algorithms by Incentivizing Plausible Novel States},
+    author  = {Chayan Banerjee and Zhiyong Chen and Nasimul Noman},
+    journal = {2023 62nd IEEE Conference on Decision and Control (CDC)},
+    year    = {2022},
+    pages   = {7009-7014},
+    url     = {https://api.semanticscholar.org/CorpusID:252682944}
+}
+```
+
 *Evolution is cleverer than you are.* - Leslie Orgel

{evolutionary_policy_optimization-0.1.4.dist-info → evolutionary_policy_optimization-0.1.6.dist-info}/RECORD

@@ -1,10 +1,10 @@
 evolutionary_policy_optimization/__init__.py,sha256=NyiYDYU7DlpmOTM7xiBQET3r1WwX0ebrgMCBLSQrW3c,288
 evolutionary_policy_optimization/distributed.py,sha256=7KgZdeS_wxBHo_du9XZFB1Cu318J-Bp66Xdr6Log_20,2423
 evolutionary_policy_optimization/env_wrappers.py,sha256=bDL06o9_b1iW6k3fw2xifnOnYlzs643tdW6Yv2gsIdw,803
-evolutionary_policy_optimization/epo.py,sha256=3PEmFE032fQz7mAfIoJzwmysbU6lnjCCngEMc9Jf46M,42180
+evolutionary_policy_optimization/epo.py,sha256=P-a8A1ky7FgpENUgb8VHk9qADwyQdzpUp40JoaSG2HY,43395
 evolutionary_policy_optimization/experimental.py,sha256=-IgqjJ_Wk_CMB1y9YYWpoYqTG9GZHAS6kbRdTluVevg,1563
 evolutionary_policy_optimization/mock_env.py,sha256=TLyyRm6tOD0Kdn9QqJJQriaSnsR-YmNQHo4OohmZFG4,1410
-evolutionary_policy_optimization-0.1.4.dist-info/METADATA,sha256=81Q8wK2N7zlOsSGuqcXrftpd1TIYkrplSPBuldMj8uU,6330
-evolutionary_policy_optimization-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-evolutionary_policy_optimization-0.1.4.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
-evolutionary_policy_optimization-0.1.4.dist-info/RECORD,,
+evolutionary_policy_optimization-0.1.6.dist-info/METADATA,sha256=Bc4MZKhe2H6q7H2mB4g2qObQYHjG-8ZJkcF1XKxgTgw,6742
+evolutionary_policy_optimization-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+evolutionary_policy_optimization-0.1.6.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
+evolutionary_policy_optimization-0.1.6.dist-info/RECORD,,