rlgym-learn-algos 0.2.4__tar.gz → 0.2.5__tar.gz

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/Cargo.lock

@@ -229,7 +229,7 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
 [[package]]
 name = "rlgym-learn-algos"
-version = "0.2.4"
+version = "0.2.5"
 dependencies = [
  "itertools",
  "numpy",

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rlgym-learn-algos"
-version = "0.2.4"
+version = "0.2.5"
 edition = "2021"
 description = "Rust backend for the more expensive parts of the rlgym-learn-algos python module"
 license="Apache-2.0"

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rlgym-learn-algos
-Version: 0.2.4
+Version: 0.2.5
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Dist: pydantic>=2.8.2

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/rlgym_learn_algos/ppo/gae_trajectory_processor_pure_python.py

@@ -117,14 +117,17 @@ class GAETrajectoryProcessorPurePython(
         else:
             avg_return = np.nan
             return_std = np.nan
-        avg_reward = reward_sum[0] / exp_len
+        avg_reward = reward_sum.item() / exp_len
+        average_episode_return = reward_sum.item() / len(trajectories)
         trajectory_processor_data = GAETrajectoryProcessorData(
-            average_undiscounted_episodic_return=avg_reward,
+            average_undiscounted_episodic_return=average_episode_return,
             average_return=avg_return,
             return_standard_deviation=return_std,
+            average_reward=avg_reward
         )
         return (
             (
+                agent_ids,
                 observations,
                 actions,
                 torch.stack(log_probs_list).to(device=self.device),
@@ -146,7 +149,7 @@ class GAETrajectoryProcessorPurePython(
         self.max_returns_per_stats_increment = (
             config.trajectory_processor_config.max_returns_per_stats_increment
         )
-        self.dtype = np.dtype(config.dtype)
+        self.dtype = np.dtype(str(config.dtype).replace("torch.", ""))
         self.device = config.device
         self.checkpoint_load_folder = config.checkpoint_load_folder
         if self.checkpoint_load_folder is not None:

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/rlgym_learn_algos/ppo/ppo_learner.py

@@ -284,6 +284,15 @@ class PPOLearner(
                     batch_advantages,
                 ) = batch
                 batch_target_values = batch_values + batch_advantages
+                if self.config.learner_config.advantage_normalization:
+                    old_device = batch_advantages.device
+                    batch_advantages = batch_advantages.to(
+                        self.config.learner_config.device
+                    )
+                    std, mean = torch.std_mean(batch_advantages)
+                    batch_advantages = (batch_advantages - mean) / (std + 1e-8)
+                    batch_advantages = batch_advantages.to(old_device)
+
                 self.actor_optimizer.zero_grad()
                 self.critic_optimizer.zero_grad()
 
@@ -306,10 +315,6 @@ class PPOLearner(
                     advantages = batch_advantages[start:stop].to(
                         self.config.learner_config.device
                     )
-                    if self.config.learner_config.advantage_normalization:
-                        advantages = (advantages - torch.mean(advantages)) / (
-                            torch.std(advantages) + 1e-8
-                        )
                     old_probs = batch_old_probs[start:stop].to(
                         self.config.learner_config.device
                     )

{rlgym_learn_algos-0.2.4 → rlgym_learn_algos-0.2.5}/src/ppo/gae_trajectory_processor.rs

@@ -73,7 +73,11 @@ macro_rules! define_process_trajectories {
                 for trajectory in trajectories.into_iter() {
                     let trajectory_len = trajectory.obs_list.len();
                     let mut cur_return = 0 as $dtype;
-                    let mut next_val_pred = trajectory.final_val_pred.extract::<$dtype>()?;
+                    let mut next_val_pred = if trajectory.truncated {
+                        trajectory.final_val_pred.extract::<$dtype>()?
+                    } else {
+                        0 as $dtype
+                    };
                     let mut cur_advantage = 0 as $dtype;
                     let timesteps_rewards = batch_reward_type_numpy_converter
                         .call_method1(intern!(py, "as_numpy"), (&trajectory.reward_list,))?