multi-agent-rlenv 3.5.0__tar.gz → 3.5.2__tar.gz

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: multi-agent-rlenv
-Version: 3.5.0
+Version: 3.5.2
 Summary: A strongly typed Multi-Agent Reinforcement Learning framework
 Project-URL: repository, https://github.com/yamoling/multi-agent-rlenv
 Author-email: Yannick Molinghen <yannick.molinghen@ulb.be>

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/__init__.py

@@ -62,7 +62,7 @@ print(env.extras_shape) # (1, )
 If you want to create a new environment, you can simply create a class that inherits from `MARLEnv`. If you want to create a wrapper around an existing `MARLEnv`, you probably want to subclass `RLEnvWrapper` which implements a default behaviour for every method.
 """
 
-__version__ = "3.5.0"
+__version__ = "3.5.2"
 
 from . import models
 from .models import (

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/models/env.py

@@ -199,7 +199,7 @@ class MARLEnv(ABC, Generic[ActionSpaceType]):
             episode.add(step, action)
         return episode
 
-    def has_same_inouts(self, other) -> bool:
+    def has_same_inouts(self, other: "MARLEnv[ActionSpaceType]") -> bool:
         """Alias for `have_same_inouts(self, other)`."""
         if not isinstance(other, MARLEnv):
             return False

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/models/episode.py

@@ -66,13 +66,13 @@ class Episode:
         if target_len < self.episode_len:
             raise ValueError(f"Cannot pad episode to a smaller size: {target_len} < {self.episode_len}")
         padding_size = target_len - self.episode_len
-        obs = self.all_observations + [self.all_observations[0]] * padding_size
-        extras = self.all_extras + [self.all_extras[0]] * padding_size
-        actions = self.actions + [self.actions[0]] * padding_size
-        rewards = self.rewards + [self.rewards[0]] * padding_size
+        obs = self.all_observations + [np.zeros_like(self.all_observations[0])] * padding_size
+        extras = self.all_extras + [np.zeros_like(self.all_extras[0])] * padding_size
+        actions = self.actions + [np.zeros_like(self.actions[0])] * padding_size
+        rewards = self.rewards + [np.zeros_like(self.rewards[0])] * padding_size
         availables = self.all_available_actions + [self.all_available_actions[0]] * padding_size
-        states = self.all_states + [self.all_states[0]] * padding_size
-        states_extras = self.all_states_extras + [self.all_states_extras[0]] * padding_size
+        states = self.all_states + [np.zeros_like(self.all_states[0])] * padding_size
+        states_extras = self.all_states_extras + [np.zeros_like(self.all_states_extras[0])] * padding_size
         other = {key: value + [value[0]] * padding_size for key, value in self.other.items()}
         return Episode(
             all_observations=obs,

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/models/step.py

@@ -39,7 +39,7 @@ class Step:
             case int() | float():
                 self.reward = np.array([reward], dtype=np.float32)
             case np.ndarray():
-                self.reward = reward
+                self.reward = reward.astype(np.float32)
             case other:
                 # We assume this is a sequence of some sort
                 self.reward = np.array(other, dtype=np.float32)

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/wrappers/__init__.py

@@ -10,6 +10,7 @@ from .blind_wrapper import Blind
 from .centralised import Centralized
 from .available_actions_mask import AvailableActionsMask
 from .delayed_rewards import DelayedReward
+from .potential_shaping import PotentialShaping
 
 __all__ = [
     "RLEnvWrapper",
@@ -26,4 +27,5 @@ __all__ = [
     "Blind",
     "Centralized",
     "DelayedReward",
+    "PotentialShaping",
 ]

multi_agent_rlenv-3.5.2/src/marlenv/wrappers/potential_shaping.py

@@ -0,0 +1,54 @@
+from abc import abstractmethod, ABC
+from .rlenv_wrapper import RLEnvWrapper
+from marlenv import Space, MARLEnv, Observation
+from typing import TypeVar, Optional
+import numpy as np
+import numpy.typing as npt
+
+from dataclasses import dataclass
+
+A = TypeVar("A", bound=Space)
+
+
+@dataclass
+class PotentialShaping(RLEnvWrapper[A], ABC):
+    """
+    Potential shaping for the Laser Learning Environment (LLE).
+
+    https://people.eecs.berkeley.edu/~pabbeel/cs287-fa09/readings/NgHaradaRussell-shaping-ICML1999.pdf
+    """
+
+    gamma: float
+
+    def __init__(
+        self,
+        env: MARLEnv,
+        gamma: float = 1.0,
+        extra_shape: Optional[tuple[int]] = None,
+    ):
+        super().__init__(env, extra_shape=extra_shape)
+        self.gamma = gamma
+        self._current_potential = self.compute_potential()
+
+    def add_extras(self, obs: Observation) -> Observation:
+        """Add the extras related to potential shaping. Does nothing by default."""
+        return obs
+
+    def reset(self):
+        obs, state = super().reset()
+        self._current_potential = self.compute_potential()
+        return self.add_extras(obs), state
+
+    def step(self, actions):
+        prev_potential = self._current_potential
+        step = super().step(actions)
+
+        self._current_potential = self.compute_potential()
+        shaped_reward = self.gamma * self._current_potential - prev_potential
+        step.obs = self.add_extras(step.obs)
+        step.reward += shaped_reward
+        return step
+
+    @abstractmethod
+    def compute_potential(self) -> float | npt.NDArray[np.float32]:
+        """Compute the potential of the current state of the environment."""

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/src/marlenv/wrappers/rlenv_wrapper.py

@@ -1,7 +1,6 @@
 from typing import Optional, Sequence
 from typing_extensions import TypeVar
 from dataclasses import dataclass
-from abc import ABC
 import numpy as np
 
 from marlenv.models import MARLEnv, Space, DiscreteSpace, State
@@ -11,7 +10,7 @@ AS = TypeVar("AS", bound=Space, default=Space)
 
 
 @dataclass
-class RLEnvWrapper(MARLEnv[AS], ABC):
+class RLEnvWrapper(MARLEnv[AS]):
     """Parent class for all RLEnv wrappers"""
 
     wrapped: MARLEnv[AS]

{multi_agent_rlenv-3.5.0 → multi_agent_rlenv-3.5.2}/tests/test_wrappers.py

@@ -357,3 +357,36 @@ def test_delayed_rewards_from_builder():
         for end_game in range(delay + 1, delay * 2):
             env = Builder(DiscreteMockEnv(reward_step=10, end_game=end_game, n_agents=2)).delay_rewards(delay).build()
             _test_delayed_rewards(env)
+
+
+def test_potential_shaping():
+    from marlenv.wrappers.potential_shaping import PotentialShaping
+
+    class PS(PotentialShaping):
+        def __init__(self, env: MARLEnv):
+            self.phi = 10
+            super().__init__(env)
+
+        def reset(self):
+            self.phi = 10
+            return super().reset()
+
+        def compute_potential(self) -> float:
+            return self.phi
+
+        def step(self, actions):
+            self.phi = max(0, self.phi - 1)
+            return super().step(actions)
+
+    EP_LENGTH = 20
+    env = PS(DiscreteMockEnv(reward_step=0, end_game=EP_LENGTH))
+    env.reset()
+    step = None
+
+    for i in range(10):
+        step = env.random_step()
+        assert step.reward.item() == -1
+
+    for i in range(10):
+        step = env.random_step()
+        assert step.reward.item() == 0