multi-agent-rlenv 3.3.6__py3-none-any.whl → 3.4.0__py3-none-any.whl

marlenv/__init__.py

@@ -62,7 +62,7 @@ print(env.extras_shape) # (1, )
 If you want to create a new environment, you can simply create a class that inherits from `MARLEnv`. If you want to create a wrapper around an existing `MARLEnv`, you probably want to subclass `RLEnvWrapper` which implements a default behaviour for every method.
 """
 
-__version__ = "3.3.6"
+__version__ = "3.4.0"
 
 from . import models
 from . import wrappers

marlenv/adapters/overcooked_adapter.py

@@ -1,6 +1,6 @@
 import sys
 from dataclasses import dataclass
-from typing import Literal, Sequence
+from typing import Literal, Sequence, Optional
 from copy import deepcopy
 
 import cv2
@@ -8,6 +8,7 @@ import numpy as np
 import numpy.typing as npt
 import pygame
 from marlenv.models import ContinuousSpace, DiscreteActionSpace, MARLEnv, Observation, State, Step
+from marlenv.utils import Schedule
 
 from overcooked_ai_py.mdp.overcooked_env import OvercookedEnv
 from overcooked_ai_py.mdp.overcooked_mdp import Action, OvercookedGridworld, OvercookedState
@@ -17,8 +18,17 @@ from overcooked_ai_py.visualization.state_visualizer import StateVisualizer
 @dataclass
 class Overcooked(MARLEnv[Sequence[int] | npt.NDArray, DiscreteActionSpace]):
     horizon: int
+    shaping_factor: Schedule
 
-    def __init__(self, oenv: OvercookedEnv):
+    def __init__(
+        self,
+        oenv: OvercookedEnv,
+        shaping_factor: float | Schedule = 1.0,
+        name_suffix: Optional[str] = None,
+    ):
+        if isinstance(shaping_factor, (int, float)):
+            shaping_factor = Schedule.constant(shaping_factor)
+        self.shaping_factor = shaping_factor
         self._oenv = oenv
         assert isinstance(oenv.mdp, OvercookedGridworld)
         self._mdp = oenv.mdp
@@ -40,6 +50,8 @@ class Overcooked(MARLEnv[Sequence[int] | npt.NDArray, DiscreteActionSpace]):
             reward_space=ContinuousSpace.from_shape(1),
         )
         self.horizon = int(self._oenv.horizon)
+        if name_suffix is not None:
+            self.name = f"{self.name}-{name_suffix}"
 
     @property
     def state(self) -> OvercookedState:
@@ -84,12 +96,15 @@ class Overcooked(MARLEnv[Sequence[int] | npt.NDArray, DiscreteActionSpace]):
         return np.array(available_actions, dtype=np.bool)
 
     def step(self, actions: Sequence[int] | npt.NDArray[np.int32 | np.int64]) -> Step:
+        self.shaping_factor.update()
         actions = [Action.ALL_ACTIONS[a] for a in actions]
         _, reward, done, info = self._oenv.step(actions, display_phi=True)
+
+        reward += sum(info["shaped_r_by_agent"]) * self.shaping_factor
         return Step(
             obs=self.get_observation(),
             state=self.get_state(),
-            reward=np.array([reward]),
+            reward=np.array([reward], dtype=np.float32),
             done=done,
             truncated=False,
             info=info,
@@ -99,19 +114,25 @@ class Overcooked(MARLEnv[Sequence[int] | npt.NDArray, DiscreteActionSpace]):
         self._oenv.reset()
         return self.get_observation(), self.get_state()
 
-    def __deepcopy__(self, memo: dict):
+    def __deepcopy__(self, _):
+        """
+        Note: a specific implementation is needed because `pygame.font.Font` objects are not deep-copiable by default.
+        """
         mdp = deepcopy(self._mdp)
-        return Overcooked(OvercookedEnv.from_mdp(mdp, horizon=self.horizon))
+        copy = Overcooked(OvercookedEnv.from_mdp(mdp, horizon=self.horizon), deepcopy(self.shaping_factor))
+        copy.name = self.name
+        return copy
 
     def __getstate__(self):
-        return {"horizon": self.horizon, "mdp": self._mdp}
+        return {"horizon": self.horizon, "mdp": self._mdp, "name": self.name, "schedule": self.shaping_factor}
 
     def __setstate__(self, state: dict):
         from overcooked_ai_py.mdp.overcooked_mdp import Recipe
 
         mdp = state["mdp"]
         Recipe.configure(mdp.recipe_config)
-        self.__init__(OvercookedEnv.from_mdp(state["mdp"], horizon=state["horizon"]))
+        self.__init__(OvercookedEnv.from_mdp(state["mdp"], horizon=state["horizon"]), shaping_factor=state["schedule"])
+        self.name = state["name"]
 
     def get_image(self):
         rewards_dict = {}  # dictionary of details you want rendered in the UI
@@ -185,6 +206,37 @@ class Overcooked(MARLEnv[Sequence[int] | npt.NDArray, DiscreteActionSpace]):
             "you_shall_not_pass",
         ],
         horizon: int = 400,
+        reward_shaping_factor: float | Schedule = 1.0,
     ):
         mdp = OvercookedGridworld.from_layout_name(layout)
-        return Overcooked(OvercookedEnv.from_mdp(mdp, horizon=horizon))
+        return Overcooked(OvercookedEnv.from_mdp(mdp, horizon=horizon, info_level=0), reward_shaping_factor, layout)
+
+    @staticmethod
+    def from_grid(
+        grid: Sequence[Sequence[Literal["S", "P", "X", "O", "D", "T", "1", "2", " "] | str]],
+        horizon: int = 400,
+        shaping_factor: float | Schedule = 1.0,
+        layout_name: Optional[str] = None,
+    ):
+        """
+        Create an Overcooked environment from a grid layout where
+        - S is a serving location
+        - P is a cooking pot
+        - X is a counter
+        - O is an onion dispenser
+        - D is a dish dispenser
+        - T is a tomato dispenser
+        - 1 is a player 1 starting location
+        - 2 is a player 2 starting location
+        - ' ' is a walkable space
+
+        If provided, `custom_name` is added to the environment name.
+        """
+        # It is necessary to add an explicit layout name because Overcooked saves some files under this
+        # name. By default the name is a concatenation of the grid elements, which may include characters
+        # such as white spaces, pipes ('|') and square brackets ('[' and ']') that are invalid Windows file paths.
+        if layout_name is None:
+            layout_name = "custom-layout"
+        mdp = OvercookedGridworld.from_grid(grid, base_layout_params={"layout_name": layout_name})
+
+        return Overcooked(OvercookedEnv.from_mdp(mdp, horizon=horizon, info_level=0), shaping_factor, layout_name)

marlenv/models/env.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from itertools import product
-from typing import Generic, Optional, Sequence
+from typing import Any, Generic, Optional, Sequence
 
 import cv2
 import numpy as np
@@ -13,8 +13,8 @@ from .spaces import ActionSpace, ContinuousSpace, Space
 from .state import State
 from .step import Step
 
-ActionType = TypeVar("ActionType", default=npt.NDArray)
-ActionSpaceType = TypeVar("ActionSpaceType", bound=ActionSpace, default=ActionSpace)
+ActionType = TypeVar("ActionType", default=Any)
+ActionSpaceType = TypeVar("ActionSpaceType", bound=ActionSpace, default=Any)
 
 
 @dataclass
@@ -108,6 +108,11 @@ class MARLEnv(ABC, Generic[ActionType, ActionSpaceType]):
         """Whether the environment is multi-objective."""
         return self.reward_space.size > 1
 
+    @property
+    def n_objectives(self) -> int:
+        """The number of objectives in the environment."""
+        return self.reward_space.size
+
     def sample_action(self) -> ActionType:
         """Sample an available action from the action space."""
         return self.action_space.sample(self.available_actions())  # type: ignore

marlenv/models/episode.py

@@ -179,9 +179,9 @@ class Episode(Generic[A]):
     @cached_property
     def dones(self):
         """The done flags for each transition"""
-        dones = np.zeros_like(self.rewards, dtype=np.float32)
+        dones = np.zeros_like(self.rewards, dtype=np.bool)
         if self.is_done:
-            dones[self.episode_len - 1 :] = 1.0
+            dones[self.episode_len - 1 :] = True
         return dones
 
     @property

marlenv/utils/__init__.py

@@ -0,0 +1,10 @@
+from .schedule import Schedule, MultiSchedule, RoundedSchedule, LinearSchedule, ExpSchedule
+
+
+__all__ = [
+    "Schedule",
+    "LinearSchedule",
+    "ExpSchedule",
+    "MultiSchedule",
+    "RoundedSchedule",
+]

marlenv/utils/schedule.py

@@ -0,0 +1,281 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Callable, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class Schedule:
+    """
+    Schedules the value of a varaible over time.
+    """
+
+    name: str
+    start_value: float
+    end_value: float
+    _t: int
+    n_steps: int
+
+    def __init__(self, start_value: float, end_value: float, n_steps: int):
+        self.start_value = start_value
+        self.end_value = end_value
+        self.n_steps = n_steps
+        self.name = self.__class__.__name__
+        self._t = 0
+        self._current_value = self.start_value
+
+    def update(self, step: Optional[int] = None):
+        """Update the value of the schedule. Force a step if given."""
+        if step is not None:
+            self._t = step
+        else:
+            self._t += 1
+        if self._t >= self.n_steps:
+            self._current_value = self.end_value
+        else:
+            self._current_value = self._compute()
+
+    @abstractmethod
+    def _compute(self) -> float:
+        """Compute the value of the schedule"""
+
+    @property
+    def value(self) -> float:
+        """Returns the current value of the schedule"""
+        return self._current_value
+
+    @staticmethod
+    def constant(value: float):
+        return ConstantSchedule(value)
+
+    @staticmethod
+    def linear(start_value: float, end_value: float, n_steps: int):
+        return LinearSchedule(start_value, end_value, n_steps)
+
+    @staticmethod
+    def exp(start_value: float, end_value: float, n_steps: int):
+        return ExpSchedule(start_value, end_value, n_steps)
+
+    @staticmethod
+    def arbitrary(func: Callable[[int], float], n_steps: Optional[int] = None):
+        if n_steps is None:
+            n_steps = 0
+        return ArbitrarySchedule(func, n_steps)
+
+    def rounded(self, n_digits: int = 0) -> "RoundedSchedule":
+        return RoundedSchedule(self, n_digits)
+
+    # Operator overloading
+    def __mul__(self, other: T) -> T:
+        return self.value * other  # type: ignore
+
+    def __rmul__(self, other: T) -> T:
+        return self.value * other  # type: ignore
+
+    def __pow__(self, exp: float) -> float:
+        return self.value**exp
+
+    def __rpow__(self, other: T) -> T:
+        return other**self.value  # type: ignore
+
+    def __add__(self, other: T) -> T:
+        return self.value + other  # type: ignore
+
+    def __radd__(self, other: T) -> T:
+        return self.value + other  # type: ignore
+
+    def __neg__(self):
+        return -self.value
+
+    def __pos__(self):
+        return +self.value
+
+    def __sub__(self, other: T) -> T:
+        return self.value - other  # type: ignore
+
+    def __rsub__(self, other: T) -> T:
+        return other - self.value  # type: ignore
+
+    def __div__(self, other: T) -> T:
+        return self.value // other  # type: ignore
+
+    def __rdiv__(self, other: T) -> T:
+        return other // self.value  # type: ignore
+
+    def __truediv__(self, other: T) -> T:
+        return self.value / other  # type: ignore
+
+    def __rtruediv__(self, other: T) -> T:
+        return other / self.value  # type: ignore
+
+    def __lt__(self, other) -> bool:
+        return self.value < other
+
+    def __le__(self, other) -> bool:
+        return self.value <= other
+
+    def __gt__(self, other) -> bool:
+        return self.value > other
+
+    def __ge__(self, other) -> bool:
+        return self.value >= other
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, Schedule):
+            if self.start_value != other.start_value:
+                return False
+            if self.end_value != other.end_value:
+                return False
+            if self.n_steps != other.n_steps:
+                return False
+            if type(self) is not type(other):
+                return False
+        return self.value == other
+
+    def __ne__(self, other) -> bool:
+        return not (self.__eq__(other))
+
+    def __float__(self):
+        return self.value
+
+    def __int__(self) -> int:
+        return int(self.value)
+
+
+@dataclass(eq=False)
+class LinearSchedule(Schedule):
+    a: float
+    b: float
+
+    def __init__(self, start_value: float, end_value: float, n_steps: int):
+        super().__init__(start_value, end_value, n_steps)
+        self._current_value = self.start_value
+        # y = ax + b
+        self.a = (self.end_value - self.start_value) / self.n_steps
+        self.b = self.start_value
+
+    def _compute(self):
+        return self.a * (self._t) + self.b
+
+    @property
+    def value(self) -> float:
+        return self._current_value
+
+
+@dataclass(eq=False)
+class ExpSchedule(Schedule):
+    """Exponential schedule. After n_steps, the value will be min_value.
+
+    Update formula is next_value = start_value * (min_value / start_value) ** (step / (n - 1))
+    """
+
+    n_steps: int
+    base: float
+    last_update_step: int
+
+    def __init__(self, start_value: float, min_value: float, n_steps: int):
+        super().__init__(start_value, min_value, n_steps)
+        self.base = self.end_value / self.start_value
+        self.last_update_step = self.n_steps - 1
+
+    def _compute(self):
+        return self.start_value * (self.base) ** (self._t / (self.n_steps - 1))
+
+    @property
+    def value(self) -> float:
+        return self._current_value
+
+
+@dataclass(eq=False)
+class ConstantSchedule(Schedule):
+    def __init__(self, value: float):
+        super().__init__(value, value, 0)
+        self._value = value
+
+    def update(self, step=None):
+        return
+
+    @property
+    def value(self) -> float:
+        return self._value
+
+
+@dataclass(eq=False)
+class RoundedSchedule(Schedule):
+    def __init__(self, schedule: Schedule, n_digits: int):
+        super().__init__(schedule.start_value, schedule.end_value, schedule.n_steps)
+        self.schedule = schedule
+        self.n_digits = n_digits
+
+    def update(self, step: int | None = None):
+        return self.schedule.update(step)
+
+    def _compute(self) -> float:
+        return self.schedule._compute()
+
+    @property
+    def value(self) -> float:
+        return round(self.schedule.value, self.n_digits)
+
+
+@dataclass(eq=False)
+class MultiSchedule(Schedule):
+    def __init__(self, schedules: dict[int, Schedule]):
+        ordered_schedules, ordered_steps = MultiSchedule._verify(schedules)
+        n_steps = ordered_steps[-1] + ordered_schedules[-1].n_steps
+        super().__init__(ordered_schedules[0].start_value, ordered_schedules[-1].end_value, n_steps)
+        self.schedules = iter(ordered_schedules)
+        self.current_schedule = next(self.schedules)
+        self.offset = 0
+        self.current_end = ordered_steps[1]
+
+    @staticmethod
+    def _verify(schedules: dict[int, Schedule]):
+        sorted_steps = sorted(schedules.keys())
+        sorted_schedules = [schedules[t] for t in sorted_steps]
+        if sorted_steps[0] != 0:
+            raise ValueError("First schedule must start at t=0")
+        current_step = 0
+        for i in range(len(sorted_steps)):
+            # Artificially set the end step of ConstantSchedules to the next step
+            if isinstance(sorted_schedules[i], ConstantSchedule):
+                if i + 1 < len(sorted_steps):
+                    sorted_schedules[i].n_steps = sorted_steps[i + 1] - sorted_steps[i]
+            if sorted_steps[i] != current_step:
+                raise ValueError(f"Schedules are not contiguous at t={current_step}")
+            current_step += sorted_schedules[i].n_steps
+        return sorted_schedules, sorted_steps
+
+    def update(self, step: int | None = None):
+        if step is not None:
+            raise NotImplementedError("Cannot update MultiSchedule with a specific step")
+        super().update(step)
+        # If we reach the end of the current schedule, update to the next one
+        # except if we are at the end.
+        if self._t == self.current_end and self._t < self.n_steps:
+            self.current_schedule = next(self.schedules)
+            self.offset = self._t
+            self.current_end += self.current_schedule.n_steps
+        self.current_schedule.update(self.relative_step)
+
+    @property
+    def relative_step(self):
+        return self._t - self.offset
+
+    def _compute(self) -> float:
+        return self.current_schedule._compute()
+
+    @property
+    def value(self):
+        return self.current_schedule.value
+
+
+@dataclass(eq=False)
+class ArbitrarySchedule(Schedule):
+    def __init__(self, fn: Callable[[int], float], n_steps: int):
+        super().__init__(fn(0), fn(n_steps), n_steps)
+        self._func = fn
+
+    def _compute(self) -> float:
+        return self._func(self._t)

{multi_agent_rlenv-3.3.6.dist-info → multi_agent_rlenv-3.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: multi-agent-rlenv
-Version: 3.3.6
+Version: 3.4.0
 Summary: A strongly typed Multi-Agent Reinforcement Learning framework
 Project-URL: repository, https://github.com/yamoling/multi-agent-rlenv
 Author-email: Yannick Molinghen <yannick.molinghen@ulb.be>

{multi_agent_rlenv-3.3.6.dist-info → multi_agent_rlenv-3.4.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-marlenv/__init__.py,sha256=iEVXbl4mQmey7P2uFdeKEYWEmZ8QxNS_f52jNdw4nZs,3741
+marlenv/__init__.py,sha256=sDucG8AdBbAOoO21QpTuMymZSPvp5fq6EBqCgTzFLkk,3741
 marlenv/env_builder.py,sha256=s_lQANqP3iNc8nmcr3CanRVsExnn9qh0ihh4lFr0c4c,5560
 marlenv/env_pool.py,sha256=R3WIrnQ5Zvff4HR1ecfkDmuO2zl7v1ywQ0K2_nvWFzs,1070
 marlenv/exceptions.py,sha256=gJUC_2rVAvOfK_ypVFc7Myh-pIfSU3To38VBVS_0rZA,1179
@@ -6,18 +6,20 @@ marlenv/mock_env.py,sha256=qB0fYFIfbopJf7Va8kCeVI5vsOy1-2JdEYe9gdV1Ruw,4761
 marlenv/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 marlenv/adapters/__init__.py,sha256=rWiqQOqTx3kVL5ZkPo3rkczrlQBBhQbU55zGI26SEeY,929
 marlenv/adapters/gym_adapter.py,sha256=6CBEjANViTJBTUBmtVyrhJrzjBJxNs_4hmMnXXG2mkU,2906
-marlenv/adapters/overcooked_adapter.py,sha256=JZhB50cQGWGjaHWuPwskUKr6YthEptpYC3cD7i9GVvk,6832
+marlenv/adapters/overcooked_adapter.py,sha256=gu0TOp-FiLUVOskDnaqGA2D44P0qJNk4KWznHA2M8L8,9174
 marlenv/adapters/pettingzoo_adapter.py,sha256=4F1au6uctsqRhGfcZOeDRH-8hmrFXnA5xH1Z1Pnek3s,2870
 marlenv/adapters/pymarl_adapter.py,sha256=x__E90XpFbfSWhnBHtkcD6WYkmKki1LByNbUFoDBUcg,3416
 marlenv/adapters/smac_adapter.py,sha256=fOfKo1hL4ioKtM5qQGcwtfdkdwUEACjAZqaGmkoQUcU,8373
 marlenv/models/__init__.py,sha256=9M-rnj94nsdyO4zm_VEtyYBmde3iD2_eIY4bMB-IqCo,555
-marlenv/models/env.py,sha256=S0Yd75X4O7desCHrsdEdz045Aw1Zrtu-pkfz5obYQJI,7433
-marlenv/models/episode.py,sha256=ZGBx6lb2snrUhDgFEwHPV1dp-XvMA7k4quQVUNQxsP0,15140
+marlenv/models/env.py,sha256=54I6SWkzO3nXZ0L7bRWpKJH_ywDu8iS-S2YwHL3yXDU,7569
+marlenv/models/episode.py,sha256=hExUIcOjXImA-hfOgIWnq6sQPJAgZYhT6pQ1x4SBAjk,15138
 marlenv/models/observation.py,sha256=kAmh1hIoC2TGrZlGVzV0y4TXXCSrI7gcmG0raeoncYk,3153
 marlenv/models/spaces.py,sha256=pw8Sum_fHBkR-lyfTqUij4azMCNm8oBZrYZe4WVR7rA,7652
 marlenv/models/state.py,sha256=958PXTHadi3gtRnhGgcGtqBnF44R11kdcx62NN2gwxA,1717
 marlenv/models/step.py,sha256=LKGAV2Cu-k9Gz1hwrfvGx51l8axtQRqDE9WVL5r2A1Q,3037
 marlenv/models/transition.py,sha256=2vvuhSSq911weCXio9nuyfsLVh_7ORSU_znOqpLLdLg,5107
+marlenv/utils/__init__.py,sha256=C3qhvkVwctBP8mG3G5nkAZ5DKfErVRkdbHo7oeWVsM0,209
+marlenv/utils/schedule.py,sha256=slhtpQiBHSUNyPmSkKb2yBgiHJqPhoPxa33GxvyV8Jc,8565
 marlenv/wrappers/__init__.py,sha256=wl23NUYcl0vPJb2QLpe4Xj8ZocUIOarAZX8CgWqdSQE,808
 marlenv/wrappers/agent_id_wrapper.py,sha256=oTIAYxKD1JtHfrZN43mf-3e8pxjd0nxm07vxs3BfrGY,1187
 marlenv/wrappers/available_actions_mask.py,sha256=JoCJ9eqHlkY8wfY-oaceEi8yp1Efs1iK6IO2Ibf9oZA,1468
@@ -31,7 +33,7 @@ marlenv/wrappers/penalty_wrapper.py,sha256=v4_H8OEN2-yujLzRb6P7W7KwmXHtjAFsxcdp3
 marlenv/wrappers/rlenv_wrapper.py,sha256=C2XekgBIM4x3Wa2Mtsn7rihRD4ymC2hORI473Af0sfw,2962
 marlenv/wrappers/time_limit.py,sha256=CDIMMJPMyIDHSFxUJaC7nb7Kd86-07NgZeFhrpZm82o,3985
 marlenv/wrappers/video_recorder.py,sha256=d5AFu6qHqby9mOcBsYWYPxAPiK1vtnfMYdZ81AnCekI,2624
-multi_agent_rlenv-3.3.6.dist-info/METADATA,sha256=oHsLxFw-wlgzPyswB6r3QIWZWM_injRIFJuJczyZDTo,4897
-multi_agent_rlenv-3.3.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-multi_agent_rlenv-3.3.6.dist-info/licenses/LICENSE,sha256=_eeiGVoIJ7kYt6l1zbIvSBQppTnw0mjnYk1lQ4FxEjE,1074
-multi_agent_rlenv-3.3.6.dist-info/RECORD,,
+multi_agent_rlenv-3.4.0.dist-info/METADATA,sha256=5TitLGgWA_BrpC_xhFHDufA4QB7v1EZ0FHP3Hdtbf5k,4897
+multi_agent_rlenv-3.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+multi_agent_rlenv-3.4.0.dist-info/licenses/LICENSE,sha256=_eeiGVoIJ7kYt6l1zbIvSBQppTnw0mjnYk1lQ4FxEjE,1074
+multi_agent_rlenv-3.4.0.dist-info/RECORD,,