multi-agent-rlenv 3.7.0__tar.gz → 3.7.1__tar.gz

{multi_agent_rlenv-3.7.0 → multi_agent_rlenv-3.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: multi-agent-rlenv
-Version: 3.7.0
+Version: 3.7.1
 Summary: A strongly typed Multi-Agent Reinforcement Learning framework
 Project-URL: repository, https://github.com/yamoling/multi-agent-rlenv
 Author-email: Yannick Molinghen <yannick.molinghen@ulb.be>

{multi_agent_rlenv-3.7.0 → multi_agent_rlenv-3.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "multi-agent-rlenv"
-version = "3.7.0"
+version = "3.7.1"
 description = "A strongly typed Multi-Agent Reinforcement Learning framework"
 authors = [
     { "name" = "Yannick Molinghen", "email" = "yannick.molinghen@ulb.be" },

{multi_agent_rlenv-3.7.0 → multi_agent_rlenv-3.7.1}/src/marlenv/adapters/smac_adapter.py

@@ -3,7 +3,7 @@ from typing import overload
 
 import numpy as np
 import numpy.typing as npt
-from smac.env import StarCraft2Env  # type:ignore[import]
+from smac.env import StarCraft2Env  # pyright: ignore[reportMissingImports]
 
 from marlenv.models import MARLEnv, Observation, State, Step, MultiDiscreteSpace, DiscreteSpace
 

{multi_agent_rlenv-3.7.0 → multi_agent_rlenv-3.7.1}/src/marlenv/catalog/__init__.py

@@ -1,13 +1,10 @@
 from marlenv.adapters import SMAC
 from .deepsea import DeepSea
+from .matrix_game import MatrixGame
+from .coordinated_grid import CoordinatedGrid
 
 
-__all__ = [
-    "SMAC",
-    "DeepSea",
-    "lle",
-    "overcooked",
-]
+__all__ = ["SMAC", "DeepSea", "lle", "overcooked", "MatrixGame", "connect_n", "CoordinatedGrid"]
 
 
 def lle():
@@ -20,3 +17,9 @@ def overcooked():
     from overcooked import Overcooked  # pyright: ignore[reportMissingImports]
 
     return Overcooked
+
+
+def connect_n():
+    from .connectn import ConnectN
+
+    return ConnectN

multi_agent_rlenv-3.7.1/src/marlenv/catalog/connectn/__init__.py

@@ -0,0 +1,11 @@
+"""
+Connect-N game environment.
+
+Inspiration from: https://github.com/Gualor/connect4-montecarlo
+"""
+
+from .board import GameBoard
+from .env import ConnectN
+
+
+__all__ = ["ConnectN", "GameBoard"]

multi_agent_rlenv-3.7.1/src/marlenv/catalog/connectn/board.py

@@ -0,0 +1,186 @@
+from enum import IntEnum
+
+import numpy as np
+
+
+class StepResult(IntEnum):
+    NOTHING = 0
+    TIE = 1
+    WIN = 2
+
+
+class GameBoard:
+    """Connect4 game board class."""
+
+    def __init__(self, width: int, height: int, n: int):
+        assert width >= n or height >= height, "Impossible to win with this combination of width, height and n"
+        self.turn = 1
+        self.board = np.zeros(shape=(height, width), dtype=np.float32)
+        self.width = width
+        self.height = height
+        self.n_to_align = n
+        self.n_items_in_column = np.zeros(width, dtype=np.int32)
+
+        self.str_row = "+" + "-" * (self.width * 4 - 1) + "+"
+        self.numbers = "|" + " ".join([f" {i} " for i in range(self.width)]) + "|"
+
+    def valid_moves(self):
+        """Get list of valid moves (i.e. not full columns)."""
+        return self.n_items_in_column < self.height
+
+    def clear(self):
+        self.board = np.zeros(shape=(self.height, self.width), dtype=np.float32)
+        self.n_items_in_column = np.zeros(self.width, dtype=np.int32)
+        self.turn = 0
+
+    def show(self):
+        """Print out game board on console."""
+        print(self.str_row)
+        for j in range(self.height - 1, -1, -1):
+            for i in range(self.width):
+                match self.board[j, i]:
+                    case 1:
+                        print("| X", end=" ")
+                    case -1:
+                        print("| O", end=" ")
+                    case _:
+                        print("|  ", end=" ")
+            print("|")
+        print(self.str_row)
+        print(self.numbers)
+        print(self.str_row)
+
+    def check_win(self, move_played: tuple[int, int]) -> bool:
+        if self.check_rows(move_played):
+            return True
+        if self.check_cols(move_played):
+            return True
+        if self.check_diags(move_played):
+            return True
+        return False
+
+    def check_tie(self) -> bool:
+        """
+        Check whether the game is a tie (i.e. the board is full).
+
+        Note that it does not check for a win, so it should be called after check_win.
+        """
+        # If the last row is full, the game is a tie
+        return bool(np.all(self.board[-1] != 0))
+
+    def check_rows(self, move_played: tuple[int, int]) -> bool:
+        row, col = move_played
+        start_index = max(0, col - self.n_to_align + 1)
+        end_index = min(self.width - self.n_to_align, col) + 1
+        for start in range(start_index, end_index):
+            slice = self.board[row, start : start + self.n_to_align]
+            if np.all(slice == self.turn):
+                return True
+        return False
+
+    def check_cols(self, move_played: tuple[int, int]) -> bool:
+        row, col = move_played
+        start_index = max(0, row - self.n_to_align + 1)
+        end_index = min(self.height - self.n_to_align, row) + 1
+        for start in range(start_index, end_index):
+            slice = self.board[start : start + self.n_to_align, col]
+            if np.all(slice == self.turn):
+                return True
+        return False
+
+    def check_diags(self, move_played: tuple[int, int]) -> bool:
+        row, col = move_played
+        # count the adjacent items in the / diagonal
+        n_adjacent = 0
+        # Top right
+        row_i, col_i = row + 1, col + 1
+        while row_i < self.height and col_i < self.width and self.board[row_i, col_i] == self.turn:
+            n_adjacent += 1
+            row_i += 1
+            col_i += 1
+        # Bottom left
+        row_i, col_i = row - 1, col - 1
+        while row_i >= 0 and col_i >= 0 and self.board[row_i, col_i] == self.turn:
+            n_adjacent += 1
+            row_i -= 1
+            col_i -= 1
+        if n_adjacent >= self.n_to_align - 1:
+            return True
+
+        # Count adjacent items in the \ diagonal
+        n_adjacent = 0
+        # Top left
+        row_i, col_i = row + 1, col - 1
+        while row_i < self.height and col_i >= 0 and self.board[row_i, col_i] == self.turn:
+            n_adjacent += 1
+            row_i += 1
+            col_i -= 1
+        # Bottom right
+        row_i, col_i = row - 1, col + 1
+        while row_i >= 0 and col_i < self.width and self.board[row_i, col_i] == self.turn:
+            n_adjacent += 1
+            row_i -= 1
+            col_i += 1
+
+        return n_adjacent >= self.n_to_align - 1
+
+    def play(self, column: int) -> StepResult:
+        """Apply move to board.
+
+        Args:
+            column (int): Selected column index (between 0 and the number of cols - 1).
+
+        Returns:
+            bool: whether the player has won.
+        """
+        row_index = self.n_items_in_column[column]
+        if row_index >= self.height:
+            raise ValueError(f"Column {column} is full, use `valid_moves` to check valid moves.")
+        self.n_items_in_column[column] += 1
+        self.board[row_index, column] = self.turn
+        if self.check_win((row_index, column)):
+            result = StepResult.WIN
+        elif self.check_tie():
+            result = StepResult.TIE
+        else:
+            result = StepResult.NOTHING
+        self.switch_turn()
+        return result
+
+    def switch_turn(self) -> None:
+        """Switch turn between players."""
+        self.turn = -self.turn
+
+
+def test_win():
+    board = GameBoard(4, 1, 2)
+    assert board.play(0) == StepResult.NOTHING
+    assert board.play(2) == StepResult.NOTHING
+    assert board.play(1) == StepResult.WIN
+
+
+def test_tie():
+    board = GameBoard(4, 1, 2)
+    assert board.play(0) == StepResult.NOTHING
+    assert board.play(1) == StepResult.NOTHING
+    assert board.play(2) == StepResult.NOTHING
+    assert board.play(3) == StepResult.TIE
+
+
+def test_win_diag():
+    board = GameBoard(2, 2, 2)
+    assert board.play(0) == StepResult.NOTHING
+    assert board.play(1) == StepResult.NOTHING
+    assert board.play(1) == StepResult.WIN
+
+    board.clear()
+    assert board.play(1) == StepResult.NOTHING
+    assert board.play(1) == StepResult.NOTHING
+    assert board.play(0) == StepResult.WIN
+
+
+if __name__ == "__main__":
+    test_win()
+    test_tie()
+    test_win_diag()
+    print("All tests passed!")

multi_agent_rlenv-3.7.1/src/marlenv/catalog/connectn/env.py

@@ -0,0 +1,51 @@
+from typing import Sequence
+import numpy as np
+import numpy.typing as npt
+from marlenv import MARLEnv, MultiDiscreteSpace, Step, State, Observation, DiscreteSpace
+
+from .board import GameBoard, StepResult
+
+
+class ConnectN(MARLEnv[MultiDiscreteSpace]):
+    def __init__(self, width: int = 7, height: int = 6, n: int = 4):
+        self.board = GameBoard(width, height, n)
+        action_space = DiscreteSpace(self.board.width).repeat(1)
+        observation_shape = (self.board.height, self.board.width)
+        state_shape = observation_shape
+        super().__init__(1, action_space, observation_shape, state_shape)
+
+    def reset(self):
+        self.board.clear()
+        return self.get_observation(), self.get_state()
+
+    def step(self, action: Sequence[int] | npt.NDArray[np.uint32]):
+        match self.board.play(action[0]):
+            case StepResult.NOTHING:
+                done = False
+                reward = 0
+            case StepResult.WIN:
+                done = True
+                reward = 1
+            case StepResult.TIE:
+                done = True
+                reward = 0
+        return Step(self.get_observation(), self.get_state(), reward, done, False)
+
+    def available_actions(self):
+        """Full columns are not available."""
+        return np.expand_dims(self.board.valid_moves(), axis=0)
+
+    def get_observation(self):
+        return Observation(self.board.board.copy(), self.available_actions())
+
+    def get_state(self):
+        return State(self.board.board.copy(), np.array([self.board.turn]))
+
+    def set_state(self, state: State):
+        self.board.board = state.data.copy()  # type: ignore Currently a type error because of the unchecked shape
+        self.board.turn = int(state.extras[0])
+        n_completed = np.count_nonzero(self.board.board, axis=0)
+        self.board.n_items_in_column = n_completed
+
+    def render(self):
+        self.board.show()

multi_agent_rlenv-3.7.1/src/marlenv/catalog/coordinated_grid.py

@@ -0,0 +1,139 @@
+import numpy as np
+import itertools
+from marlenv import MARLEnv, DiscreteSpace, Observation, State, Step
+
+
+N_ROWS = 11
+N_COLS = 12
+
+
+class CoordinatedGrid(MARLEnv):
+    """
+    Coordinated grid world environment used in the EMC paper to test the effectiveness of the proposed method.
+    https://proceedings.neurips.cc/paper_files/paper/2021/file/1e8ca836c962598551882e689265c1c5-Paper.pdf
+    """
+
+    def __init__(
+        self,
+        episode_limit=30,
+        time_penalty=2,
+    ):
+        super().__init__(
+            n_agents=2,
+            action_space=DiscreteSpace(5, ["SOUTH", "NORTH", "WEST", "EAST", "STAY"]).repeat(2),
+            observation_shape=(N_ROWS + N_COLS,),
+            state_shape=(N_ROWS + N_COLS,) * 2,
+        )
+        self._episode_steps = 0
+        self.episode_limit = episode_limit
+        self.center = N_COLS // 2
+        ###larger gridworld
+        visible_row = [i for i in range(N_ROWS // 2 - 2, N_ROWS // 2 + 3)]
+        visible_col = [i for i in range(N_COLS // 2 - 3, N_COLS // 2 + 3)]
+        self.vision_index = [[i, j] for i, j in list(itertools.product(visible_row, visible_col))]
+        self.agents_location = [[0, 0], [N_ROWS - 1, N_COLS - 1]]
+        self.time_penalty = time_penalty
+
+    def reset(self):
+        self.agents_location = [[0, 0], [N_ROWS - 1, N_COLS - 1]]
+        self._episode_steps = 0
+        return self.get_observation(), self.get_state()
+
+    def get_observation(self):
+        obs_1 = [[0 for _ in range(N_ROWS)], [0 for _ in range(N_COLS)]]
+        # obs_2 = obs_1.copy()
+        import copy
+
+        obs_2 = copy.deepcopy(obs_1)
+
+        obs_1[0][self.agents_location[0][0]] = 1
+        obs_1[1][self.agents_location[0][1]] = 1
+        obs_1 = obs_1[0] + obs_1[1]
+
+        obs_2[0][self.agents_location[1][0]] = 1
+        obs_2[1][self.agents_location[1][1]] = 1
+        obs_2 = obs_2[0] + obs_2[1]
+
+        if self.agents_location[0] in self.vision_index and self.agents_location[1] in self.vision_index:
+            temp = obs_1.copy()
+            obs_1 += obs_2.copy()
+            obs_2 += temp.copy()
+        elif self.agents_location[0] in self.vision_index:
+            obs_2 += obs_1.copy()
+            obs_1 += [0 for _ in range(N_ROWS + N_COLS)]
+        elif self.agents_location[1] in self.vision_index:
+            obs_1 += obs_2.copy()
+            obs_2 += [0 for _ in range(N_ROWS + N_COLS)]
+        else:
+            obs_2 += [0 for _ in range(N_ROWS + N_COLS)]
+            obs_1 += [0 for _ in range(N_ROWS + N_COLS)]
+
+        obs_data = np.array([obs_1, obs_2])
+        return Observation(obs_data, self.available_actions())
+
+    def get_state(self):
+        obs = self.get_observation()
+        state_data = obs.data.reshape(-1)
+        return State(state_data)
+
+    def available_actions(self):
+        avail_actions = np.full((self.n_agents, self.n_actions), True)
+        for agent_num, (y, x) in enumerate(self.agents_location):
+            if x == 0:
+                avail_actions[agent_num, 0] = 0
+            elif x == N_ROWS - 1:
+                avail_actions[agent_num, 1] = 0
+            if y == 0:
+                avail_actions[agent_num, 2] = 0
+            # Check for center line (depends on the agent number)
+            elif y == self.center + agent_num - 1:
+                avail_actions[agent_num, 3] = 0
+        return avail_actions
+
+    def step(self, action):
+        for idx, action in enumerate(action):
+            match action:
+                case 0:
+                    self.agents_location[idx][0] -= 1
+                case 1:
+                    self.agents_location[idx][0] += 1
+                case 2:
+                    self.agents_location[idx][1] -= 1
+                case 3:
+                    self.agents_location[idx][1] += 1
+                case 4:
+                    pass
+                case _:
+                    raise ValueError(f"Invalid action {action} for agent {idx}!")
+
+        self._episode_steps += 1
+        terminated = self._episode_steps >= self.episode_limit
+        env_info = {"battle_won": False}
+        n_arrived = self.n_agents_arrived()
+        if n_arrived == 1:
+            reward = -self.time_penalty
+        elif n_arrived == 2:
+            reward = 10
+            terminated = True
+            env_info = {"battle_won": True}
+        else:
+            reward = 0
+        return Step(self.get_observation(), self.get_state(), reward, terminated, terminated, env_info)
+
+    def n_agents_arrived(self):
+        n = 0
+        if self.agents_location[0] == [N_ROWS // 2, self.center - 1]:
+            n += 1
+        if self.agents_location[1] == [N_ROWS // 2, self.center]:
+            n += 1
+        return n
+
+    def render(self):
+        print("Agents location: ", self.agents_location)
+        for row in range(N_ROWS):
+            for col in range(N_COLS):
+                if [row, col] in self.agents_location:
+                    print("X", end=" ")
+                else:
+                    print(".", end=" ")
+            print()

multi_agent_rlenv-3.7.1/src/marlenv/catalog/matrix_game.py

@@ -0,0 +1,52 @@
+import numpy as np
+from marlenv import MARLEnv, Observation, DiscreteSpace, State, Step
+
+
+class MatrixGame(MARLEnv):
+    """Single step matrix game used in QTRAN, Qatten and QPLEX papers."""
+
+    N_AGENTS = 2
+    UNIT_DIM = 1
+    OBS_SHAPE = (1,)
+    STATE_SIZE = UNIT_DIM * N_AGENTS
+
+    QPLEX_PAYOFF_MATRIX = [
+        [8.0, -12.0, -12.0],
+        [-12.0, 0.0, 0.0],
+        [-12.0, 0.0, 0.0],
+    ]
+
+    def __init__(self, payoff_matrix: list[list[float]]):
+        action_names = [chr(ord("A") + i) for i in range(len(payoff_matrix[0]))]
+        super().__init__(
+            2,
+            action_space=DiscreteSpace(len(payoff_matrix[0]), action_names).repeat(2),
+            observation_shape=MatrixGame.OBS_SHAPE,
+            state_shape=(MatrixGame.STATE_SIZE,),
+        )
+        self.current_step = 0
+        self.payoffs = payoff_matrix
+
+    def reset(self):
+        self.current_step = 0
+        return self.get_observation(), self.get_state()
+
+    def get_observation(self):
+        return Observation(
+            np.array([[self.current_step]] * MatrixGame.N_AGENTS, np.float32),
+            self.available_actions(),
+        )
+
+    def step(self, action):
+        action = list(action)
+        self.current_step += 1
+        return Step(self.get_observation(), self.get_state(), self.payoffs[action[0]][action[1]], True)
+
+    def render(self):
+        return
+
+    def get_state(self):
+        return State(np.zeros((MatrixGame.STATE_SIZE,), np.float32))
+
+    def seed(self, seed_value):
+        return

multi_agent_rlenv-3.7.1/src/marlenv/wrappers/state_counter.py

@@ -0,0 +1,35 @@
+from typing import Sequence
+from marlenv import RLEnvWrapper, MARLEnv, Space, State
+import numpy as np
+from dataclasses import dataclass
+
+
+@dataclass
+class StateCounter[S: Space](RLEnvWrapper[S]):
+    def __init__(self, wrapped: MARLEnv[S]):
+        super().__init__(wrapped)
+        self._per_agent = [set[int]() for _ in range(self.n_agents)]
+        self._joint = set[int]()
+
+    def _register(self, state: State):
+        self._joint.add(hash(state))
+        eh = hash(state.extras.tobytes())
+        for i in range(self.n_agents):
+            agent_data = state.data[i * self.agent_state_size : (i + 1) * self.agent_state_size]
+            h = hash((agent_data.tobytes(), eh))
+            self._per_agent[i].add(h)
+
+    def step(self, action: np.ndarray | Sequence):
+        step = super().step(action)
+        self._register(step.state)
+        if step.is_terminal:
+            step.info = step.info | {
+                "joint-count": len(self._joint),
+                **{f"agent-{i}-count": len(agent_set) for i, agent_set in enumerate(self._per_agent)},
+            }
+        return step
+
+    def reset(self):
+        obs, state = super().reset()
+        self._register(state)
+        return obs, state

{multi_agent_rlenv-3.7.0 → multi_agent_rlenv-3.7.1}/tests/test_adapters.py

@@ -114,7 +114,7 @@ def _check_env_3m(env):
 
 @pytest.mark.skipif(skip_smac, reason="SMAC is not installed")
 def test_smac_from_class():
-    from smac.env import StarCraft2Env  # type: ignore[import]
+    from smac.env import StarCraft2Env  # pyright: ignore[reportMissingImports]
     from marlenv.adapters import SMAC
 
     env = SMAC(StarCraft2Env("3m"))