gym-csle-stopping-game 0.9.24__py3-none-any.whl

gym_csle_stopping_game/envs/stopping_game_env.py

@@ -0,0 +1,393 @@
+import random
+from typing import Tuple, Dict, List, Any, Union
+import numpy as np
+import numpy.typing as npt
+import time
+import math
+import csle_common.constants.constants as constants
+from csle_common.dao.simulation_config.base_env import BaseEnv
+from csle_common.dao.simulation_config.simulation_trace import SimulationTrace
+from gym_csle_stopping_game.util.stopping_game_util import StoppingGameUtil
+from gym_csle_stopping_game.dao.stopping_game_config import StoppingGameConfig
+from gym_csle_stopping_game.dao.stopping_game_state import StoppingGameState
+import gym_csle_stopping_game.constants.constants as env_constants
+
+
+class StoppingGameEnv(BaseEnv):
+    """
+    OpenAI Gym Env for the csle-stopping-game
+    """
+
+    def __init__(self, config: StoppingGameConfig):
+        """
+        Initializes the environment
+
+        :param config: the environment configuration
+        """
+        self.config = config
+
+        # Initialize environment state
+        self.state = StoppingGameState(b1=self.config.b1, L=self.config.L)
+        # Setup spaces
+        self.attacker_observation_space = self.config.attacker_observation_space()
+        self.defender_observation_space = self.config.defender_observation_space()
+        self.attacker_action_space = self.config.attacker_action_space()
+        self.defender_action_space = self.config.defender_action_space()
+
+        self.action_space = self.defender_action_space
+        self.observation_space = self.defender_observation_space
+
+        # Setup traces
+        self.traces: List[SimulationTrace] = []
+        self.trace = SimulationTrace(simulation_env=self.config.env_name)
+
+        # Reset
+        self.reset()
+        super().__init__()
+
+    def step(self, action_profile: Tuple[int, Tuple[npt.NDArray[Any], int]]) \
+            -> Tuple[Tuple[npt.NDArray[Any], npt.NDArray[Any]], Tuple[int, int], bool, bool, Dict[str, Any]]:
+        """
+        Takes a step in the environment by executing the given action
+
+        :param action_profile: the actions to take (both players actions
+        :return: (obs, reward, terminated, truncated, info)
+        """
+
+        # Setup initial values
+        a1, a2_profile = action_profile
+        pi2, a2 = a2_profile
+        assert pi2.shape[0] == len(self.config.S)
+        assert pi2.shape[1] == len(self.config.A2)
+        done = False
+        info: Dict[str, Any] = {}
+
+        o = max(self.config.O)
+        if self.state.s == 2:
+            done = True
+            r = 0
+        else:
+            # Compute r, s', b',o'
+            r = self.config.R[self.state.l - 1][a1][a2][self.state.s]
+            self.state.s = StoppingGameUtil.sample_next_state(l=self.state.l, a1=a1, a2=a2, T=self.config.T,
+                                                              S=self.config.S, s=self.state.s)
+            o = StoppingGameUtil.sample_next_observation(Z=self.config.Z,
+                                                         O=self.config.O, s_prime=self.state.s)
+            if self.config.compute_beliefs:
+                self.state.b = StoppingGameUtil.next_belief(o=o, a1=a1, b=self.state.b, pi2=pi2,
+                                                            config=self.config,
+                                                            l=self.state.l, a2=a2)
+
+            # Update stops remaining
+            self.state.l = self.state.l - a1
+
+        # Update time-step
+        self.state.t += 1
+
+        # Populate info dict
+        info[env_constants.ENV_METRICS.STOPS_REMAINING] = self.state.l
+        info[env_constants.ENV_METRICS.STATE] = self.state.s
+        info[env_constants.ENV_METRICS.DEFENDER_ACTION] = a1
+        info[env_constants.ENV_METRICS.ATTACKER_ACTION] = a2
+        info[env_constants.ENV_METRICS.OBSERVATION] = o
+        info[env_constants.ENV_METRICS.TIME_STEP] = self.state.t
+
+        # Get observations
+        attacker_obs = self.state.attacker_observation()
+        defender_obs = self.state.defender_observation()
+
+        # Log trace
+        if self.config.save_trace:
+            self.trace.defender_rewards.append(r)
+            self.trace.attacker_rewards.append(-r)
+            self.trace.attacker_actions.append(a2)
+            self.trace.defender_actions.append(a1)
+            self.trace.infos.append(info)
+            self.trace.states.append(self.state.s)
+            self.trace.beliefs.append(self.state.b[1])
+            self.trace.infrastructure_metrics.append(o)
+            if not done:
+                self.trace.attacker_observations.append(attacker_obs)
+                self.trace.defender_observations.append(defender_obs)
+
+        # Populate info
+        info = self._info(info)
+
+        return (defender_obs, attacker_obs), (r, -r), done, done, info
+
+    def mean(self, prob_vector):
+        """
+        Utility function for getting the mean of a vector
+
+        :param prob_vector: the vector to take the mean of
+        :return: the mean
+        """
+        m = 0
+        for i in range(len(prob_vector)):
+            m += prob_vector[i] * i
+        return m
+
+    def weighted_intrusion_prediction_distance(self, intrusion_start: int, first_stop: int):
+        """
+        Computes the weighted intrusion start time prediction distance (Wang, Hammar, Stadler, 2022)
+
+        :param intrusion_start: the intrusion start time
+        :param first_stop: the predicted start time
+        :return: the weighted distance
+        """
+        if first_stop <= intrusion_start:
+            return 1 - (10 / 10)
+        else:
+            return 1 - (min(10, (first_stop - (intrusion_start + 1))) / 2) / 10
+
+    def _info(self, info: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Adds the cumulative reward and episode length to the info dict
+
+        :param info: the info dict to update
+        :return: the updated info dict
+        """
+        R = 0
+        for i in range(len(self.trace.defender_rewards)):
+            R += self.trace.defender_rewards[i] * math.pow(self.config.gamma, i)
+        info[env_constants.ENV_METRICS.RETURN] = sum(self.trace.defender_rewards)
+        info[env_constants.ENV_METRICS.TIME_HORIZON] = len(self.trace.defender_actions)
+        stop = self.config.L
+        for i in range(1, self.config.L + 1):
+            info[f"{env_constants.ENV_METRICS.STOP}_{i}"] = len(self.trace.states)
+        for i in range(len(self.trace.defender_actions)):
+            if self.trace.defender_actions[i] == 1:
+                info[f"{env_constants.ENV_METRICS.STOP}_{stop}"] = i
+                stop -= 1
+        intrusion_start = len(self.trace.defender_actions)
+        for i in range(len(self.trace.attacker_actions)):
+            if self.trace.attacker_actions[i] == 1:
+                intrusion_start = i
+                break
+        intrusion_end = len(self.trace.attacker_actions)
+        info[env_constants.ENV_METRICS.INTRUSION_START] = intrusion_start
+        info[env_constants.ENV_METRICS.INTRUSION_END] = intrusion_end
+        info[env_constants.ENV_METRICS.START_POINT_CORRECT] = \
+            int(intrusion_start == (info[f"{env_constants.ENV_METRICS.STOP}_1"] + 1))
+        info[env_constants.ENV_METRICS.WEIGHTED_INTRUSION_PREDICTION_DISTANCE] = \
+            self.weighted_intrusion_prediction_distance(intrusion_start=intrusion_start,
+                                                        first_stop=info[f"{env_constants.ENV_METRICS.STOP}_1"])
+        info[env_constants.ENV_METRICS.INTRUSION_LENGTH] = intrusion_end - intrusion_start
+        upper_bound_return = 0
+        defender_baseline_stop_on_first_alert_return = 0
+        upper_bound_stops_remaining = self.config.L
+        defender_baseline_stop_on_first_alert_stops_remaining = self.config.L
+        for i in range(len(self.trace.states)):
+            if defender_baseline_stop_on_first_alert_stops_remaining > 0:
+                if self.trace.infrastructure_metrics[i] > 0:
+                    defender_baseline_stop_on_first_alert_return += \
+                        self.config.R[int(defender_baseline_stop_on_first_alert_stops_remaining) - 1][1][
+                            self.trace.attacker_actions[i]][self.trace.states[i]] * math.pow(self.config.gamma, i)
+                    defender_baseline_stop_on_first_alert_stops_remaining -= 1
+                else:
+                    defender_baseline_stop_on_first_alert_return += \
+                        self.config.R[int(defender_baseline_stop_on_first_alert_stops_remaining) - 1][0][
+                            self.trace.attacker_actions[i]][self.trace.states[i]] * math.pow(self.config.gamma, i)
+            if upper_bound_stops_remaining > 0:
+                if self.trace.states[i] == 0:
+                    r = self.config.R[int(upper_bound_stops_remaining) - 1][0][self.trace.attacker_actions[i]][
+                        self.trace.states[i]]
+                    upper_bound_return += r * math.pow(self.config.gamma, i)
+                else:
+                    r = self.config.R[int(upper_bound_stops_remaining) - 1][1][self.trace.attacker_actions[i]][
+                        self.trace.states[i]]
+                    upper_bound_return += r * math.pow(self.config.gamma, i)
+                    upper_bound_stops_remaining -= 1
+        info[env_constants.ENV_METRICS.AVERAGE_UPPER_BOUND_RETURN] = upper_bound_return
+        info[env_constants.ENV_METRICS.AVERAGE_DEFENDER_BASELINE_STOP_ON_FIRST_ALERT_RETURN] = \
+            defender_baseline_stop_on_first_alert_return
+        return info
+
+    def reset(self, seed: Union[None, int] = None, soft: bool = False, options: Union[Dict[str, Any], None] = None) \
+            -> Tuple[Tuple[npt.NDArray[Any], npt.NDArray[Any]], Dict[str, Any]]:
+        """
+        Resets the environment state, this should be called whenever step() returns <done>
+
+        :param seed: the random seed
+        :param soft: boolean flag indicating whether it is a soft reset or not
+        :param options: optional configuration parameters
+        :return: initial observation and info
+        """
+        super().reset(seed=seed)
+        self.state.reset()
+        if len(self.trace.attacker_rewards) > 0:
+            self.traces.append(self.trace)
+        self.trace = SimulationTrace(simulation_env=self.config.env_name)
+        attacker_obs = self.state.attacker_observation()
+        defender_obs = self.state.defender_observation()
+        if self.config.save_trace:
+            self.trace.attacker_observations.append(attacker_obs)
+            self.trace.defender_observations.append(defender_obs)
+        info: Dict[str, Any] = {}
+        info[env_constants.ENV_METRICS.STOPS_REMAINING] = self.state.l
+        info[env_constants.ENV_METRICS.STATE] = self.state.s
+        info[env_constants.ENV_METRICS.OBSERVATION] = 0
+        info[env_constants.ENV_METRICS.TIME_STEP] = self.state.t
+        return (defender_obs, attacker_obs), info
+
+    def render(self, mode: str = 'human'):
+        """
+        Renders the environment.  Supported rendering modes: (1) human; and (2) rgb_array
+
+        :param mode: the rendering mode
+        :return: True (if human mode) otherwise an rgb array
+        """
+        raise NotImplementedError("Rendering is not implemented for this environment")
+
+    def is_defense_action_legal(self, defense_action_id: int) -> bool:
+        """
+        Checks whether a defender action in the environment is legal or not
+
+        :param defense_action_id: the id of the action
+        :return: True or False
+        """
+        return True
+
+    def is_attack_action_legal(self, attack_action_id: int) -> bool:
+        """
+        Checks whether an attacker action in the environment is legal or not
+
+        :param attack_action_id: the id of the attacker action
+        :return: True or False
+        """
+        return True
+
+    def get_traces(self) -> List[SimulationTrace]:
+        """
+        :return: the list of simulation traces
+        """
+        return self.traces
+
+    def reset_traces(self) -> None:
+        """
+        Resets the list of traces
+
+        :return: None
+        """
+        self.traces = []
+
+    def __checkpoint_traces(self) -> None:
+        """
+        Checkpoints agent traces
+        :return: None
+        """
+        ts = time.time()
+        SimulationTrace.save_traces(traces_save_dir=constants.LOGGING.DEFAULT_LOG_DIR,
+                                    traces=self.traces, traces_file=f"taus{ts}.json")
+
+    def set_model(self, model) -> None:
+        """
+        Sets the model. Useful when using RL frameworks where the stage policy is not easy to extract
+
+        :param model: the model
+        :return: None
+        """
+        self.model = model
+
+    def set_state(self, state: Union[StoppingGameState, int, Tuple[int, int]]) -> None:
+        """
+        Sets the state. Allows to simulate samples from specific states
+
+        :param state: the state
+        :return: None
+        """
+        if isinstance(state, StoppingGameState):
+            self.state = state
+        elif type(state) is int or type(state) is np.int64:
+            self.state.s = state
+            self.state.l = self.config.L
+        elif type(state) is tuple:
+            self.state.s = state[0]
+            self.state.l = state[1]
+        else:
+            raise ValueError(f"state: {state} not valid")
+
+    def is_state_terminal(self, state: Union[StoppingGameState, int, Tuple[int, int]]) -> bool:
+        """
+        Checks whether a given state is terminal or not
+
+        :param state: the state
+        :return: True if terminal, else false
+        """
+        if isinstance(state, StoppingGameState):
+            return state.s == 2
+        elif type(state) is int or type(state) is np.int64:
+            return state == 2
+        elif type(state) is tuple:
+            return state[0] == 2
+        else:
+            raise ValueError(f"state: {state} not valid")
+
+    def get_observation_from_history(self, history: List[int], pi2: npt.NDArray[Any], l: int) -> List[Any]:
+        """
+        Utility method to get a hidden observation based on a history
+
+        :param history: the history to get the observation from
+        :param pi2: the attacker stage strategy
+        :param l: the number of stops remaining
+        :return: the observation
+        """
+        if not history:
+            raise ValueError("History must not be empty")
+        return [history[-1]]
+
+    def generate_random_particles(self, o: int, num_particles: int) -> List[int]:
+        """
+        Generates a random list of state particles from a given observation
+
+        :param o: the latest observation
+        :param num_particles: the number of particles to generate
+        :return: the list of random particles
+        """
+        particles = []
+        for i in range(num_particles):
+            particles.append(random.choice([0, 1]))
+        return particles
+
+    def manual_play(self) -> None:
+        """
+        An interactive loop to test the environment manually
+
+        :return: None
+        """
+        done = False
+        while True:
+            raw_input = input("> ")
+            raw_input = raw_input.strip()
+            if raw_input == "help":
+                print("Enter an action id to execute the action, "
+                      "press R to reset,"
+                      "press S to print the state, press A to print the actions, "
+                      "press D to check if done"
+                      "press H to print the history of actions")
+            elif raw_input == "A":
+                print(f"Attacker space: {self.action_space}")
+            elif raw_input == "S":
+                print(self.state)
+            elif raw_input == "D":
+                print(done)
+            elif raw_input == "H":
+                print(self.trace)
+            elif raw_input == "R":
+                print("Resetting the state")
+                self.reset()
+            else:
+                action_profile = raw_input
+                parts = action_profile.split(",")
+                a1 = int(parts[0])
+                a2 = int(parts[1])
+                stage_policy = []
+                for s in self.config.S:
+                    if s != 2:
+                        dist = [0.0, 0.0]
+                        dist[a2] = 1.0
+                        stage_policy.append(dist)
+                    else:
+                        stage_policy.append([0.5, 0.5])
+                pi2 = np.array(stage_policy)
+                _, _, done, _, _ = self.step(action_profile=(a1, (pi2, a2)))

gym_csle_stopping_game/envs/stopping_game_mdp_attacker_env.py

@@ -0,0 +1,282 @@
+from typing import Tuple, List, Dict, Any, Union
+import numpy as np
+import numpy.typing as npt
+import torch
+import math
+from csle_common.dao.simulation_config.base_env import BaseEnv
+from csle_common.dao.training.mixed_multi_threshold_stopping_policy import MixedMultiThresholdStoppingPolicy
+from gym_csle_stopping_game.dao.stopping_game_attacker_mdp_config import StoppingGameAttackerMdpConfig
+from csle_common.dao.simulation_config.simulation_trace import SimulationTrace
+from gym_csle_stopping_game.util.stopping_game_util import StoppingGameUtil
+import gym_csle_stopping_game.constants.constants as env_constants
+from gym_csle_stopping_game.envs.stopping_game_env import StoppingGameEnv
+
+
+class StoppingGameMdpAttackerEnv(BaseEnv):
+    """
+    OpenAI Gym Env for the MDP of the attacker when facing a static defender
+    """
+
+    def __init__(self, config: StoppingGameAttackerMdpConfig):
+        """
+        Initializes the environment
+
+        :param config: the configuration of the environment
+        """
+        self.config = config
+        self.stopping_game_env: StoppingGameEnv = StoppingGameEnv(config=self.config.stopping_game_config)
+
+        # Setup spaces
+        self.observation_space = self.config.stopping_game_config.attacker_observation_space()
+        self.action_space = self.config.stopping_game_config.attacker_action_space()
+
+        # Setup static defender
+        self.static_defender_strategy = self.config.defender_strategy
+
+        # Setup Config
+        self.viewer: Union[None, Any] = None
+        self.metadata = {
+            'render.modes': ['human', 'rgb_array'],
+            'video.frames_per_second': 50  # Video rendering speed
+        }
+
+        self.latest_defender_obs: Union[None, List[Any], npt.NDArray[Any]] = None
+        self.latest_attacker_obs: Union[None, List[Any], npt.NDArray[Any]] = None
+        self.model: Union[None, Any] = None
+
+        # Reset
+        self.reset()
+        super().__init__()
+
+    def step(self, pi2: Union[npt.NDArray[Any], int, float, int, np.float64]) \
+            -> Tuple[npt.NDArray[Any], int, bool, bool, Dict[str, Any]]:
+        """
+        Takes a step in the environment by executing the given action
+
+        :param pi2: attacker stage policy
+        :return: (obs, reward, terminated, truncated, info)
+        """
+        if (type(pi2) is int or type(pi2) is float or type(pi2) is np.int64 or type(pi2) is np.int32 # type: ignore
+                or type(pi2) is np.float64): # type: ignore
+            a2 = pi2
+            if self.latest_attacker_obs is None:
+                raise ValueError("Attacker observation is None")
+            pi2 = self.calculate_stage_policy(o=list(self.latest_attacker_obs), a2=int(a2))
+        else:
+            if self.model is not None:
+                if self.latest_attacker_obs is None:
+                    raise ValueError("Attacker observation is None")
+                pi2 = self.calculate_stage_policy(o=list(self.latest_attacker_obs))
+                a2 = StoppingGameUtil.sample_attacker_action(pi2=pi2, s=self.stopping_game_env.state.s)
+            else:
+                pi2 = np.array(pi2)
+                try:
+                    if self.latest_attacker_obs is None:
+                        raise ValueError("Attacker observation is None")
+                    pi2 = self.calculate_stage_policy(o=list(self.latest_attacker_obs))
+                except Exception:
+                    pass
+                a2 = StoppingGameUtil.sample_attacker_action(pi2=pi2, s=self.stopping_game_env.state.s)
+
+        # a2 = pi2
+        # pi2 = np.array([
+        #     [0.5,0.5],
+        #     [0.5,0.5],
+        #     [0.5,0.5]
+        # ])
+        assert pi2.shape[0] == len(self.config.stopping_game_config.S)
+        assert pi2.shape[1] == len(self.config.stopping_game_config.A1)
+
+        # Get defender action from static strategy
+        a1 = self.static_defender_strategy.action(o=self.latest_defender_obs)
+
+        # Step the game
+        o, r, d, _, info = self.stopping_game_env.step((int(a1), (pi2, int(a2))))
+        self.latest_defender_obs = o[0]
+        self.latest_attacker_obs = o[1]
+        attacker_obs = o[1]
+
+        info[env_constants.ENV_METRICS.RETURN] = -info[env_constants.ENV_METRICS.RETURN]
+        info[env_constants.ENV_METRICS.AVERAGE_UPPER_BOUND_RETURN] = \
+            -info[env_constants.ENV_METRICS.AVERAGE_UPPER_BOUND_RETURN]
+
+        return attacker_obs, r[1], d, d, info
+
+    def reset(self, seed: Union[int, None] = None, soft: bool = False, options: Union[Dict[str, Any], None] = None) \
+            -> Tuple[npt.NDArray[Any], Dict[str, Any]]:
+        """
+        Resets the environment state, this should be called whenever step() returns <done>
+
+        :param seed: the random seed
+        :param soft: boolean flag indicating whether it is a soft reset or not
+        :param options: optional configuration parameters
+        :return: initial observation
+        """
+        o, _ = self.stopping_game_env.reset()
+        self.latest_defender_obs = o[0]
+        self.latest_attacker_obs = o[1]
+        attacker_obs = o[1]
+        info: Dict[str, Any] = {}
+        return attacker_obs, info
+
+    def set_model(self, model) -> None:
+        """
+        Sets the model. Useful when using RL frameworks where the stage policy is not easy to extract
+
+        :param model: the model
+        :return: None
+        """
+        self.model = model
+
+    def set_state(self, state: Any) -> None:
+        """
+        Sets the state. Allows to simulate samples from specific states
+
+        :param state: the state
+        :return: None
+        """
+        self.stopping_game_env.set_state(state=state)
+
+    def calculate_stage_policy(self, o: List[Any], a2: int = 0) -> npt.NDArray[Any]:
+        """
+        Calculates the stage policy of a given model and observation
+
+        :param o: the observation
+        :return: the stage policy
+        """
+        if self.model is None:
+            stage_policy = []
+            for s in self.config.stopping_game_config.S:
+                if s != 2:
+                    dist = [0.0, 0.0]
+                    dist[a2] = 1.0
+                    stage_policy.append(dist)
+                else:
+                    stage_policy.append([0.5, 0.5])
+            return np.array(stage_policy)
+        if isinstance(self.model, MixedMultiThresholdStoppingPolicy):
+            return np.array(self.model.stage_policy(o=o))
+        else:
+            b1 = o[1]
+            l = int(o[0])
+            stage_policy = []
+            for s in self.config.stopping_game_config.S:
+                if s != 2:
+                    o = [l, b1, s]
+                    stage_policy.append(self._get_attacker_dist(obs=o))
+                else:
+                    stage_policy.append([0.5, 0.5])
+            return np.array(stage_policy)
+
+    def _get_attacker_dist(self, obs: List[Any]) -> List[float]:
+        """
+        Utility function for getting the attacker's action distribution based on a given observation
+
+        :param obs: the given observation
+        :return:  the action distribution
+        """
+        np_obs = np.array([obs])
+        if self.model is None:
+            raise ValueError("Model is None")
+        actions, values, log_prob = self.model.policy.forward(obs=torch.tensor(np_obs).to(self.model.device))
+        action = actions[0]
+        if action == 1:
+            stop_prob = math.exp(log_prob)
+        else:
+            stop_prob = 1 - math.exp(log_prob)
+        return [1 - stop_prob, stop_prob]
+
+    def render(self, mode: str = 'human'):
+        """
+        Renders the environment.  Supported rendering modes: (1) human; and (2) rgb_array
+
+        :param mode: the rendering mode
+        :return: True (if human mode) otherwise an rgb array
+        """
+        raise NotImplementedError("Rendering is not implemented for this environment")
+
+    def is_defense_action_legal(self, defense_action_id: int) -> bool:
+        """
+        Checks whether a defender action in the environment is legal or not
+
+        :param defense_action_id: the id of the action
+        :return: True or False
+        """
+        return True
+
+    def is_attack_action_legal(self, attack_action_id: int) -> bool:
+        """
+        Checks whether an attacker action in the environment is legal or not
+
+        :param attack_action_id: the id of the attacker action
+        :return: True or False
+        """
+        return True
+
+    def get_traces(self) -> List[SimulationTrace]:
+        """
+        :return: the list of simulation traces
+        """
+        return self.stopping_game_env.get_traces()
+
+    def reset_traces(self) -> None:
+        """
+        Resets the list of traces
+
+        :return: None
+        """
+        return self.stopping_game_env.reset_traces()
+
+    def generate_random_particles(self, o: int, num_particles: int) -> List[int]:
+        """
+        Generates a random list of state particles from a given observation
+
+        :param o: the latest observation
+        :param num_particles: the number of particles to generate
+        :return: the list of random particles
+        """
+        return self.stopping_game_env.generate_random_particles(o=o, num_particles=num_particles)
+
+    def get_actions_from_particles(self, particles: List[int], t: int, observation: int,
+                                   verbose: bool = False) -> List[int]:
+        """
+        Prunes the set of actiosn based on the current particle set
+
+        :param particles: the set of particles
+        :param t: the current time step
+        :param observation: the latest observation
+        :param verbose: boolean flag indicating whether logging should be verbose or not
+        :return: the list of pruned actions
+        """
+        return list(self.config.stopping_game_config.A2)
+
+    def manual_play(self) -> None:
+        """
+        An interactive loop to test the environment manually
+
+        :return: None
+        """
+        done = False
+        while True:
+            raw_input = input("> ")
+            raw_input = raw_input.strip()
+            if raw_input == "help":
+                print("Enter an action id to execute the action, "
+                      "press R to reset,"
+                      "press S to print the state, press A to print the actions, "
+                      "press D to check if done"
+                      "press H to print the history of actions")
+            elif raw_input == "A":
+                print(f"Action space: {self.action_space}")
+            elif raw_input == "S":
+                print(self.stopping_game_env.state)
+            elif raw_input == "D":
+                print(done)
+            elif raw_input == "H":
+                print(self.stopping_game_env.trace)
+            elif raw_input == "R":
+                print("Resetting the state")
+                self.reset()
+            else:
+                action_idx = int(raw_input)
+                _, _, done, _, _ = self.step(pi2=action_idx)