gymcts 1.2.0__tar.gz → 1.2.1__tar.gz

{gymcts-1.2.0/src/gymcts.egg-info → gymcts-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gymcts
-Version: 1.2.0
+Version: 1.2.1
 Summary: A minimalistic implementation of the Monte Carlo Tree Search algorithm for planning problems fomulated as gymnaisum reinforcement learning environments.
 Author: Alexander Nasuta
 Author-email: Alexander Nasuta <alexander.nasuta@wzl-iqs.rwth-aachen.de>
@@ -47,7 +47,7 @@ Requires-Dist: graph-matrix-jsp-env; extra == "examples"
 Requires-Dist: graph-jsp-env; extra == "examples"
 Provides-Extra: dev
 Requires-Dist: jsp-instance-utils; extra == "dev"
-Requires-Dist: graph-matrix-jsp-env; extra == "dev"
+Requires-Dist: graph-matrix-jsp-env>=0.3.0; extra == "dev"
 Requires-Dist: graph-jsp-env; extra == "dev"
 Requires-Dist: JSSEnv; extra == "dev"
 Requires-Dist: pip-tools; extra == "dev"
@@ -59,21 +59,24 @@ Requires-Dist: stable_baselines3; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
 Requires-Dist: myst-parser; extra == "dev"
 Requires-Dist: sphinx-autobuild; extra == "dev"
+Requires-Dist: sphinx-copybutton; extra == "dev"
 Requires-Dist: furo; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Requires-Dist: sphinx-copybutton; extra == "dev"
 Requires-Dist: nbsphinx; extra == "dev"
+Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: jupytext; extra == "dev"
 Requires-Dist: jupyter; extra == "dev"
+Requires-Dist: typing_extensions>=4.12.0; extra == "dev"
 Dynamic: license-file
 
 # Graph Matrix Job Shop Env
 
 A Monte Carlo Tree Search Implementation for Gymnasium-style Environments.
 
-- Github: [GYMCTS on Github](https://github.com/Alexander-Nasuta/GraphMatrixJobShopEnv)
-- Pypi: [GYMCTS on PyPi](https://pypi.org/project/graph-matrix-jsp-env/)
-- Documentation: [GYMCTS Docs](https://graphmatrixjobshopenv.readthedocs.io/en/latest/)
+- Github: [GYMCTS on Github](https://github.com/Alexander-Nasuta/gymcts)
+- Pypi: [GYMCTS on PyPi](https://pypi.org/project/gymcts/)
+- Documentation: [GYMCTS Docs](https://gymcts.readthedocs.io/en/latest/)
 
 ## Description
 
@@ -101,22 +104,26 @@ The usage of a MCTS agent can roughly organised into the following steps:
 - Render the solution
 
 The GYMCTS package provides a two types of wrappers for Gymnasium-style environments:
-- `NaiveSoloMCTSGymEnvWrapper`: A wrapper that uses deepcopies of the environment to save a snapshot of the environment state for each node in the MCTS tree.
-- `DeterministicSoloMCTSGymEnvWrapper`: A wrapper that saves the action sequence that lead to the current state in the MCTS node.
+- `DeepCopyMCTSGymEnvWrapper`: A wrapper that uses deepcopies of the environment to save a snapshot of the environment state for each node in the MCTS tree.
+- `ActionHistoryMCTSGymEnvWrapper`: A wrapper that saves the action sequence that lead to the current state in the MCTS node.
 
-These wrappers can be used with the `SoloMCTSAgent` to solve the environment. 
-The wrapper implement methods that are required by the `SoloMCTSAgent` to interact with the environment.
+These wrappers can be used with the `GymctsAgent` to solve the environment. 
+The wrapper implement methods that are required by the `GymctsAgent` to interact with the environment.
 GYMCTS is designed to use a single environment instance and reconstructing the environment state form a state snapshot, when needed.
 
 NOTE: MCTS works best when the return of an episode is in the range of [-1, 1]. Please adjust the reward function of the environment accordingly (or change the ubc-scaling parameter of the MCTS agent).
 Adjusting the reward function of the environment is easily done with a [NormalizeReward](https://gymnasium.farama.org/api/wrappers/reward_wrappers/#gymnasium.wrappers.NormalizeReward) or [TransformReward](https://gymnasium.farama.org/api/wrappers/reward_wrappers/#gymnasium.wrappers.TransformReward) Wrapper.
+```python
+env = NormalizeReward(env, gamma=0.99, epsilon=1e-8)
+```
 
-NormalizeReward(env, gamma=0.99, epsilon=1e-8)
-env = TransformReward(env, lambda r: r / 36)
-### FrozenLake Example (NaiveSoloMCTSGymEnvWrapper)
+```python
+env = TransformReward(env, lambda r: r / n_steps_per_episode)
+```
+### FrozenLake Example (DeepCopyMCTSGymEnvWrapper)
 
 A minimal example of how to use the package with the FrozenLake environment and the NaiveSoloMCTSGymEnvWrapper is provided in the following code snippet below.
-The NaiveSoloMCTSGymEnvWrapper can be used with non-deterministic environments, such as the FrozenLake environment with slippery ice.
+The DeepCopyMCTSGymEnvWrapper can be used with non-deterministic environments, such as the FrozenLake environment with slippery ice.
 
 ```python
 import gymnasium as gym
@@ -135,7 +142,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=True, render_mode="ansi")
     env.reset()
 
-    # 1. wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # 1. wrap the environment with the deep copy wrapper or a custom gymcts wrapper
     env = DeepCopyMCTSGymEnvWrapper(env)
 
     # 2. create the agent
@@ -158,7 +165,7 @@ if __name__ == '__main__':
 
     # 5. print the solution
     # read the solution from the info provided by the RecordEpisodeStatistics wrapper 
-    # (that NaiveSoloMCTSGymEnvWrapper uses internally)
+    # (that DeepCopyMCTSGymEnvWrapper uses internally)
     episode_length = info["episode"]["l"]
     episode_return = info["episode"]["r"]
 
@@ -251,7 +258,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=False, render_mode="rgb_array")
     env.reset()
 
-    # 1. wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # 1. wrap the environment with the deep copy wrapper or a custom gymcts wrapper
     env = DeepCopyMCTSGymEnvWrapper(env)
 
     # 2. create the agent
@@ -280,7 +287,7 @@ if __name__ == '__main__':
     env.close()
 
     # 5. print the solution
-    # read the solution from the info provided by the RecordEpisodeStatistics wrapper (that NaiveSoloMCTSGymEnvWrapper wraps internally)
+    # read the solution from the info provided by the RecordEpisodeStatistics wrapper (that DeepCopyMCTSGymEnvWrapper wraps internally)
     episode_length = info["episode"]["l"]
     episode_return = info["episode"]["r"]
 
@@ -321,13 +328,13 @@ import gymnasium as gym
 from graph_jsp_env.disjunctive_graph_jsp_env import DisjunctiveGraphJspEnv
 from jsp_instance_utils.instances import ft06, ft06_makespan
 
-from gymcts.gymcts_agent import SoloMCTSAgent
-from gymcts.gymcts_gym_env import SoloMCTSGymEnv
+from gymcts.gymcts_agent import GymctsAgent
+from gymcts.gymcts_env_abc import GymctsABC
 
 from gymcts.logger import log
 
 
-class GraphJspGYMCTSWrapper(SoloMCTSGymEnv, gym.Wrapper):
+class GraphJspGYMCTSWrapper(GymctsABC, gym.Wrapper):
 
     def __init__(self, env: DisjunctiveGraphJspEnv):
         gym.Wrapper.__init__(self, env)
@@ -378,7 +385,7 @@ if __name__ == '__main__':
 
     env = GraphJspGYMCTSWrapper(env)
 
-    agent = SoloMCTSAgent(
+    agent = GymctsAgent(
         env=env,
         clear_mcts_tree_after_step=True,
         render_tree_after_step=True,
@@ -421,7 +428,6 @@ import gymnasium as gym
 
 from gymcts.gymcts_agent import GymctsAgent
 from gymcts.gymcts_action_history_wrapper import ActionHistoryMCTSGymEnvWrapper
-from gymcts.gymcts_deepcopy_wrapper import DeepCopyMCTSGymEnvWrapper
 
 from gymcts.logger import log
 
@@ -434,7 +440,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=False, render_mode="ansi")
     env.reset()
 
-    # wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # wrap the environment with the wrapper or a custom gymcts wrapper
     env = ActionHistoryMCTSGymEnvWrapper(env)
 
     # create the agent
@@ -505,11 +511,11 @@ clone the repository in your favorite code editor (for example PyCharm, VSCode,
 
 using https:
 ```shell
-git clone https://github.com/Alexander-Nasuta/todo
+git clone https://github.com/Alexander-Nasuta/gymcts.git
 ```
 or by using the GitHub CLI:
 ```shell
-gh repo clone Alexander-Nasuta/todo
+gh repo clone Alexander-Nasuta/gymcts
 ```
 
 if you are using PyCharm, I recommend doing the following additional steps:
@@ -518,9 +524,6 @@ if you are using PyCharm, I recommend doing the following additional steps:
 - mark the `tests` folder as test root (by right-clicking on the folder and selecting `Mark Directory as` -> `Test Sources Root`)
 - mark the `resources` folder as resources root (by right-clicking on the folder and selecting `Mark Directory as` -> `Resources Root`)
 
-at the end your project structure should look like this:
-
-todo
 
 ### Create a Virtual Environment (optional)
 
@@ -586,12 +589,6 @@ For testing with `tox` run the following command:
 tox
 ```
 
-Here is a screenshot of what the output might look like:
-
-![](https://github.com/Alexander-Nasuta/GraphMatrixJobShopEnv/raw/master/resources/tox-screenshot.png)
-
-Tox will run the tests in a separate environment and will also check if the requirements are installed correctly.
-
 ### Builing and Publishing the Project to PyPi 
 
 In order to publish the project to PyPi, the project needs to be built and then uploaded to PyPi.
@@ -630,7 +627,6 @@ sphinx-autobuild ./docs/source/ ./docs/build/html/
 This project features most of the extensions featured in this Tutorial: [Document Your Scientific Project With Markdown, Sphinx, and Read the Docs | PyData Global 2021](https://www.youtube.com/watch?v=qRSb299awB0).
 
 
-
 ## Contact
 
 If you have any questions or feedback, feel free to contact me via [email](mailto:alexander.nasuta@wzl-iqs.rwth-aachen.de) or open an issue on repository.

{gymcts-1.2.0 → gymcts-1.2.1}/README.md

@@ -2,9 +2,9 @@
 
 A Monte Carlo Tree Search Implementation for Gymnasium-style Environments.
 
-- Github: [GYMCTS on Github](https://github.com/Alexander-Nasuta/GraphMatrixJobShopEnv)
-- Pypi: [GYMCTS on PyPi](https://pypi.org/project/graph-matrix-jsp-env/)
-- Documentation: [GYMCTS Docs](https://graphmatrixjobshopenv.readthedocs.io/en/latest/)
+- Github: [GYMCTS on Github](https://github.com/Alexander-Nasuta/gymcts)
+- Pypi: [GYMCTS on PyPi](https://pypi.org/project/gymcts/)
+- Documentation: [GYMCTS Docs](https://gymcts.readthedocs.io/en/latest/)
 
 ## Description
 
@@ -32,22 +32,26 @@ The usage of a MCTS agent can roughly organised into the following steps:
 - Render the solution
 
 The GYMCTS package provides a two types of wrappers for Gymnasium-style environments:
-- `NaiveSoloMCTSGymEnvWrapper`: A wrapper that uses deepcopies of the environment to save a snapshot of the environment state for each node in the MCTS tree.
-- `DeterministicSoloMCTSGymEnvWrapper`: A wrapper that saves the action sequence that lead to the current state in the MCTS node.
+- `DeepCopyMCTSGymEnvWrapper`: A wrapper that uses deepcopies of the environment to save a snapshot of the environment state for each node in the MCTS tree.
+- `ActionHistoryMCTSGymEnvWrapper`: A wrapper that saves the action sequence that lead to the current state in the MCTS node.
 
-These wrappers can be used with the `SoloMCTSAgent` to solve the environment. 
-The wrapper implement methods that are required by the `SoloMCTSAgent` to interact with the environment.
+These wrappers can be used with the `GymctsAgent` to solve the environment. 
+The wrapper implement methods that are required by the `GymctsAgent` to interact with the environment.
 GYMCTS is designed to use a single environment instance and reconstructing the environment state form a state snapshot, when needed.
 
 NOTE: MCTS works best when the return of an episode is in the range of [-1, 1]. Please adjust the reward function of the environment accordingly (or change the ubc-scaling parameter of the MCTS agent).
 Adjusting the reward function of the environment is easily done with a [NormalizeReward](https://gymnasium.farama.org/api/wrappers/reward_wrappers/#gymnasium.wrappers.NormalizeReward) or [TransformReward](https://gymnasium.farama.org/api/wrappers/reward_wrappers/#gymnasium.wrappers.TransformReward) Wrapper.
+```python
+env = NormalizeReward(env, gamma=0.99, epsilon=1e-8)
+```
 
-NormalizeReward(env, gamma=0.99, epsilon=1e-8)
-env = TransformReward(env, lambda r: r / 36)
-### FrozenLake Example (NaiveSoloMCTSGymEnvWrapper)
+```python
+env = TransformReward(env, lambda r: r / n_steps_per_episode)
+```
+### FrozenLake Example (DeepCopyMCTSGymEnvWrapper)
 
 A minimal example of how to use the package with the FrozenLake environment and the NaiveSoloMCTSGymEnvWrapper is provided in the following code snippet below.
-The NaiveSoloMCTSGymEnvWrapper can be used with non-deterministic environments, such as the FrozenLake environment with slippery ice.
+The DeepCopyMCTSGymEnvWrapper can be used with non-deterministic environments, such as the FrozenLake environment with slippery ice.
 
 ```python
 import gymnasium as gym
@@ -66,7 +70,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=True, render_mode="ansi")
     env.reset()
 
-    # 1. wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # 1. wrap the environment with the deep copy wrapper or a custom gymcts wrapper
     env = DeepCopyMCTSGymEnvWrapper(env)
 
     # 2. create the agent
@@ -89,7 +93,7 @@ if __name__ == '__main__':
 
     # 5. print the solution
     # read the solution from the info provided by the RecordEpisodeStatistics wrapper 
-    # (that NaiveSoloMCTSGymEnvWrapper uses internally)
+    # (that DeepCopyMCTSGymEnvWrapper uses internally)
     episode_length = info["episode"]["l"]
     episode_return = info["episode"]["r"]
 
@@ -182,7 +186,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=False, render_mode="rgb_array")
     env.reset()
 
-    # 1. wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # 1. wrap the environment with the deep copy wrapper or a custom gymcts wrapper
     env = DeepCopyMCTSGymEnvWrapper(env)
 
     # 2. create the agent
@@ -211,7 +215,7 @@ if __name__ == '__main__':
     env.close()
 
     # 5. print the solution
-    # read the solution from the info provided by the RecordEpisodeStatistics wrapper (that NaiveSoloMCTSGymEnvWrapper wraps internally)
+    # read the solution from the info provided by the RecordEpisodeStatistics wrapper (that DeepCopyMCTSGymEnvWrapper wraps internally)
     episode_length = info["episode"]["l"]
     episode_return = info["episode"]["r"]
 
@@ -252,13 +256,13 @@ import gymnasium as gym
 from graph_jsp_env.disjunctive_graph_jsp_env import DisjunctiveGraphJspEnv
 from jsp_instance_utils.instances import ft06, ft06_makespan
 
-from gymcts.gymcts_agent import SoloMCTSAgent
-from gymcts.gymcts_gym_env import SoloMCTSGymEnv
+from gymcts.gymcts_agent import GymctsAgent
+from gymcts.gymcts_env_abc import GymctsABC
 
 from gymcts.logger import log
 
 
-class GraphJspGYMCTSWrapper(SoloMCTSGymEnv, gym.Wrapper):
+class GraphJspGYMCTSWrapper(GymctsABC, gym.Wrapper):
 
     def __init__(self, env: DisjunctiveGraphJspEnv):
         gym.Wrapper.__init__(self, env)
@@ -309,7 +313,7 @@ if __name__ == '__main__':
 
     env = GraphJspGYMCTSWrapper(env)
 
-    agent = SoloMCTSAgent(
+    agent = GymctsAgent(
         env=env,
         clear_mcts_tree_after_step=True,
         render_tree_after_step=True,
@@ -352,7 +356,6 @@ import gymnasium as gym
 
 from gymcts.gymcts_agent import GymctsAgent
 from gymcts.gymcts_action_history_wrapper import ActionHistoryMCTSGymEnvWrapper
-from gymcts.gymcts_deepcopy_wrapper import DeepCopyMCTSGymEnvWrapper
 
 from gymcts.logger import log
 
@@ -365,7 +368,7 @@ if __name__ == '__main__':
     env = gym.make('FrozenLake-v1', desc=None, map_name="4x4", is_slippery=False, render_mode="ansi")
     env.reset()
 
-    # wrap the environment with the naive wrapper or a custom gymcts wrapper
+    # wrap the environment with the wrapper or a custom gymcts wrapper
     env = ActionHistoryMCTSGymEnvWrapper(env)
 
     # create the agent
@@ -436,11 +439,11 @@ clone the repository in your favorite code editor (for example PyCharm, VSCode,
 
 using https:
 ```shell
-git clone https://github.com/Alexander-Nasuta/todo
+git clone https://github.com/Alexander-Nasuta/gymcts.git
 ```
 or by using the GitHub CLI:
 ```shell
-gh repo clone Alexander-Nasuta/todo
+gh repo clone Alexander-Nasuta/gymcts
 ```
 
 if you are using PyCharm, I recommend doing the following additional steps:
@@ -449,9 +452,6 @@ if you are using PyCharm, I recommend doing the following additional steps:
 - mark the `tests` folder as test root (by right-clicking on the folder and selecting `Mark Directory as` -> `Test Sources Root`)
 - mark the `resources` folder as resources root (by right-clicking on the folder and selecting `Mark Directory as` -> `Resources Root`)
 
-at the end your project structure should look like this:
-
-todo
 
 ### Create a Virtual Environment (optional)
 
@@ -517,12 +517,6 @@ For testing with `tox` run the following command:
 tox
 ```
 
-Here is a screenshot of what the output might look like:
-
-![](https://github.com/Alexander-Nasuta/GraphMatrixJobShopEnv/raw/master/resources/tox-screenshot.png)
-
-Tox will run the tests in a separate environment and will also check if the requirements are installed correctly.
-
 ### Builing and Publishing the Project to PyPi 
 
 In order to publish the project to PyPi, the project needs to be built and then uploaded to PyPi.
@@ -561,7 +555,6 @@ sphinx-autobuild ./docs/source/ ./docs/build/html/
 This project features most of the extensions featured in this Tutorial: [Document Your Scientific Project With Markdown, Sphinx, and Read the Docs | PyData Global 2021](https://www.youtube.com/watch?v=qRSb299awB0).
 
 
-
 ## Contact
 
 If you have any questions or feedback, feel free to contact me via [email](mailto:alexander.nasuta@wzl-iqs.rwth-aachen.de) or open an issue on repository.

{gymcts-1.2.0 → gymcts-1.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gymcts"
-version = "1.2.0"
+version = "1.2.1"
 description = "A minimalistic implementation of the Monte Carlo Tree Search algorithm for planning problems fomulated as gymnaisum reinforcement learning environments."
 readme = "README.md"
 authors = [{ name = "Alexander Nasuta", email = "alexander.nasuta@wzl-iqs.rwth-aachen.de" }]
@@ -32,7 +32,7 @@ examples = [
 ]
 dev = [
     "jsp-instance-utils",
-    "graph-matrix-jsp-env",
+    "graph-matrix-jsp-env>=0.3.0",
     "graph-jsp-env",
     "JSSEnv",
 
@@ -49,13 +49,16 @@ dev = [
     "myst-parser", # .md support for sphinx
     "sphinx-autobuild",
     #
+    "sphinx-copybutton", # for code copy buttons
     "furo", # cool theme
     "twine",
     "sphinx-copybutton", # for code copy buttons
     "nbsphinx", # for jupyter notebook support in sphinx
+    "pandoc",
 
     "jupytext", # converting .py examples to jupyter notebook jupytext --to notebook *.py
     "jupyter", # for jupyter notebook kernel
+    "typing_extensions>=4.12.0",
 ]
 
 [project.urls]

{gymcts-1.2.0 → gymcts-1.2.1}/setup.cfg

@@ -25,9 +25,6 @@ testing =
 	flake8>=3.9
 	tox>=3.24
 
-[options.package_data]
-phantomderopfa = py.typed
-
 [flake8]
 max-line-length = 160
 

{gymcts-1.2.0 → gymcts-1.2.1}/src/gymcts/colorful_console_utils.py

@@ -106,6 +106,18 @@ def wrap_with_color_codes(s: object, /, r: int | float, g: int | float, b: int |
 
 
 def wrap_evenly_spaced_color(s: Any, n_of_item: int, n_classes: int, c_map="rainbow") -> str:
+    """
+    Wraps a string with a color scale (a matplotlib c_map) based on the n_of_item and n_classes.
+    This function is used to color code the available actions in the MCTS tree visualisation.
+    The children of the MCTS tree are colored based on their action for a clearer visualisation.
+
+    :param s: the string (or object) to be wrapped. objects are converted to string (using the __str__ function).
+    :param n_of_item: the index of the item to be colored. In a mcts tree, this is the (parent-)action of the node.
+    :param n_classes: the number of classes (or items) to be colored. In a mcts tree, this is the number of available actions.
+    :param c_map: the colormap to be used (default is 'rainbow').
+                  The colormap can be any matplotlib colormap, e.g. 'viridis', 'plasma', 'inferno', 'magma', 'cividis'.
+    :return: a string that contains the color-codes (prefix and suffix) and the string s in between.
+    """
     if s is None or n_of_item is None or n_classes is None:
         return s
 
@@ -119,6 +131,16 @@ def wrap_evenly_spaced_color(s: Any, n_of_item: int, n_classes: int, c_map="rain
 
 
 def wrap_with_color_scale(s: str, value: float, min_val: float, max_val: float, c_map=None) -> str:
+    """
+    Wraps a string with a color scale (a matplotlib c_map) based on the value, min_val, and max_val.
+
+    :param s: the string to be wrapped
+    :param value: the value to be mapped to a color
+    :param min_val: the minimum value of the scale
+    :param max_val: the maximum value of the scale
+    :param c_map: the colormap to be used (default is 'rainbow')
+    :return:
+    """
     if s is None or min_val is None or max_val is None or min_val >= max_val:
         return s
 

{gymcts-1.2.0 → gymcts-1.2.1}/src/gymcts/gymcts_action_history_wrapper.py

@@ -1,8 +1,7 @@
 import random
-import copy
 
 import numpy as np
-from typing import TypeVar, Any, SupportsFloat, Callable
+from typing import Any, SupportsFloat, Callable
 import gymnasium as gym
 from gymnasium.core import WrapperActType, WrapperObsType
 from gymnasium.wrappers import RecordEpisodeStatistics
@@ -13,6 +12,21 @@ from gymcts.logger import log
 
 
 class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
+    """
+    A wrapper for gym environments that implements the GymctsABC interface.
+    It uses the action history as state representation.
+    Please note that this is not the most efficient way to implement the state representation.
+    It is supposed to be used to see if your use-case works well with the MCTS algorithm.
+    If it does, you can consider implementing all GymctsABC methods in a more efficient way.
+    The action history is a list of actions taken in the environment.
+    The state is represented as a list of actions taken in the environment.
+    The state is used to restore the environment using the load_state method.
+
+    It is supposed to be used to see if your use-case works well with the MCTS algorithm.
+    If it does, you can consider implementing all GymctsABC methods in a more efficient way.
+    """
+
+    # helper attributes for the wrapper
     _terminal_flag: bool = False
     _last_reward: SupportsFloat = 0
     _step_tuple: tuple[WrapperObsType, SupportsFloat, bool, bool, dict[str, Any]] = None
@@ -25,6 +39,17 @@ class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
             action_mask_fn: str | Callable[[gym.Env], np.ndarray] | None = None,
             buffer_length: int = 100,
     ):
+        """
+        A wrapper for gym environments that implements the GymctsABC interface.
+        It uses the action history as state representation.
+        Please note that this is not the most efficient way to implement the state representation.
+        It is supposed to be used to see if your use-case works well with the MCTS algorithm.
+        If it does, you can consider implementing all GymctsABC methods in a more efficient way.
+
+        :param env: the environment to wrap
+        :param action_mask_fn: a function that takes the environment as input and returns a mask of valid actions
+        :param buffer_length: the length of the buffer for recording episodes for determining their rollout returns
+        """
         # wrap with RecordEpisodeStatistics if it is not already wrapped
         env = RecordEpisodeStatistics(env, buffer_length=buffer_length)
 
@@ -48,6 +73,17 @@ class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
                 self._action_mask_fn = action_mask_fn
 
     def load_state(self, state: list[int]) -> None:
+        """
+        Loads the state of the environment. The state is a list of actions taken in the environment.
+
+        The environment is reset and all actions in the state are performed in order to restore the environment to the
+        same state.
+
+        This works only for deterministic environments!
+
+        :param state: the state to load
+        :return: None
+        """
         self.env.reset()
         self._wrapper_action_history = []
 
@@ -56,15 +92,30 @@ class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
             self._wrapper_action_history.append(action)
 
     def is_terminal(self) -> bool:
+        """
+        Returns True if the environment is in a terminal state, False otherwise.
+
+        :return:
+        """
         if not len(self.get_valid_actions()):
             return True
         else:
             return self._terminal_flag
 
     def action_masks(self) -> np.ndarray | None:
+        """
+        Returns the action masks for the environment. If the action_mask_fn is not set, it returns None.
+
+        :return:
+        """
         return self._action_mask_fn(self.env) if self._action_mask_fn is not None else None
 
     def get_valid_actions(self) -> list[int]:
+        """
+        Returns a list of valid actions for the current state of the environment.
+
+        :return: a list of valid actions
+        """
         if self._action_mask_fn is None:
             action_space: gym.spaces.Discrete = self.env.action_space  # Type hinting
             return list(range(action_space.n))
@@ -72,6 +123,12 @@ class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
             return [i for i, mask in enumerate(self.action_masks()) if mask]
 
     def rollout(self) -> float:
+        """
+        Performs a random rollout from the current state of the environment and returns the return (sum of rewards)
+        of the rollout.
+
+        :return: the return of the rollout
+        """
         log.debug("performing rollout")
         # random rollout
         # perform random valid action util terminal
@@ -92,11 +149,24 @@ class ActionHistoryMCTSGymEnvWrapper(GymctsABC, gym.Wrapper):
         return episode_return
 
     def get_state(self) -> list[int]:
+        """
+        Returns the current state of the environment. The state is a list of actions taken in the environment,
+        namely all action that have been taken in the environment so far (since the last reset).
+
+        :return: a list of actions taken in the environment
+        """
+
         return self._wrapper_action_history.copy()
 
     def step(
             self, action: WrapperActType
     ) -> tuple[WrapperObsType, SupportsFloat, bool, bool, dict[str, Any]]:
+        """
+        Performs a step in the environment. It adds the action to the action history and updates the terminal flag.
+
+        :param action: action to perform in the environment
+        :return: the step tuple of the environment (obs, reward, terminated, truncated, info)
+        """
         step_tuple = self.env.step(action)
         self._wrapper_action_history.append(action)
         obs, reward, terminated, truncated, info = step_tuple

{gymcts-1.2.0 → gymcts-1.2.1}/src/gymcts/gymcts_agent.py

@@ -1,4 +1,5 @@
 import copy
+import random
 import gymnasium as gym
 
 from typing import TypeVar, Any, SupportsFloat, Callable
@@ -63,7 +64,10 @@ class GymctsAgent:
         # NAVIGATION STRATEGY
         # select child with highest UCB score
         while not temp_node.is_leaf():
-            temp_node = max(temp_node.children.values(), key=lambda child: child.ucb_score())
+            children = list(temp_node.children.values())
+            max_ucb_score = max(child.ucb_score() for child in children)
+            best_children = [child for child in children if child.ucb_score() == max_ucb_score]
+            temp_node = random.choice(best_children)
         log.debug(f"Selected leaf node: {temp_node}")
         return temp_node