gr-libs 0.1.8__py3-none-any.whl → 0.2.5__py3-none-any.whl

gr_libs/ml/planner/mcts/mcts_model.py

@@ -1,14 +1,16 @@
+""" model that performs mcts to find a plan in discrete state/action environments. """
+
 import os
+import pickle
 import random
-from math import sqrt, log
+from math import log, sqrt
 
+import gymnasium as gym
 from tqdm import tqdm
-import pickle
 
 from gr_libs.ml.utils.storage import get_agent_model_dir
-from .utils import Node
-from .utils import Tree
-import gymnasium as gym
+
+from ._utils import Node, Tree
 
 PROB = 0.8
 UNIFORM_PROB = 0.1
@@ -18,8 +20,20 @@ dict_action_id_to_str = {0: "turn left", 1: "turn right", 2: "go straight"}
 
 
 def save_figure(steps, env_name, problem_name, img_path, env_prop):
+    """
+    Save a figure representing the sequence of steps taken in a problem.
+
+    Args:
+        steps (list): List of tuples representing the state, position, and action taken at each step.
+        env_name (str): Name of the environment.
+        problem_name (str): Name of the problem.
+        img_path (str): Path to save the generated image.
+        env_prop: Object with methods to create the sequence image.
+
+    Returns:
+        None
+    """
     sequence = [pos for ((state, pos), action) in steps]
-    # print(f"sequence to {self.problem_name} is:\n\t{steps}\ngenerating image at {img_path}.")
     print(f"generating sequence image at {img_path}.")
     env_prop.create_sequence_image(sequence, img_path, problem_name)
 
@@ -27,16 +41,39 @@ def save_figure(steps, env_name, problem_name, img_path, env_prop):
 # TODO add number of expanded nodes and debug by putting breakpoint on the creation of nodes representing (8,4) and checking if they're invalid or something
 
 
-# Explanation on hashing and uncertainty in the acto outcome:
-# We want to detect circles, while not preventing expected behavior. To achieve it, hasing must include previous state, action, and resulting state.
-# Hashing the direction means coming to the same position from different positions gets different id's.
-# Example: the agent might have stood at (2,2), picked action 2 (forward), and accidently turned right, resulting at state ((2,2), right).
-# 		   later, when the agent stood at (2,1), looked right and walked forward, it got to the same state. We would want to enable that, because
-# 		   this is the expected behavior, so these nodes must have unique id's.
-# The situations where circles will indeed be detected, are only if the outcome was the same for the previous state, consistent with the action - whether it was or wasn't expected.
 class MonteCarloTreeSearch:
+    """
+    Monte Carlo Tree Search class for performing search on an environment using a tree data structure.
+
+    Explanation on hashing and uncertainty in the acto outcome:
+    We want to detect circles, while not preventing expected behavior.
+    To achieve it, hasing must include previous state, action, and resulting state.
+    Hashing the direction means coming to the same position from different positions gets different id's.
+    Example: the agent might have stood at (2,2), picked action 2 (forward), and accidently turned right,
+    resulting at state ((2,2), right).
+    later, when the agent stood at (2,1), looked right and walked forward,
+    it got to the same state. We would want to enable that, because
+    this is the expected behavior, so these nodes must have unique id's.
+    The situations where circles will indeed be detected, are only if the outcome was the same for the previous state,
+    consistent with the action - whether it was or wasn't expected.
+
+    Args:
+        env (gym.Env): The environment to perform the search on.
+        tree (Tree): The tree data structure to store the search tree.
+        goal (object): The goal state of the search.
+        use_heuristic (bool, optional): Whether to use a heuristic function during the search. Defaults to True.
+    """
 
     def __init__(self, env, tree, goal, use_heuristic=True):
+        """
+        Initializes the Monte Carlo Tree Search.
+
+        Args:
+            env (gym.Env): The environment to perform the search on.
+            tree (Tree): The tree data structure to store the search tree.
+            goal (object): The goal state of the search.
+            use_heuristic (bool, optional): Whether to use a heuristic function during the search. Defaults to True.
+        """
         self.env = env
         self.tree = tree
         self.action_space = self.env.action_space.n
@@ -81,13 +118,13 @@ class MonteCarloTreeSearch:
         new_node_ptr = new_node_father
         old_node_ptr = old_node
 
-        while new_node_ptr != None:
+        while new_node_ptr is not None:
             new_visits[0] += new_node_ptr.num_visits
             new_visits[1] += 1
             new_node_ptr = self.tree.parent(new_node_ptr)
 
         while (
-            old_node_ptr != None
+            old_node_ptr is not None
         ):  # getting to the old node wasn't necessarily through the current root. check all the way until None, the original root's parent.
             old_visits[0] += old_node_ptr.num_visits
             old_visits[1] += 1
@@ -128,7 +165,7 @@ class MonteCarloTreeSearch:
         while (
             new_identifier in self.tree.nodes.keys()
         ):  # iterate over all circle nodes. important not to hash the parent node id to get the next id, because it will not be the same for all circle nodes.
-            if self.tree.nodes[new_identifier].invalid == False:
+            if self.tree.nodes[new_identifier].invalid is False:
                 valid_id = new_identifier
             new_identifier = hash((666, new_identifier))
         # after this while, the id is for sure unused.
@@ -194,7 +231,7 @@ class MonteCarloTreeSearch:
         while (
             resulting_identifier in self.tree.nodes.keys()
         ):  # iterate over all circle nodes. important not to hash the parent node id to get the next id, because it will not be the same for all circle nodes.
-            if self.tree.nodes[resulting_identifier].invalid == False:
+            if self.tree.nodes[resulting_identifier].invalid is False:
                 valid_id = resulting_identifier
             resulting_identifier = hash((666, resulting_identifier))
         # after this while, the id is for sure unused.
@@ -234,6 +271,7 @@ class MonteCarloTreeSearch:
         if self.use_heuristic:
             # taken from Monte-Carlo Planning for Pathfinding in Real-Time Strategy Games , 2010.
             # need to handle the case of walking into a wall here: the resulting node will be considered invalid and it's reward and performance needs to be 0, but must handle stochasticity
+            pass
             # suggestion to handle stochasticity - consider *all* the children associated with taking action 2 towards a wall as performance 0, even if they accidently led in walking to another direction.
             # which suggests the invalidity needs to be checked not according to the resulting state, rather according to the intended action itself and the environment! remember, you cannot access the "stochastic_action", it is meant to be hidden from you.
             if node.pos[0] == self.goal[0] and node.pos[1] == self.goal[1]:
@@ -382,7 +420,7 @@ class MonteCarloTreeSearch:
     def backpropagation(self, node, value):
         while node != self.tree.parent(self.tree.root):
             assert (
-                node != None
+                node is not None
             )  # if we got to None it means we got to the actual root with the backpropogation instead of to the current root, which means in this path, someone had a differrent parent than it should, probably a double id.
             node.num_visits += 1
             node.total_simulation_reward += value
@@ -411,10 +449,10 @@ class MonteCarloTreeSearch:
             )  # need to add the previous node with the action leading to the next node which is a property of the next node
             prev_node = node
         if save_fig:
-            assert fig_path != None
+            assert fig_path is not None
             save_figure(trace, env_name, problem_name, fig_path, env_prop)
         else:
-            assert fig_path == None
+            assert fig_path is None
         return trace
 
 
@@ -430,6 +468,17 @@ def save_model_and_generate_policy(
 
 
 def plan(env_name, problem_name, goal, save_fig=False, fig_path=None, env_prop=None):
+    """
+    Plan a path using Monte Carlo Tree Search (MCTS) algorithm.
+
+    Args:
+        env_name (str): Name of the environment.
+        problem_name (str): Name of the problem.
+        goal (tuple): Goal state to reach.
+        save_fig (bool): Flag to save the figure of the plan.
+        fig_path (str): Path to save the figure.
+        env_prop: Object with methods to create the sequence image.
+    """
     global newely_expanded
     model_dir = get_agent_model_dir(
         env_name=env_name, model_name=problem_name, class_name="MCTS"
@@ -440,16 +489,14 @@ def plan(env_name, problem_name, goal, save_fig=False, fig_path=None, env_prop=N
         with open(model_file_path, "rb") as file:  # Load the pre-existing model
             try:
                 monteCarloTreeSearch = pickle.load(file)
-            except Exception as e:
+            except Exception:
 
                 class RenameUnpickler(pickle.Unpickler):
                     def find_class(self, module, name):
                         renamed_module = module
                         if module.startswith("ml"):
                             renamed_module = "gr_libs." + renamed_module
-                        return super(RenameUnpickler, self).find_class(
-                            renamed_module, name
-                        )
+                        return super().find_class(renamed_module, name)
 
                 def renamed_load(file_obj):
                     return RenameUnpickler(file_obj).load()
@@ -540,13 +587,3 @@ def plan(env_name, problem_name, goal, save_fig=False, fig_path=None, env_prop=N
     return mcts.generate_full_policy_sequence(
         env_name, problem_name, save_fig, fig_path
     )
-
-
-if __name__ == "__main__":
-    # register(
-    # 	id="MiniGrid-DynamicGoalEmpty-8x8-3x6-v0",
-    # 	entry_point="minigrid.envs:DynamicGoalEmpty",
-    # 	kwargs={"size": 8, "agent_start_pos" : (1, 1), "goal_pos": (3,6) },
-    # )
-    # plan("MiniGrid-DynamicGoalEmpty-8x8-3x6-v0")
-    pass

gr_libs/ml/sequential/__init__.py

@@ -1 +0,0 @@
-from gr_libs.ml.sequential.lstm_model import LstmObservations

gr_libs/ml/sequential/{lstm_model.py → _lstm_model.py}

@@ -1,13 +1,10 @@
-import os
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-import torch.optim as optim
-from types import MethodType
-import numpy as np
-from gr_libs.ml.utils import device
 from torch.nn.utils.rnn import pack_padded_sequence
 
+from gr_libs.ml.utils import device
+
 
 def accuracy_per_epoch(model, data_loader):
     model.eval()
@@ -119,7 +116,7 @@ class LstmObservations(nn.Module):
     def __init__(
         self, input_size, hidden_size
     ):  # TODO make sure the right cuda is used!
-        super(LstmObservations, self).__init__()
+        super().__init__()
         # self.embeddor = CNNImageEmbeddor(obs_space, action_space)
         # check if the traces are a bunch of images
         self.lstm = nn.LSTM(
@@ -221,10 +218,10 @@ def train_metric_model(model, train_loader, dev_loader, nepochs=5, patience=2):
             no_improvement_count = 1
 
         print(
-            "epoch - {}/{}...".format(epoch + 1, nepochs),
-            "train loss - {:.6f}...".format(sum_loss / denominator),
-            "dev loss - {:.6f}...".format(dev_loss),
-            "dev accuracy - {:.6f}".format(dev_accuracy),
+            f"epoch - {epoch + 1}/{nepochs}...",
+            f"train loss - {sum_loss / denominator:.6f}...",
+            f"dev loss - {dev_loss:.6f}...",
+            f"dev accuracy - {dev_accuracy:.6f}",
         )
 
         if no_improvement_count >= patience:
@@ -266,8 +263,8 @@ def train_metric_model_cont(model, train_loader, dev_loader, nepochs=5):
         devAccuracy.append(dev_accuracy)
 
         print(
-            "epoch - {}/{}...".format(epoch + 1, nepochs),
-            "train loss - {:.6f}...".format(sum_loss / denominator),
-            "dev loss - {:.6f}...".format(dev_loss),
-            "dev accuracy - {:.6f}".format(dev_accuracy),
+            f"epoch - {epoch + 1}/{nepochs}...",
+            f"train loss - {sum_loss / denominator:.6f}...",
+            f"dev loss - {dev_loss:.6f}...",
+            f"dev accuracy - {dev_accuracy:.6f}",
         )

gr_libs/ml/tabular/__init__.py

@@ -1,3 +1 @@
-from gr_libs.ml.tabular.state import TabularState
-from gr_libs.ml.tabular.tabular_q_learner import TabularQLearner
-from gr_libs.ml.sequential.lstm_model import LstmObservations
+from .state import TabularState

gr_libs/ml/tabular/tabular_q_learner.py

@@ -1,18 +1,17 @@
-# Don't import stuff from metrics! it's a higher level module.
+""" implementation of q-learning """
+
 import os.path
 import pickle
-import random
+from collections.abc import Iterable
+from random import Random
 from types import MethodType
+from typing import Any
 
 import dill
-from gymnasium import register
 import numpy as np
-
-from tqdm import tqdm
-from typing import Any
-from random import Random
-from typing import List, Iterable
 from gymnasium.error import InvalidAction
+from tqdm import tqdm
+
 from gr_libs.environment.environment import QLEARNING, EnvProperty
 from gr_libs.ml.tabular import TabularState
 from gr_libs.ml.tabular.tabular_rl_agent import TabularRLAgent
@@ -44,6 +43,25 @@ class TabularQLearner(TabularRLAgent):
         check_partial_goals: bool = True,
         valid_only: bool = False,
     ):
+        """
+        Initialize a TabularQLearner object.
+
+        Args:
+            domain_name (str): The name of the domain.
+            problem_name (str): The name of the problem.
+            env_prop (EnvProperty): The environment properties.
+            algorithm (str): The algorithm to use.
+            num_timesteps (int): The number of timesteps.
+            decaying_eps (bool, optional): Whether to use decaying epsilon. Defaults to True.
+            eps (float, optional): The initial epsilon value. Defaults to 1.0.
+            alpha (float, optional): The learning rate. Defaults to 0.5.
+            decay (float, optional): The decay rate. Defaults to 0.000002.
+            gamma (float, optional): The discount factor. Defaults to 0.9.
+            rand (Random, optional): The random number generator. Defaults to Random().
+            learning_rate (float, optional): The learning rate. Defaults to 0.001.
+            check_partial_goals (bool, optional): Whether to check partial goals. Defaults to True.
+            valid_only (bool, optional): Whether to use valid goals only. Defaults to False.
+        """
         super().__init__(
             domain_name=domain_name,
             problem_name=problem_name,
@@ -169,7 +187,7 @@ class TabularQLearner(TabularRLAgent):
     def add_new_state(self, state: TabularState):
         self.q_table[str(state)] = [0.0] * self.number_of_actions
 
-    def get_all_q_values(self, state: TabularState) -> List[float]:
+    def get_all_q_values(self, state: TabularState) -> list[float]:
         if str(state) in self.q_table:
             return self.q_table[str(state)]
         else:

gr_libs/ml/tabular/tabular_rl_agent.py

@@ -1,11 +1,11 @@
-import gymnasium as gym
 from abc import abstractmethod
-from typing import Collection, Literal, Any
 from random import Random
+from typing import Any
+
+import gymnasium as gym
 import numpy as np
 
-from gr_libs.ml.base import RLAgent
-from gr_libs.ml.base import State
+from gr_libs.ml.base import RLAgent, State
 
 
 class TabularRLAgent(RLAgent):
@@ -28,6 +28,24 @@ class TabularRLAgent(RLAgent):
         rand: Random,
         learning_rate,
     ):
+        """
+        Initializes a TabularRLAgent object.
+
+        Args:
+            domain_name (str): The name of the domain.
+            problem_name (str): The name of the problem.
+            episodes (int): The number of episodes to run.
+            decaying_eps (bool): Whether to use decaying epsilon.
+            eps (float): The initial epsilon value.
+            alpha (float): The learning rate.
+            decay (float): The decay rate for epsilon.
+            gamma (float): The discount factor.
+            rand (Random): The random number generator.
+            learning_rate: The learning rate.
+
+        Returns:
+            None
+        """
         super().__init__(
             episodes=episodes,
             decaying_eps=decaying_eps,
@@ -60,7 +78,6 @@ class TabularRLAgent(RLAgent):
         Returns:
             (int) the first action the agent takes.
         """
-        pass
 
     @abstractmethod
     def agent_step(self, reward: float, state: State) -> Any:
@@ -73,7 +90,6 @@ class TabularRLAgent(RLAgent):
         Returns:
             The action the agent is taking.
         """
-        pass
 
     @abstractmethod
     def agent_end(self, reward: float) -> Any:
@@ -83,7 +99,6 @@ class TabularRLAgent(RLAgent):
             reward (float): the reward the agent received for entering the
                 terminal state.
         """
-        pass
 
     @abstractmethod
     def policy(self, state: State) -> Any:
@@ -94,7 +109,6 @@ class TabularRLAgent(RLAgent):
         Returns:
              The action prescribed for that state
         """
-        pass
 
     @abstractmethod
     def softmax_policy(self, state: State) -> np.array:
@@ -106,7 +120,6 @@ class TabularRLAgent(RLAgent):
         Returns:
             np.array: probability of taking each action in self.actions given a state
         """
-        pass
 
     @abstractmethod
     def learn(self, init_threshold: int = 20):

gr_libs/ml/utils/__init__.py

@@ -1,12 +1,5 @@
-# from .agent import *
 from .env import make_env
-from .format import (
-    Vocabulary,
-    preprocess_images,
-    preprocess_texts,
-    get_obss_preprocessor,
-    random_subset_with_order,
-)
+from .format import random_subset_with_order
+from .math import softmax
 from .other import device, seed, synthesize
 from .storage import *
-from .math import softmax

gr_libs/ml/utils/format.py

@@ -1,62 +1,22 @@
-import numpy
-import re
-import torch
-import gr_libs.ml
-import gymnasium as gym
-import random
-
-
-def get_obss_preprocessor(obs_space):
-    # Check if obs_space is an image space
-    if isinstance(obs_space, gym.spaces.Box):
-        obs_space = {"image": obs_space.shape}
-
-        def preprocess_obss(obss, device=None):
-            return ml.DictList({"image": preprocess_images(obss, device=device)})
-
-    # Check if it is a MiniGrid observation space
-    elif isinstance(obs_space, gym.spaces.Dict) and "image" in obs_space.spaces.keys():
-        obs_space = {"image": obs_space.spaces["image"].shape, "text": 100}
-
-        vocab = Vocabulary(obs_space["text"])
-
-        def preprocess_obss(obss, device=None):
-            return ml.DictList(
-                {
-                    "image": preprocess_images(
-                        [obs["image"] for obs in obss], device=device
-                    ),
-                    "text": preprocess_texts(
-                        [obs["mission"] for obs in obss], vocab, device=device
-                    ),
-                }
-            )
+""" formatting-related utilities """
 
-        preprocess_obss.vocab = vocab
-
-    # Check if it is a MiniGrid observation space
-    elif (
-        isinstance(obs_space, gym.spaces.Dict)
-        and "observation" in obs_space.spaces.keys()
-    ):
-        obs_space = {"observation": obs_space.spaces["observation"].shape}
-
-        def preprocess_obss(obss, device=None):
-            return ml.DictList({"observation": preprocess_images(obss, device=device)})
-
-    else:
-        raise ValueError("Unknown observation space: " + str(obs_space))
+import random
 
-    return obs_space, preprocess_obss
 
+def random_subset_with_order(sequence, subset_size, is_consecutive=True):
+    """
+    Returns a random subset of elements from the given sequence with a specified subset size.
 
-def preprocess_images(images, device=None):
-    # Bug of Pytorch: very slow if not first converted to numpy array
-    images = numpy.array(images)
-    return torch.tensor(images, device=device, dtype=torch.float)
+    Args:
+        sequence (list): The sequence of elements to select from.
+        subset_size (int): The size of the desired subset.
+        is_consecutive (bool, optional): Whether the selected subset should be consecutive elements from the sequence.
+            Defaults to True.
 
+    Returns:
+        list: A random subset of elements from the sequence.
 
-def random_subset_with_order(sequence, subset_size, is_consecutive=True):
+    """
     if subset_size >= len(sequence):
         return sequence
     else:
@@ -69,40 +29,3 @@ def random_subset_with_order(sequence, subset_size, is_consecutive=True):
         return [
             sequence[i] for i in indices_to_select
         ]  # Return the elements corresponding to the selected indices
-
-
-def preprocess_texts(texts, vocab, device=None):
-    var_indexed_texts = []
-    max_text_len = 0
-
-    for text in texts:
-        tokens = re.findall("([a-z]+)", text.lower())
-        var_indexed_text = numpy.array([vocab[token] for token in tokens])
-        var_indexed_texts.append(var_indexed_text)
-        max_text_len = max(len(var_indexed_text), max_text_len)
-
-    indexed_texts = numpy.zeros((len(texts), max_text_len))
-
-    for i, indexed_text in enumerate(var_indexed_texts):
-        indexed_texts[i, : len(indexed_text)] = indexed_text
-
-    return torch.tensor(indexed_texts, device=device, dtype=torch.long)
-
-
-class Vocabulary:
-    """A mapping from tokens to ids with a capacity of `max_size` words.
-    It can be saved in a `vocab.json` file."""
-
-    def __init__(self, max_size):
-        self.max_size = max_size
-        self.vocab = {}
-
-    def load_vocab(self, vocab):
-        self.vocab = vocab
-
-    def __getitem__(self, token):
-        if not token in self.vocab.keys():
-            if len(self.vocab) >= self.max_size:
-                raise ValueError("Maximum vocabulary capacity reached")
-            self.vocab[token] = len(self.vocab) + 1
-        return self.vocab[token]

gr_libs/ml/utils/math.py

@@ -1,8 +1,9 @@
+""" math-related functions """
+
 import math
-from typing import Callable, Generator, List
 
 
-def softmax(values: List[float]) -> List[float]:
+def softmax(values: list[float]) -> list[float]:
     """Computes softmax probabilities for an array of values
     TODO We should probably use numpy arrays here
     Args:

gr_libs/ml/utils/other.py

@@ -1,8 +1,8 @@
+import collections
 import random
+
 import numpy
 import torch
-import collections
-
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")