gr-libs 0.1.7.post0__py3-none-any.whl → 0.2.2__py3-none-any.whl

gr_libs/metrics/metrics.py

@@ -1,22 +1,24 @@
+""" metrics for GR algorithms, to perform distance, similarity, likelihood and other measurements and metrics. """
+
 import math
+from collections.abc import Callable, Generator
+from math import log2
+from typing import Any
+
 import dill
 import gymnasium
 import numpy as np
-
-from typing import Callable, Generator, List, Dict, Tuple, Any
-from math import log2
-from scipy.stats import wasserstein_distance
 from gymnasium.spaces.discrete import Discrete
-# import torch
-# from torch.distributions.categorical import Categorical
+from scipy.stats import wasserstein_distance
 
 from ..ml.base import State
 from ..ml.base.rl_agent import RLAgent
 from ..ml.neural.deep_rl_learner import DeepRLAgent
 
 
-def kl_divergence(p1: List[float], p2: List[float]) -> float:
-    """Computes Kullback–Leibler divergence from two probabilities distributions p1 and p2.
+def kl_divergence(p1: list[float], p2: list[float]) -> float:
+    """
+    Computes Kullback–Leibler divergence from two probabilities distributions p1 and p2.
     We follow the formula in Wikipedia https://en.wikipedia.org/wiki/Kullback–Leibler_divergence
 
     Args:
@@ -26,21 +28,35 @@ def kl_divergence(p1: List[float], p2: List[float]) -> float:
     Returns:
         float: The KL-divergence between p1 and p2
     """
-    assert (len(p1) == len(p2))
+    assert len(p1) == len(p2)
     return sum(p1[i] * log2(p1[i] / p2[i]) for i in range(len(p1)))
 
 
-def kl_divergence_norm_softmax(observations: List[Tuple[State, Any]], agent, actions: Discrete):
+def kl_divergence_norm_softmax(
+    observations: list[tuple[State, Any]], agent, actions: Discrete
+):
+    """
+    Calculates the Kullback-Leibler (KL) divergence between two probability distributions.
+
+    Args:
+        observations (list[tuple[State, Any]]): List of observations and corresponding actions.
+        agent: The agent object.
+        actions: The discrete actions.
+
+    Returns:
+        float: The mean KL divergence between the two distributions.
+    """
     distances = []
     p_traj = traj_to_policy(observations=observations, actions=actions)
 
     for (observation, agent_pos), action in observations:
-        state = observation['image']
+        state = observation["image"]
         state_pickled = dill.dumps(state)
 
         qp1 = p_traj[state_pickled]
-        qp2_flatten_distribution_list: List[float] = agent.get_actions_probabilities(
-            observation=(observation, agent_pos))
+        qp2_flatten_distribution_list: list[float] = agent.get_actions_probabilities(
+            observation=(observation, agent_pos)
+        )
         distances.append(kl_divergence(qp1, qp2_flatten_distribution_list))
     return np.mean(distances)
 
@@ -53,22 +69,58 @@ def amplify(values, alpha=1.0):
     Returns:
         np.array: amplified softmax probabilities
     """
-    values = values[:3]**alpha # currently only choose to turn or move forward
+    values = values[:3] ** alpha  # currently only choose to turn or move forward
     return values / np.sum(values)
 
+
 def stochastic_amplified_selection(actions_probs, alpha=8.0):
+    """
+    Selects an action based on the given action probabilities, with amplification using the specified alpha value.
+
+    Parameters:
+        actions_probs (list): A list of action probabilities.
+        alpha (float): Amplification factor (default: 8.0).
+
+    Returns:
+        int: The selected action.
+
+    """
     action_probs_amplified = amplify(actions_probs, alpha)
     choice = np.random.choice(len(action_probs_amplified), p=action_probs_amplified)
     if choice == 3:
         choice = 6
     return choice
 
+
+import numpy as np
+
+
 def stochastic_selection(actions_probs):
+    """
+    Selects an action based on the given probabilities using a stochastic selection method.
+
+    Parameters:
+        actions_probs (list): A list of probabilities for each action.
+
+    Returns:
+        int: The index of the selected action.
+    """
     return np.random.choice(len(actions_probs), p=actions_probs)
 
+
 def greedy_selection(actions_probs):
+    """
+    Selects the action with the highest probability.
+
+    Args:
+        actions_probs (numpy.ndarray): Array of action probabilities.
+
+    Returns:
+        int: Index of the selected action.
+    """
     return np.argmax(actions_probs)
 
+
 def measure_average_sequence_distance(seq1, seq2):
     """Measures the sequence similarity between two sequences of observations and actions.
 
@@ -82,45 +134,89 @@ def measure_average_sequence_distance(seq1, seq2):
 
     # Ensure both sequences have the same length
     min_seq_len = np.min([len(seq1), len(seq2)])
-    assert np.max([len(seq1), len(seq2)]) <= 30*min_seq_len, "We can't really measure similarity in case the sequences are really not the same... maybe just return a default NOT_SIMILAR here."
+    assert (
+        np.max([len(seq1), len(seq2)]) <= 30 * min_seq_len
+    ), "We can't really measure similarity in case the sequences are really not the same... maybe just return a default NOT_SIMILAR here."
 
     # Calculate the Euclidean distance between corresponding elements in the sequences
     distances = []
     for i in range(0, min_seq_len):
-        distances.append(np.sum(np.abs(np.array(seq1[i])-np.array(seq2[i]))))
+        distances.append(np.sum(np.abs(np.array(seq1[i]) - np.array(seq2[i]))))
 
     # Calculate the average distance over all elements
     return np.mean(np.array(distances))
 
 
-def traj_to_policy(observations: List[Tuple[State, Any]], actions: Discrete, epsilon: float = 0.) -> Dict[
-    str, List[float]]:
-    # converts a trajectory from a planner to a policy
-    # where the taken action has 99.99999% probability
+def traj_to_policy(
+    observations: list[tuple[State, Any]], actions: Discrete, epsilon: float = 0.0
+) -> dict[str, list[float]]:
+    """
+    Converts a trajectory from a planner to a policy.
+
+    Args:
+        observations (list[tuple[State, Any]]): List of tuples containing the observation and the corresponding action.
+        actions (Discrete): Discrete action space.
+        epsilon (float, optional): Exploration parameter. Defaults to 0.0.
+
+    Returns:
+        dict[str, list[float]]: Dictionary mapping serialized states to action probabilities.
+    """
     trajectory_as_policy = {}
-    for (observation, agent_pos), action in observations:
-        # in the discrete world the action is the index
+    for (observation, _agent_pos), action in observations:
         action_index = action
 
         actions_len = actions.n
         qs = [1e-6 + epsilon / actions_len for _ in range(actions_len)]
-        qs[action_index] = 1. - 1e-6 * (actions_len - 1) - epsilon
+        qs[action_index] = 1.0 - 1e-6 * (actions_len - 1) - epsilon
 
-        state = observation['image']
+        state = observation["image"]
         state_pickled = dill.dumps(state)
         trajectory_as_policy[state_pickled] = qs
     return trajectory_as_policy
 
-def pass_observation_patcher(observations: List[Any], agent: RLAgent) -> Generator[None, None, None]:
-    for observation in observations:
-        yield observation
+
+from collections.abc import Generator
+from typing import Any
+
+
+def pass_observation_patcher(
+    observations: list[Any], agent: RLAgent
+) -> Generator[None, None, None]:
+    """
+    Generator function that yields observations.
+
+    Args:
+        observations (list): List of observations.
+        agent (RLAgent): RL agent object.
+
+    Yields:
+        None: Yields each observation from the list.
+
+    """
+    yield from observations
+
 
 def mean_wasserstein_distance(
-        observations: List[Tuple[State, Any]],
-        agent: DeepRLAgent,
-        actions: gymnasium.spaces.Box,
-        observation_patcher: Callable[[List[Any], RLAgent], Generator[None, None, None]] = pass_observation_patcher
+    observations: list[tuple[State, Any]],
+    agent: DeepRLAgent,
+    actions: gymnasium.spaces.Box,
+    observation_patcher: Callable[
+        [list[Any], RLAgent], Generator[None, None, None]
+    ] = pass_observation_patcher,
 ):
+    """
+    Calculates the mean Wasserstein distance between observed actions and actor means.
+
+    Args:
+        observations (list[tuple[State, Any]]): List of observations and corresponding actions.
+        agent (DeepRLAgent): The deep reinforcement learning agent.
+        actions (gymnasium.spaces.Box): The action space.
+        observation_patcher (Callable[[list[Any], RLAgent], Generator[None, None, None]], optional):
+            A function that patches the observations. Defaults to pass_observation_patcher.
+
+    Returns:
+        float: The mean Wasserstein distance between observed actions and actor means.
+    """
     distances = []
 
     for observation, observed_action in observation_patcher(observations, agent):
@@ -141,49 +237,98 @@ def mean_wasserstein_distance(
             wasserstein_distances.append(
                 wasserstein_distance([observation_action], [actor_mean])
             )
-        distances.append(mean(wasserstein_distances))
-    return mean(distances)
+        distances.append(np.mean(wasserstein_distances))
+    return np.mean(distances)
 
 
-def mean_action_distance_continuous(observations: List[Tuple[State, Any]], agent: DeepRLAgent, actions: gymnasium.spaces.Box):
+def mean_action_distance_continuous(
+    observations: list[tuple[State, Any]],
+    agent: DeepRLAgent,
+    actions: gymnasium.spaces.Box,
+):
+    """
+    Calculates the mean distance between the predicted actions and the actual actions for a continuous action space.
+
+    Args:
+        observations (list[tuple[State, Any]]): A list of tuples containing the observations and corresponding actions.
+        agent (DeepRLAgent): The deep reinforcement learning agent used to predict actions.
+        actions (gymnasium.spaces.Box): The action space.
+
+    Returns:
+        float: The mean distance between the predicted actions and the actual actions.
+    """
     distances = []
     for observation, action in observations:
         action2, _ = agent.model.predict(
             observation,
             state=None,
             deterministic=True,
-            episode_start=np.ones((1,), dtype=bool)
+            episode_start=np.ones((1,), dtype=bool),
         )
         action_arr, action2_arr = action[0], action2[0]
         print(f"actor means:{action2}")
-        assert len(action_arr) == len(action2_arr), f"Actions should be on the same length:{action},{action2}"
+        assert len(action_arr) == len(
+            action2_arr
+        ), f"Actions should be on the same length:{action},{action2}"
 
         total_diff = 0
-        # total_diff = []
         for action1, action2 in zip(action_arr, action2_arr):
             total_diff += math.fabs(action1 - action2)
-        # distances.append(statistics.mean(total_diff))
         distances.append(total_diff)
-    # print(f"distances:{distances}")
-    return mean(distances)
+    return np.mean(distances)
+
+
+from collections.abc import Generator
+from typing import Any
 
 
-def set_agent_goal_observation(observations: List[Any], agent: RLAgent) -> Generator[None, None, None]:
+def set_agent_goal_observation(
+    observations: list[Any], agent: RLAgent
+) -> Generator[None, None, None]:
+    """
+    Sets the desired goal in each observation to the agent's goal.
+
+    Args:
+        observations (list): List of observations.
+        agent (RLAgent): The RL agent.
+
+    Yields:
+        tuple: A tuple containing the modified observation and the corresponding action.
+    """
     copy_observation = observations.copy()
     for observation, action in copy_observation:
-        observation['desired_goal'] = agent.goal
+        observation["desired_goal"] = agent.goal
         yield observation, action
 
 
 def z_score(x, mean_action: float, std_dev: float):
     return (x - mean_action) / std_dev
 
+
 def mean_p_value(
-        observations: List[Tuple[State, Any]],
-        agent: DeepRLAgent,
-        actions: gymnasium.spaces.Box,
-        observation_patcher: Callable[[List[Any], RLAgent], Generator[None, None, None]] = pass_observation_patcher
+    observations: list[tuple[State, Any]],
+    agent: DeepRLAgent,
+    actions: gymnasium.spaces.Box,
+    observation_patcher: Callable[
+        [list[Any], RLAgent], Generator[None, None, None]
+    ] = pass_observation_patcher,
 ):
+    """
+    Calculate the mean p-value for a given set of observations.
+
+    Args:
+        observations (list[tuple[State, Any]]): List of observations and corresponding actions.
+        agent (DeepRLAgent): The deep reinforcement learning agent.
+        actions (gymnasium.spaces.Box): The action space.
+        observation_patcher (Callable[[list[Any], RLAgent], Generator[None, None, None]], optional):
+            A function that patches the observations. Defaults to pass_observation_patcher.
+
+    Returns:
+        float: The mean p-value.
+
+    Raises:
+        Exception: If the lengths of observed actions, actor mean, and std-dev are not equal.
+    """
     distances = []
     for observation, observed_action in observation_patcher(observations, agent):
         # execute prediction X times and add to list (observed_action * X) |X| Len
@@ -194,30 +339,62 @@ def mean_p_value(
         observed_actions = observed_action[0]
         log_std_dev = log_std_dev[0]
 
-        if len(actor_means) != len(observed_actions) or len(actor_means) != len(log_std_dev) or len(observed_actions) != len(log_std_dev):
+        if (
+            len(actor_means) != len(observed_actions)
+            or len(actor_means) != len(log_std_dev)
+            or len(observed_actions) != len(log_std_dev)
+        ):
             raise Exception(
                 f"Length of observed actions, actor mean and std-dev should be equal! "
                 f"{len(observed_actions)},{len(actor_means)},{len(log_std_dev)}"
             )
         z_scores = []
-        for actor_mean, observation_action, action_log_std_dev in zip(actor_means, observed_actions, log_std_dev):
+        for actor_mean, observation_action, action_log_std_dev in zip(
+            actor_means, observed_actions, log_std_dev
+        ):
             z_scores.append(
-                math.fabs(z_score(observation_action, actor_mean, math.pow(2, math.fabs(action_log_std_dev))))
+                math.fabs(
+                    z_score(
+                        observation_action,
+                        actor_mean,
+                        math.pow(2, math.fabs(action_log_std_dev)),
+                    )
+                )
             )
-        mean_distances = mean(z_scores)
+        mean_distances = np.mean(z_scores)
 
         distances.append(mean_distances)
-    return mean(distances)
+    return np.mean(distances)
+
 
-def normalize(values: List[float]) -> List[float]:
+def normalize(values: list[float]) -> list[float]:
+    """
+    Normalize a list of values by dividing each value by the sum of all values.
+
+    Args:
+        values (list[float]): The list of values to be normalized.
+
+    Returns:
+        list[float]: The normalized list of values.
+    """
     values /= sum(values)
     return values
 
-def max(values: List[float]) -> List[float]:
+
+def maximum(values: list[float]) -> list[float]:
+    """
+    Returns a list with the same length as the input list, where the maximum value is set to 1.0 and all other values are set to 0.0.
+
+    Args:
+        values (list[float]): The input list of values.
+
+    Returns:
+        list[float]: A list with the same length as the input list, where the maximum value is set to 1.0 and all other values are set to 0.0.
+    """
     if not len(values):
         return values
     vals = np.array(values)
     argmax = vals.argmax()
     vals[:] = 0.0
     vals[argmax] = 1.0
-    return vals
+    return vals

gr_libs/ml/__init__.py

@@ -1,6 +1,3 @@
-from ..ml.utils import device, seed, synthesize
-# from ml.neural import PPOAlgo
-from ..ml.tabular import TabularQLearner
 # from ml.neural import ACModel, RecurrentACModel
-from ..ml.neural import DictList
-from ..ml.agent import Agent
+
+# from ml.neural import PPOAlgo

gr_libs/ml/agent.py

@@ -2,6 +2,7 @@ import torch
 
 from gr_libs.ml import utils
 from gr_libs.ml.utils.other import device
+
 # from ml.neural import ACModel
 
 
@@ -12,15 +13,27 @@ class Agent:
     - to choose an action given an observation,
     - to analyze the feedback (i.e. reward and done state) of its action."""
 
-    def __init__(self, obs_space, action_space, model_dir,
-                 argmax=False, num_envs=1, use_memory=True, use_text=False):
+    def __init__(
+        self,
+        obs_space,
+        action_space,
+        model_dir,
+        argmax=False,
+        num_envs=1,
+        use_memory=True,
+        use_text=False,
+    ):
         obs_space, self.preprocess_obss = utils.get_obss_preprocessor(obs_space)
-        self.acmodel = ACModel(obs_space, action_space, use_memory=use_memory, use_text=use_text)
+        self.acmodel = ACModel(
+            obs_space, action_space, use_memory=use_memory, use_text=use_text
+        )
         self.argmax = argmax
         self.num_envs = num_envs
 
         if self.acmodel.recurrent:
-            self.memories = torch.zeros(self.num_envs, self.acmodel.memory_size, device=device)
+            self.memories = torch.zeros(
+                self.num_envs, self.acmodel.memory_size, device=device
+            )
 
         self.acmodel.load_state_dict(utils.get_model_state(model_dir))
         self.acmodel.to(device)
@@ -49,8 +62,10 @@ class Agent:
 
     def analyze_feedbacks(self, rewards, dones):
         if self.acmodel.recurrent:
-            masks = 1 - torch.tensor(dones, dtype=torch.float, device=device).unsqueeze(1)
+            masks = 1 - torch.tensor(dones, dtype=torch.float, device=device).unsqueeze(
+                1
+            )
             self.memories *= masks
 
     def analyze_feedback(self, reward, done):
-        return self.analyze_feedbacks([reward], [done])
+        return self.analyze_feedbacks([reward], [done])

gr_libs/ml/base/__init__.py

@@ -1 +1,3 @@
-from gr_libs.ml.base.rl_agent import RLAgent, State, ContextualAgent
+""" base ML classes for other modules to extend. """
+
+from gr_libs.ml.base.rl_agent import ContextualAgent, RLAgent, State

gr_libs/ml/base/rl_agent.py

@@ -1,26 +1,61 @@
-from typing import Any
 from abc import ABC, abstractmethod
-import numpy as np
+from typing import Any
 
 State = Any
 
+
 class ContextualAgent:
+    """
+    A class representing a contextual agent for reinforcement learning, including gym properties.
+
+    Args:
+        problem_name (str): The name of the problem the agent is designed to solve.
+        problem_goal (str): The goal of the problem the agent is designed to achieve.
+        agent: The underlying agent implementation.
+
+    Attributes:
+        problem_name (str): The name of the problem the agent is designed to solve.
+        problem_goal (str): The goal of the problem the agent is designed to achieve.
+        agent: The underlying agent implementation.
+    """
+
     def __init__(self, problem_name, problem_goal, agent):
+        """
+        Initializes a reinforcement learning agent.
+
+        Args:
+            problem_name (str): The name of the problem.
+            problem_goal (str): The goal of the problem.
+            agent: The agent object.
+        """
         self.problem_name = problem_name
         self.problem_goal = problem_goal
         self.agent = agent
 
+
 class RLAgent(ABC):
     def __init__(
-            self,
-            episodes: int,
-            decaying_eps: bool,
-            epsilon: float,
-            learning_rate: float,
-            gamma: float,
-            problem_name: str,
-            domain_name: str
+        self,
+        episodes: int,
+        decaying_eps: bool,
+        epsilon: float,
+        learning_rate: float,
+        gamma: float,
+        problem_name: str,
+        domain_name: str,
     ):
+        """
+        Initializes a reinforcement learning agent.
+
+        Args:
+            episodes (int): The number of episodes to train the agent.
+            decaying_eps (bool): Whether to use decaying epsilon-greedy exploration.
+            epsilon (float): The exploration rate.
+            learning_rate (float): The learning rate.
+            gamma (float): The discount factor.
+            problem_name (str): The name of the problem.
+            domain_name (str): The name of the domain.
+        """
         self.episodes = episodes
         self.decaying_eps = decaying_eps
         self.epsilon = epsilon
@@ -33,22 +68,55 @@ class RLAgent(ABC):
 
     @abstractmethod
     def learn(self):
-        pass
+        """
+        Abstract method for the agent to learn from the environment.
+        """
 
     def class_name(self):
+        """
+        Returns the name of the agent's class.
+
+        Returns:
+            str: The name of the agent's class.
+        """
         return self.__class__.__name__
 
     def get_actions_probabilities(self, observation):
+        """
+        Get the probabilities of available actions given an observation.
+
+        Args:
+            observation: The observation from the environment.
+
+        Raises:
+            Exception: This function is unimplemented.
+
+        Returns:
+            Any: The probabilities of available actions.
+        """
         raise Exception("function get_actions_probabilities is unimplemented")
 
     def get_number_of_unique_states(self):
+        """
+        Get the number of unique states encountered by the agent.
+
+        Returns:
+            int: The number of unique states encountered.
+        """
         return len(self.states_counter)
 
     def update_states_counter(self, observation_str: str):
+        """
+        Update the counter for the number of times each observation state is encountered.
+
+        Args:
+            observation_str (str): The string representation of the observation state.
+        """
         if observation_str in self.states_counter:
-            self.states_counter[observation_str] = self.states_counter[observation_str] + 1
+            self.states_counter[observation_str] = (
+                self.states_counter[observation_str] + 1
+            )
         else:
             self.states_counter[observation_str] = 1
         if len(self.states_counter) % 10000 == 0:
             print(f"probably error to many {len(self.states_counter)}")
-

gr_libs/ml/consts.py

@@ -19,4 +19,4 @@ OPTIM_EPS = 1e-8
 OPTIM_ALPHA = 0.99
 CLIP_EPS = 0.2
 RECURRENCE = 1
-TEXT = False
+TEXT = False

gr_libs/ml/neural/__init__.py

@@ -1,3 +1 @@
-# from ml.neural.model import AbstractACModel, RecurrentACModel, ACModel
-# from ml.neural.algorithms import BaseAlgo, A2CAlgo, PPOAlgo
-from gr_libs.ml.neural.utils import DictList
+""" Algorithms that involve using neural networks. """