pyrlutils 0.1.2__py3-none-any.whl

pyrlutils/action.py

@@ -0,0 +1,27 @@
+
+from types import LambdaType, FunctionType
+from typing import Union
+
+from .state import State
+
+
+DiscreteActionValueType = Union[float, str]
+
+class Action:
+    def __init__(self, actionfunc: Union[FunctionType, LambdaType]):
+        self._actionfunc = actionfunc
+
+    def act(self, state: State, *args, **kwargs) -> State:
+        self._actionfunc(state, *args, **kwargs)
+        return state
+
+    def __call__(self, state: State) -> State:
+        return self.act(state)
+
+    @property
+    def action_function(self) -> Union[FunctionType, LambdaType]:
+        return self._actionfunc
+
+    @action_function.setter
+    def action_function(self, new_func: Union[FunctionType, LambdaType]) -> None:
+        self._actionfunc = new_func

pyrlutils/bandit/algo.py

@@ -0,0 +1,128 @@
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+from .reward import IndividualBanditRewardFunction
+
+
+class BanditAlgorithm(ABC):
+    def __init__(self, action_values: list, reward_function: IndividualBanditRewardFunction):
+        self._action_values = action_values
+        self._reward_function = reward_function
+
+    @abstractmethod
+    def _go_one_loop(self):
+        pass
+
+    def loop(self, nbiterations: int):
+        for _ in range(nbiterations):
+            self._go_one_loop()
+
+    def reward(self, action_value) -> float:
+        return self._reward_function(action_value)
+
+    @abstractmethod
+    def get_action(self):
+        pass
+
+    @property
+    def action_values(self):
+        return self._action_values
+
+    @property
+    def reward_function(self) -> IndividualBanditRewardFunction:
+        return self._reward_function
+
+
+class SimpleBandit(BanditAlgorithm):
+    def __init__(
+            self,
+            action_values: list,
+            reward_function: IndividualBanditRewardFunction,
+            epsilon: float=0.05
+    ):
+        super().__init__(action_values, reward_function)
+        self._epsilon = epsilon
+        self._initialize()
+
+    def _initialize(self):
+        self._Q = np.zeros(len(self._action_values))
+        self._N = np.zeros(len(self._action_values), dtype=np.int32)
+
+    def _go_one_loop(self):
+        r = np.random.uniform()
+        if r < self.epsilon:
+            selected_action_idx = np.argmax(self._Q)
+        else:
+            selected_action_idx = np.random.choice(range(len(self._action_values)))
+        reward = self._reward_function(self._action_values[selected_action_idx])
+        self._N[selected_action_idx] += 1
+        self._Q[selected_action_idx] += (reward - self._Q[selected_action_idx]) / self._N[selected_action_idx]
+
+    def get_action(self):
+        selected_action_idx = np.argmax(self._Q)
+        return self._action_values[selected_action_idx]
+
+    @property
+    def epsilon(self) -> float:
+        return self._epsilon
+
+    @epsilon.setter
+    def epsilon(self, val: float):
+        self._epsilon = val
+
+
+class GradientBandit(BanditAlgorithm):
+    def __init__(self, action_values: list, reward_function: IndividualBanditRewardFunction, temperature: float=1.0, alpha: float=0.1):
+        super().__init__(action_values, reward_function)
+        self._T = temperature
+        self._alpha = alpha
+        self._initialize()
+
+    def _initialize(self):
+        self._preferences = np.zeros(len(self._action_values))
+        self._rewards_over_time = []
+
+    def _get_probs(self) -> np.ndarray:
+        # getting probabilities using softmax
+        exp_preferences = np.exp(self._preferences / self.T)
+        sum_exp_preferences = np.sum(exp_preferences)
+        return exp_preferences / sum_exp_preferences
+
+    def get_action(self):
+        selected_action_idx = np.argmax(self._preferences)
+        return self._action_values[selected_action_idx]
+
+    def _go_one_loop(self):
+        probs = self._get_probs()
+        selected_action_idx = np.random.choice(range(self._preferences.shape[0]), p=probs)
+        reward = self._reward_function(self._action_values[selected_action_idx])
+        self._rewards_over_time.append(reward)
+        average_reward = np.mean(self._rewards_over_time) if len(self._rewards_over_time) > 0 else 0.
+
+        for i in range(len(self._action_values)):
+            if i == selected_action_idx:
+                self._preferences[i] += self.alpha * (reward - average_reward) * (1 - probs[i])
+            else:
+                self._preferences[i] -= self.alpha * (reward - average_reward) * probs[i]
+
+    @property
+    def alpha(self) -> float:
+        return self._alpha
+
+    @alpha.setter
+    def alpha(self, val: float):
+        self._alpha = val
+
+    @property
+    def T(self) -> float:
+        return self._T
+
+    @T.setter
+    def T(self, val: float):
+        self._T = val
+
+    @property
+    def temperature(self) -> float:
+        return self._T

pyrlutils/bandit/reward.py

@@ -0,0 +1,12 @@
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class IndividualBanditRewardFunction(ABC):
+    @abstractmethod
+    def reward(self, action_value: Any) -> float:
+        pass
+
+    def __call__(self, action_value: Any) -> float:
+        return self.reward(action_value)

pyrlutils/dp/valuefcns.py

@@ -0,0 +1,149 @@
+
+import random
+from copy import copy
+from itertools import product
+from typing import Annotated
+
+import numpy as np
+from numpy.typing import NDArray
+
+from ..state import DiscreteStateValueType
+from ..transition import TransitionProbabilityFactory
+from ..policy import DiscreteDeterminsticPolicy
+
+
+class OptimalPolicyOnValueFunctions:
+    def __init__(
+            self,
+            discount_factor: float,
+            transprobfac: TransitionProbabilityFactory
+    ):
+        try:
+            assert 0. <= discount_factor <= 1.
+        except AssertionError:
+            raise ValueError('Discount factor must be between 0 and 1.')
+        self._gamma = discount_factor
+        self._transprobfac = transprobfac
+        self._states, self._actions_dict, self._indrewardfcn = self._transprobfac.generate_mdp_objects()
+        self._state_names = self._states.get_all_possible_state_values()
+        self._states_to_indices = {state: idx for idx, state in enumerate(self._state_names)}
+        self._action_names = list(self._actions_dict.keys())
+        self._actions_to_indices = {action_value: idx for idx, action_value in enumerate(self._action_names)}
+
+        self._evaluated = False
+        self._improved = False
+
+        self._theta = 1e-10
+        self._policy_evaluation_maxiter = 10000
+
+    def _policy_evaluation(self, policy: DiscreteDeterminsticPolicy) -> Annotated[NDArray[np.float64], "1D Array"]:
+        prev_V = np.zeros(len(self._states_to_indices))
+
+        for _ in range(self._policy_evaluation_maxiter):
+            V = np.zeros(len(self._states_to_indices))
+            for state_value in self._state_names:
+                state_index = self._states_to_indices[state_value]
+                action_value = policy.get_action_value(state_value)
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    V[state_index] += prob * (reward + (self._gamma*prev_V[next_state_index] if not terminal else 0.))
+
+            if np.max(np.abs(V-prev_V)) < self._theta:
+                break
+
+            prev_V = V.copy()
+
+        return V
+
+    def _policy_improvement(self, V: Annotated[NDArray[np.float64], "1D Array"]) -> DiscreteDeterminsticPolicy:
+        Q = np.zeros((len(self._states_to_indices), len(self._actions_to_indices)))
+
+        for state_value in self._state_names:
+            state_index = self._states_to_indices[state_value]
+            for action_value in self._action_names:
+                action_index = self._actions_to_indices[action_value]
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    Q[state_index, action_index] += prob * (reward + (self._gamma*V[next_state_index] if not terminal else 0.))
+
+        optimal_policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        optimal_action_indices = np.argmax(Q, axis=1)
+        for state_value, action_index in zip(self._state_names, optimal_action_indices):
+            action_value = self._action_names[action_index]
+            optimal_policy.add_deterministic_rule(state_value, action_value)
+        return optimal_policy
+
+    def _policy_iteration(self) -> tuple[Annotated[NDArray[np.float64], "1D Array"], DiscreteDeterminsticPolicy]:
+        policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        for state_value in self._state_names:
+            policy.add_deterministic_rule(state_value, random.choice(self._action_names))
+        V = None
+
+        done = False
+        while not done:
+            old_policy = copy(policy)
+
+            V = self._policy_evaluation(policy)
+            policy = self._policy_improvement(V)
+
+            if policy == old_policy:
+                done = True
+
+        return V, policy
+
+
+    def _value_iteration(self) -> tuple[Annotated[NDArray[np.float64], "1D Array"], DiscreteDeterminsticPolicy]:
+        V = np.zeros(len(self._state_names))
+
+        for _ in range(self._policy_evaluation_maxiter):
+            Q = np.zeros((len(self._state_names), len(self._action_names)))
+            for state_value, action_value in product(self._state_names, self._action_names):
+                state_index = self._states_to_indices[state_value]
+                action_index = self._actions_to_indices[action_value]
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    Q[state_index, action_index] += prob * (reward + (self._gamma * V[next_state_index] if not terminal else 0.))
+
+            if np.max(np.abs(V-np.max(Q, axis=1))) < self._theta:
+                break
+
+            V = np.max(Q, axis=1)
+
+        Qmaxj = np.argmax(Q, axis=1)
+
+        policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        for state_value, action_index in zip(self._state_names, Qmaxj):
+            policy.add_deterministic_rule(state_value, self._action_names[action_index])
+
+        return V, policy
+
+    def policy_iteration(self) -> tuple[dict[DiscreteStateValueType, float], DiscreteDeterminsticPolicy]:
+        V, policy = self._policy_iteration()
+        state_values_dict = {
+            self._state_names[i]: V[i]
+            for i in range(V.shape[0])
+        }
+        return state_values_dict, policy
+
+    def value_iteration(self) -> tuple[dict[DiscreteStateValueType, float], DiscreteDeterminsticPolicy]:
+        V, policy = self._value_iteration()
+        state_values_dict = {
+            self._state_names[i]: V[i]
+            for i in range(V.shape[0])
+        }
+        return state_values_dict, policy

pyrlutils/helpers/exceptions.py

@@ -0,0 +1,5 @@
+
+class InvalidRangeError(Exception):
+    def __init__(self, message=None):
+        self.message = "Invalid range error!" if message is None else message
+        super().__init__(self.message)

pyrlutils/openai/utils.py

@@ -0,0 +1,31 @@
+
+import gymnasium as gym
+
+from ..transition import TransitionProbabilityFactory, NextStateTuple
+
+
+class OpenAIGymDiscreteEnvironmentTransitionProbabilityFactory(TransitionProbabilityFactory):
+    def __init__(self, envname: str):
+        super().__init__()
+        self._envname = envname
+        self._gymenv = gym.make(envname)
+        self._convert_openai_gymenv_to_transprob()
+
+    def _convert_openai_gymenv_to_transprob(self):
+        P = self._gymenv.env.env.env.P
+        for state_value, trans_dict in P.items():
+            new_trans_dict = {}
+            for action_value, next_state_list in trans_dict.items():
+                new_trans_dict[action_value] = [
+                    NextStateTuple(next_state[1], next_state[0], next_state[2], next_state[3])
+                    for next_state in next_state_list
+                ]
+            self.add_state_transitions(state_value, new_trans_dict)
+
+    @property
+    def envname(self) -> str:
+        return self._envname
+
+    @property
+    def gymenv(self) -> gym.Env:
+        return self._gymenv

pyrlutils/policy.py

@@ -0,0 +1,151 @@
+
+from abc import ABC, abstractmethod
+from typing import Union, Annotated
+from warnings import warn
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .state import State, DiscreteState, DiscreteStateValueType
+from .action import Action, DiscreteActionValueType
+
+
+class Policy(ABC):
+    @abstractmethod
+    def get_action(self, state: State) -> Action:
+        raise NotImplemented()
+
+    @abstractmethod
+    def get_action_value(self, state: State) -> DiscreteActionValueType:
+        raise NotImplemented()
+
+    def __call__(self, state: State) -> Action:
+        return self.get_action(state)
+
+    @property
+    def is_stochastic(self) -> bool:
+        raise NotImplemented()
+
+
+class DeterministicPolicy(Policy):
+    @abstractmethod
+    def add_deterministic_rule(self, *args, **kwargs):
+        raise NotImplemented()
+
+    @property
+    def is_stochastic(self) -> bool:
+        return False
+
+
+class DiscreteDeterminsticPolicy(DeterministicPolicy):
+    def __init__(self, actions_dict: dict[DiscreteActionValueType, Action]):
+        self._state_to_action = {}
+        self._actions_dict = actions_dict
+
+    def add_deterministic_rule(
+            self,
+            state_value: DiscreteStateValueType,
+            action_value: DiscreteActionValueType
+    ) -> None:
+        if state_value in self._state_to_action:
+            warn('State value {} exists in rule; it will be replaced.'.format(state_value))
+        self._state_to_action[state_value] = action_value
+
+    def get_action_value(
+            self,
+            state_value: DiscreteStateValueType
+    ) -> DiscreteActionValueType:
+        return self._state_to_action.get(state_value)
+
+    def get_action(self, state: DiscreteState) -> Action:
+        return self._actions_dict[self.get_action_value(state.state_value)]
+
+    def __eq__(self, other) -> bool:
+        if len(self._state_to_action) != len(set(self._state_to_action.keys()).union(other._state_to_action.keys())):
+            return False
+        if len(self._actions_dict) != len(set(self._actions_dict.keys()).union(other._actions_dict.keys())):
+            return False
+        for action in self._actions_dict.keys():
+            if self._actions_dict[action] != other._actions_dict[action]:
+                return False
+        for state in self._state_to_action.keys():
+            if self._state_to_action[state] != other._state_to_action[state]:
+                return False
+        return True
+
+
+class DiscreteContinuousPolicy(DeterministicPolicy):
+    @abstractmethod
+    def get_action(self, state: State) -> Action:
+        raise NotImplemented()
+
+
+class StochasticPolicy(Policy):
+    @abstractmethod
+    def get_probability(self, *args, **kwargs) -> float:
+        raise NotImplemented()
+
+    @property
+    def is_stochastic(self) -> bool:
+        return True
+
+
+class DiscreteStochasticPolicy(StochasticPolicy):
+    def __init__(self, actions_dict: dict[DiscreteActionValueType, Action]):
+        self._state_to_action = {}
+        self._actions_dict = actions_dict
+
+    def add_stochastic_rule(
+            self,
+            state_value: DiscreteStateValueType,
+            action_values: list[DiscreteActionValueType],
+            probs: Union[list[float], Annotated[NDArray[np.float64], "1D Array"]] = None
+    ):
+        if probs is not None:
+            assert len(action_values) == len(probs)
+            probs = np.array(probs)
+        else:
+            probs = np.repeat(1./len(action_values), len(action_values))
+
+        if state_value in self._state_to_action:
+            warn('State value {} exists in rule; it will be replaced.'.format(state_value))
+        self._state_to_action[state_value] = {
+            action_value: prob
+            for action_value, prob in zip(action_values, probs)
+        }
+
+    def get_probability(
+            self,
+            state_value: DiscreteStateValueType,
+            action_value: DiscreteActionValueType
+    ) -> float:
+        if state_value not in self._state_to_action:
+            return 0.0
+        if action_value in self._state_to_action[state_value]:
+            return self._state_to_action[state_value][action_value]
+        else:
+            return 0.0
+
+    def get_action_value(self, state: State) -> DiscreteActionValueType:
+        allowed_actions = list(self._state_to_action[state].keys())
+        probs = np.array(list(self._state_to_action[state].values()))
+        sumprobs = np.sum(probs)
+        return np.random.choice(allowed_actions, p=probs/sumprobs)
+
+    def get_action(self, state: DiscreteState) -> Action:
+        return self._actions_dict[self.get_action_value(state.state_value)]
+
+
+class ContinuousStochasticPolicy(StochasticPolicy):
+    @abstractmethod
+    def get_probability(
+            self,
+            state_value: Union[float, Annotated[NDArray[np.float64], "1D Array"]],
+            action_value: DiscreteActionValueType,
+            value: Union[float, Annotated[NDArray[np.float64], "1D Array"]]
+    ) -> float:
+        raise NotImplemented()
+
+
+DiscretePolicy = Union[DiscreteDeterminsticPolicy, DiscreteStochasticPolicy]
+ContinuousPolicy = Union[ContinuousStochasticPolicy]

pyrlutils/reward.py

@@ -0,0 +1,37 @@
+
+from abc import ABC, abstractmethod
+
+
+class IndividualRewardFunction(ABC):
+    @abstractmethod
+    def reward(self, state_value, action_value, next_state_value) -> float:
+        pass
+
+    def __call__(self, state_value, action_value, next_state_value) -> float:
+        return self.reward(state_value, action_value, next_state_value)
+
+
+class RewardFunction(ABC):
+    def __init__(self, discount_factor: float, individual_reward_function: IndividualRewardFunction):
+        self._discount_factor = discount_factor
+        self._individual_reward_function = individual_reward_function
+
+    @property
+    def discount_factor(self) -> float:
+        return self._discount_factor
+
+    @discount_factor.setter
+    def discount_factor(self, discount_factor: float):
+        self._discount_factor = discount_factor
+
+    def individual_reward(self, state_value, action_value, next_state_value) -> float:
+        return self._individual_reward_function(state_value, action_value, next_state_value)
+
+    @abstractmethod
+    def total_reward(self, state_value, action_value) -> float:
+        pass
+
+    def __call__(self, state_value, action_value) -> float:
+        return self.total_reward(state_value, action_value)
+
+