pyrlutils 0.0.4__py3-none-any.whl

pyrlutils/action.py

@@ -0,0 +1,19 @@
+
+from types import LambdaType
+from typing import Union
+
+from .state import State
+
+
+DiscreteActionValueType = Union[float, str]
+
+class Action:
+    def __init__(self, actionfunc: LambdaType):
+        self._actionfunc = actionfunc
+
+    def act(self, state: State, *args, **kwargs) -> State:
+        self._actionfunc(state, *args, **kwargs)
+        return state
+
+    def __call__(self, state: State) -> State:
+        return self.act(state)

pyrlutils/bandit/algo.py

@@ -0,0 +1,128 @@
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+from .reward import IndividualBanditRewardFunction
+
+
+class BanditAlgorithm(ABC):
+    def __init__(self, action_values: list, reward_function: IndividualBanditRewardFunction):
+        self._action_values = action_values
+        self._reward_function = reward_function
+
+    @abstractmethod
+    def _go_one_loop(self):
+        pass
+
+    def loop(self, nbiterations: int):
+        for _ in range(nbiterations):
+            self._go_one_loop()
+
+    def reward(self, action_value) -> float:
+        return self._reward_function(action_value)
+
+    @abstractmethod
+    def get_action(self):
+        pass
+
+    @property
+    def action_values(self):
+        return self._action_values
+
+    @property
+    def reward_function(self) -> IndividualBanditRewardFunction:
+        return self._reward_function
+
+
+class SimpleBandit(BanditAlgorithm):
+    def __init__(
+            self,
+            action_values: list,
+            reward_function: IndividualBanditRewardFunction,
+            epsilon: float=0.05
+    ):
+        super().__init__(action_values, reward_function)
+        self._epsilon = epsilon
+        self._initialize()
+
+    def _initialize(self):
+        self._Q = np.zeros(len(self._action_values))
+        self._N = np.zeros(len(self._action_values), dtype=np.int32)
+
+    def _go_one_loop(self):
+        r = np.random.uniform()
+        if r < self.epsilon:
+            selected_action_idx = np.argmax(self._Q)
+        else:
+            selected_action_idx = np.random.choice(range(len(self._action_values)))
+        reward = self._reward_function(self._action_values[selected_action_idx])
+        self._N[selected_action_idx] += 1
+        self._Q[selected_action_idx] += (reward - self._Q[selected_action_idx]) / self._N[selected_action_idx]
+
+    def get_action(self):
+        selected_action_idx = np.argmax(self._Q)
+        return self._action_values[selected_action_idx]
+
+    @property
+    def epsilon(self) -> float:
+        return self._epsilon
+
+    @epsilon.setter
+    def epsilon(self, val: float):
+        self._epsilon = val
+
+
+class GradientBandit(BanditAlgorithm):
+    def __init__(self, action_values: list, reward_function: IndividualBanditRewardFunction, temperature: float=1.0, alpha: float=0.1):
+        super().__init__(action_values, reward_function)
+        self._T = temperature
+        self._alpha = alpha
+        self._initialize()
+
+    def _initialize(self):
+        self._preferences = np.zeros(len(self._action_values))
+        self._rewards_over_time = []
+
+    def _get_probs(self) -> np.ndarray:
+        # getting probabilities using softmax
+        exp_preferences = np.exp(self._preferences / self.T)
+        sum_exp_preferences = np.sum(exp_preferences)
+        return exp_preferences / sum_exp_preferences
+
+    def get_action(self):
+        selected_action_idx = np.argmax(self._preferences)
+        return self._action_values[selected_action_idx]
+
+    def _go_one_loop(self):
+        probs = self._get_probs()
+        selected_action_idx = np.random.choice(range(self._preferences.shape[0]), p=probs)
+        reward = self._reward_function(self._action_values[selected_action_idx])
+        self._rewards_over_time.append(reward)
+        average_reward = np.mean(self._rewards_over_time) if len(self._rewards_over_time) > 0 else 0.
+
+        for i in range(len(self._action_values)):
+            if i == selected_action_idx:
+                self._preferences[i] += self.alpha * (reward - average_reward) * (1 - probs[i])
+            else:
+                self._preferences[i] -= self.alpha * (reward - average_reward) * probs[i]
+
+    @property
+    def alpha(self) -> float:
+        return self._alpha
+
+    @alpha.setter
+    def alpha(self, val: float):
+        self._alpha = val
+
+    @property
+    def T(self) -> float:
+        return self._T
+
+    @T.setter
+    def T(self, val: float):
+        self._T = val
+
+    @property
+    def temperature(self) -> float:
+        return self._T

pyrlutils/bandit/reward.py

@@ -0,0 +1,11 @@
+
+from abc import ABC, abstractmethod
+
+
+class IndividualBanditRewardFunction(ABC):
+    @abstractmethod
+    def reward(self, action_value) -> float:
+        pass
+
+    def __call__(self, action_value) -> float:
+        return self.reward(action_value)

pyrlutils/openai/utils.py

@@ -0,0 +1,31 @@
+
+import gymnasium as gym
+
+from ..transition import TransitionProbabilityFactory, NextStateTuple
+
+
+class OpenAIGymDiscreteEnvironmentTransitionProbabilityFactory(TransitionProbabilityFactory):
+    def __init__(self, envname):
+        super().__init__()
+        self._envname = envname
+        self._gymenv = gym.make(envname)
+        self._convert_openai_gymenv_to_transprob()
+
+    def _convert_openai_gymenv_to_transprob(self):
+        P = self._gymenv.env.env.env.P
+        for state_value, trans_dict in P.items():
+            new_trans_dict = {}
+            for action_value, next_state_list in trans_dict.items():
+                new_trans_dict[action_value] = [
+                    NextStateTuple(next_state[1], next_state[0], next_state[2], next_state[3])
+                    for next_state in next_state_list
+                ]
+            self.add_state_transitions(state_value, new_trans_dict)
+
+    @property
+    def envname(self):
+        return self._envname
+
+    @property
+    def gymenv(self):
+        return self._gymenv

pyrlutils/policy.py

@@ -0,0 +1,84 @@
+
+from abc import ABC, abstractmethod
+from typing import Union, Dict
+from warnings import warn
+
+import numpy as np
+
+from .state import State, DiscreteState, DiscreteStateValueType
+from .action import Action, DiscreteActionValueType
+
+
+class Policy(ABC):
+    @abstractmethod
+    def get_action(self, state: State) -> Action:
+        pass
+
+    def __call__(self, state: State) -> Action:
+        return self.get_action(state)
+
+    @property
+    def is_stochastic(self) -> bool:
+        raise NotImplemented()
+
+
+class DeterministicPolicy(Policy):
+    @abstractmethod
+    def add_deterministic_rule(self, *args, **kwargs):
+        pass
+
+    @property
+    def is_stochastic(self) -> bool:
+        return False
+
+
+class DiscreteDeterminsticPolicy(DeterministicPolicy):
+    def __init__(self, actions_dict: Dict[DiscreteActionValueType, Action]):
+        self._state_to_action = {}
+        self._actions_dict = actions_dict
+
+    def add_deterministic_rule(self, state_value: DiscreteStateValueType, action_value: DiscreteActionValueType):
+        if state_value in self._state_to_action:
+            warn('State value {} exists in rule; it will be replaced.'.format(state_value))
+        self._state_to_action[state_value] = action_value
+
+    def get_action_value(self, state_value: DiscreteStateValueType) -> DiscreteActionValueType:
+        return self._state_to_action.get(state_value)
+
+    def get_action(self, state: DiscreteState) -> Action:
+        return self._actions_dict[self.get_action_value(state.state_value)]
+
+    def __eq__(self, other) -> bool:
+        if len(self._state_to_action) != len(set(self._state_to_action.keys()).union(other._state_to_action.keys())):
+            return False
+        if len(self._actions_dict) != len(set(self._actions_dict.keys()).union(other._actions_dict.keys())):
+            return False
+        for action in self._actions_dict.keys():
+            if self._actions_dict[action] != other._actions_dict[action]:
+                return False
+        for state in self._state_to_action.keys():
+            if self._state_to_action[state] != other._state_to_action[state]:
+                return False
+        return True
+
+
+class StochasticPolicy(Policy):
+    @abstractmethod
+    def get_probability(self, *args, **kwargs) -> float:
+        pass
+
+    @property
+    def is_stochastic(self) -> bool:
+        return True
+
+
+class DiscreteStochasticPolicy(StochasticPolicy):
+    @abstractmethod
+    def get_probability(self, state_value: DiscreteStateValueType, action_value: DiscreteActionValueType) -> float:
+        pass
+
+
+class ContinuousStochasticPolicy(StochasticPolicy):
+    @abstractmethod
+    def get_probability(self, state_value: Union[float, np.ndarray], action_value: DiscreteActionValueType, value: Union[float, np.ndarray]) -> float:
+        pass

pyrlutils/reward.py

@@ -0,0 +1,37 @@
+
+from abc import ABC, abstractmethod
+
+
+class IndividualRewardFunction(ABC):
+    @abstractmethod
+    def reward(self, state_value, action_value, next_state_value) -> float:
+        pass
+
+    def __call__(self, state_value, action_value, next_state_value) -> float:
+        return self.reward(state_value, action_value, next_state_value)
+
+
+class RewardFunction(ABC):
+    def __init__(self, discount_factor: float, individual_reward_function: IndividualRewardFunction):
+        self._discount_factor = discount_factor
+        self._individual_reward_function = individual_reward_function
+
+    @property
+    def discount_factor(self) -> float:
+        return self._discount_factor
+
+    @discount_factor.setter
+    def discount_factor(self, discount_factor: float):
+        self._discount_factor = discount_factor
+
+    def individual_reward(self, state_value, action_value, next_state_value) -> float:
+        return self._individual_reward_function(state_value, action_value, next_state_value)
+
+    @abstractmethod
+    def total_reward(self, state_value, action_value) -> float:
+        pass
+
+    def __call__(self, state_value, action_value) -> float:
+        return self.total_reward(state_value, action_value)
+
+

pyrlutils/state.py

@@ -0,0 +1,225 @@
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from dataclasses import dataclass
+from typing import Tuple, List, Optional, Union
+
+import numpy as np
+
+
+class StateValue(ABC):
+    @property
+    @abstractmethod
+    def value(self):
+        pass
+
+
+@dataclass
+class DiscreteStateValue(StateValue):
+    enum: Enum
+
+    @property
+    def value(self):
+        return self.enum.value
+
+    def name(self):
+        return self.enum.name
+
+
+class ContinuousStateValue(StateValue):
+    _value: float
+
+    @property
+    def value(self) -> float:
+        return self._value
+
+
+class State(ABC):
+    @property
+    def state_value(self):
+        return self.get_state_value()
+
+    @abstractmethod
+    def set_state_value(self, state_value):
+        pass
+
+    @abstractmethod
+    def get_state_value(self):
+        pass
+
+    @state_value.setter
+    def state_value(self, new_state_value):
+        self.set_state_value(new_state_value)
+
+
+DiscreteStateValueType = Union[float, str, Tuple[int], Enum]
+
+
+class DiscreteState(State):
+    def __init__(self, all_state_values: List[DiscreteStateValueType], initial_values: Optional[List[DiscreteStateValueType]] = None):
+        super().__init__()
+        self._all_state_values = all_state_values
+        self._state_value = initial_values if initial_values is not None and initial_values in self._all_state_values else self._all_state_values[0]
+
+    def get_state_value(self) -> DiscreteStateValueType:
+        return self._state_value
+
+    def set_state_value(self, state_value: DiscreteStateValueType):
+        if state_value in self._all_state_values:
+            self._state_value = state_value
+        else:
+            raise ValueError('State value {} is invalid.'.format(state_value))
+
+    def get_all_possible_state_values(self) -> List[DiscreteStateValueType]:
+        return self._all_state_values
+
+    @property
+    def state_value(self) -> DiscreteStateValueType:
+        return self._state_value
+
+    @state_value.setter
+    def state_value(self, new_state_value: DiscreteStateValueType):
+        self.set_state_value(new_state_value)
+
+    @property
+    def state_space_size(self):
+        return len(self._all_state_values)
+
+
+class InvalidRangeError(Exception):
+    def __init__(self, message=None):
+        self.message = "Invalid range error!" if message is None else message
+        super().__init__(self.message)
+
+
+class ContinuousState(State):
+    def __init__(self, nbdims: int, ranges: np.array, init_value: Optional[Union[float, np.ndarray]] = None):
+        self._nbdims = nbdims
+
+        try:
+            assert (ranges.dtype == np.float64) or (ranges.dtype == np.float32) or (ranges.dtype == np.float16)
+        except AssertionError:
+            raise TypeError('It has to be floating type numpy.ndarray.')
+
+        try:
+            assert self._nbdims > 0
+        except AssertionError:
+            raise ValueError('Number of dimensions must be positive.')
+
+        if self._nbdims > 1:
+            try:
+                assert self._nbdims == ranges.shape[0]
+            except AssertionError:
+                raise ValueError('Number of ranges does not meet the number of dimensions.')
+            try:
+                assert ranges.shape[1] == 2
+            except AssertionError:
+                raise ValueError("Only the smallest and largest values in `ranges'.")
+        else:
+            try:
+                assert ranges.shape[0] == 2
+            except AssertionError:
+                raise ValueError("Only the smallest and largest values in `ranges'.")
+
+        if self._nbdims > 1:
+            try:
+                for i in range(ranges.shape[0]):
+                    assert ranges[i, 0] <= ranges[i, 1]
+            except AssertionError:
+                raise InvalidRangeError()
+        else:
+            try:
+                assert ranges[0] <= ranges[1]
+            except AssertionError:
+                raise InvalidRangeError()
+
+        self._ranges = ranges if self._nbdims > 1 else np.expand_dims(ranges, axis=0)
+        if init_value is None:
+            self._state_value = np.zeros(self._nbdims)
+            for i in range(self._nbdims):
+                self._state_value[i] = np.random.uniform(self._ranges[i, 0], self._ranges[i, 1])
+        else:
+            if self._nbdims > 1:
+                try:
+                    assert init_value.shape[0] == self._nbdims
+                except AssertionError:
+                    raise ValueError('Initialized value does not have the right dimension.')
+                for i in range(self._nbdims):
+                    try:
+                        assert (init_value[i] >= self._ranges[i, 0]) and (init_value[i] <= self.ranges[i, 1])
+                    except AssertionError:
+                        raise InvalidRangeError('Initialized value at dimension {} (value: {}) is not within the permitted range ({} -> {})!'.format(i, init_value[i], self._ranges[i, 0], self._ranges[i, 1]))
+            else:
+                try:
+                    assert (init_value >= self._ranges[0, 0]) and (init_value <= self.ranges[0, 1])
+                except AssertionError:
+                    raise InvalidRangeError('Initialized value is out of range.')
+            self._state_value = init_value
+
+    def set_state_value(self, state_value: Union[float, np.ndarray]):
+        if self.nbdims > 1:
+            try:
+                assert state_value.shape[0] == self._nbdims
+            except AssertionError:
+                raise ValueError('Given value does not have the right dimension.')
+            for i in range(self.nbdims):
+                try:
+                    assert state_value[i] >= self.ranges[i, 0] and state_value[i] <= self.ranges[i, 1]
+                except AssertionError:
+                    raise InvalidRangeError()
+        else:
+            try:
+                assert state_value >= self.ranges[0, 0] and state_value <= self.ranges[0, 1]
+            except AssertionError:
+                raise InvalidRangeError()
+
+        self._state_value = state_value
+
+    def get_state_value(self) -> np.ndarray:
+        return self._state_value
+
+    def get_state_value_ranges(self) -> np.ndarray:
+        return self._ranges
+
+    def get_state_value_range_at_dimension(self, dimension: int) -> np.ndarray:
+        return self._ranges[dimension]
+
+    @property
+    def ranges(self) -> np.ndarray:
+        return self.get_state_value_ranges()
+
+    @property
+    def state_value(self) -> Union[float, np.ndarray]:
+        return self.get_state_value()
+
+    @state_value.setter
+    def state_value(self, new_state_value):
+        self.set_state_value(new_state_value)
+
+    @property
+    def nbdims(self) -> int:
+        return self._nbdims
+
+
+class Discrete2DCartesianState(DiscreteState):
+    def __init__(self, x_lowlim: int, x_hilim: int, y_lowlim: int, y_hilim: int, initial_coordinate: List[int]=None):
+        self._x_lowlim = x_lowlim
+        self._x_hilim = x_hilim
+        self._y_lowlim = y_lowlim
+        self._y_hilim = y_hilim
+        self._countx = self._x_hilim - self._x_lowlim + 1
+        self._county = self._y_hilim - self._y_lowlim + 1
+        if initial_coordinate is None:
+            initial_coordinate = [self._x_lowlim, self._y_lowlim]
+        initial_value = (initial_coordinate[1] - self._y_lowlim) * self._countx + (initial_coordinate[0] - self._x_lowlim)
+        super().__init__(list(range(self._countx*self._county)), initial_values=initial_value)
+
+    def _encode_coordinates(self, x, y) -> int:
+        return (y - self._y_lowlim) * self._countx + (x - self._x_lowlim)
+
+    def encode_coordinates(self, coordinates: List[int]) -> int:
+        assert len(coordinates) == 2
+        return self._encode_coordinates(coordinates[0], coordinates[1])
+
+    def decode_coordinates(self, hashcode) -> List[int]:
+        return [hashcode % self._countx, hashcode // self._countx]

pyrlutils/transition.py

@@ -0,0 +1,146 @@
+
+from types import LambdaType
+from typing import Tuple, Dict
+
+import numpy as np
+
+from .state import DiscreteState, DiscreteStateValueType
+from .reward import IndividualRewardFunction
+from .action import Action, DiscreteActionValueType
+
+
+class NextStateTuple:
+    def __init__(self, next_state_value: DiscreteStateValueType, probability: float, reward: float, terminal: bool):
+        self._next_state_value = next_state_value
+        self._probability = probability
+        self._reward = reward
+        self._terminal = terminal
+
+    @property
+    def next_state_value(self) -> DiscreteStateValueType:
+        return self._next_state_value
+
+    @property
+    def probability(self) -> float:
+        return self._probability
+
+    @property
+    def reward(self) -> float:
+        return self._reward
+
+    @property
+    def terminal(self) -> bool:
+        return self._terminal
+
+
+class TransitionProbabilityFactory:
+    def __init__(self):
+        self._transprobs = {}
+        self._all_state_values = []
+        self._all_action_values = []
+        self._objects_generated = False
+
+    def add_state_transitions(self, state_value: DiscreteStateValueType, action_values_to_next_state: dict):
+        if state_value not in self._all_state_values:
+            self._all_state_values.append(state_value)
+
+        this_state_transition_dict = {}
+
+        for action_value, next_state_tuples in action_values_to_next_state.items():
+            this_state_transition_dict[action_value] = []
+            for next_state_tuple in next_state_tuples:
+                if action_value not in self._all_action_values:
+                    self._all_action_values.append(action_value)
+                if not isinstance(next_state_tuple, NextStateTuple):
+                    if isinstance(next_state_tuple, dict):
+                        next_state_tuple = NextStateTuple(
+                            next_state_tuple['next_state_value'],
+                            next_state_tuple['probability'],
+                            next_state_tuple['reward'],
+                            next_state_tuple['terminal']
+                        )
+                    else:
+                        raise TypeError('"action_values_to_next_state" has to be a dictionary or NextStateTuple instance.')
+
+                if next_state_tuple.next_state_value not in self._all_state_values:
+                    self._all_state_values.append(next_state_tuple.next_state_value)
+
+                this_state_transition_dict[action_value].append(next_state_tuple)
+
+        self._transprobs[state_value] = this_state_transition_dict
+
+    def _get_probs_for_eachstate(self, action_value: DiscreteActionValueType) -> Dict[DiscreteStateValueType, NextStateTuple]:
+        state_nexttuples = {}
+        for state_value, action_nexttuples_pair in self._transprobs.items():
+            for this_action_value, nexttuples in action_nexttuples_pair.items():
+                if this_action_value == action_value:
+                    state_nexttuples[state_value] = nexttuples
+        return state_nexttuples
+
+    def _generate_action_function(self, state_nexttuples: dict) -> LambdaType:
+
+        def _action_function(state: DiscreteState) -> DiscreteState:
+            nexttuples = state_nexttuples[state.state_value]
+            nextstates = [nexttuple.next_state_value for nexttuple in nexttuples]
+            probs = [nexttuple.probability for nexttuple in nexttuples]
+            next_state_value = np.random.choice(nextstates, p=probs)
+            state.set_state_value(next_state_value)
+            return state
+
+        return _action_function
+
+    def _generate_individual_reward_function(self) -> IndividualRewardFunction:
+
+        def _individual_reward_function(state_value, action_value, next_state_value) -> float:
+            if state_value not in self._transprobs.keys():
+                return 0.
+
+            if action_value not in self._transprobs[state_value].keys():
+                return 0.
+
+            reward = 0.
+            for next_tuple in self._transprobs[state_value][action_value]:
+                if next_tuple.next_state_value == next_state_value:
+                    reward += next_tuple.reward
+            return reward
+
+        class ThisIndividualRewardFunction(IndividualRewardFunction):
+            def __init__(self):
+                super().__init__()
+
+            def reward(self, state_value, action_value, next_state_value) -> float:
+                return _individual_reward_function(state_value, action_value, next_state_value)
+
+        return ThisIndividualRewardFunction()
+
+    def get_probability(self, state_value, action_value, new_state_value) -> float:
+        if state_value not in self._transprobs.keys():
+            return 0.
+
+        if action_value not in self._transprobs[state_value]:
+            return 0.
+
+        probs = 0.
+        for next_state_tuple in self._transprobs[state_value][action_value]:
+            if next_state_tuple.next_state_value == new_state_value:
+                probs += next_state_tuple.probability
+        return probs
+
+    @property
+    def transition_probabilities(self) -> dict:
+        return self._transprobs
+
+    def generate_mdp_objects(self) -> Tuple[DiscreteState, Dict[DiscreteActionValueType, Action], IndividualRewardFunction]:
+        state = DiscreteState(self._all_state_values)
+        actions_dict = {}
+        for action_value in self._all_action_values:
+            state_nexttuple = self._get_probs_for_eachstate(action_value)
+            actions_dict[action_value] = Action(self._generate_action_function(state_nexttuple))
+
+        individual_reward_fcn = self._generate_individual_reward_function()
+
+        return state, actions_dict, individual_reward_fcn
+
+    @property
+    def objects_generated(self) -> bool:
+        return self._objects_generated

pyrlutils/valuefcns.py

@@ -0,0 +1,144 @@
+
+import random
+from copy import copy
+from typing import Tuple, Dict
+from itertools import product
+
+import numpy as np
+
+from .state import DiscreteStateValueType
+from .transition import TransitionProbabilityFactory
+from .policy import DiscreteDeterminsticPolicy
+
+
+class OptimalPolicyOnValueFunctions:
+    def __init__(self, discount_factor: float, transprobfac: TransitionProbabilityFactory):
+        try:
+            assert 0. <= discount_factor <= 1.
+        except AssertionError:
+            raise ValueError('Discount factor must be between 0 and 1.')
+        self._gamma = discount_factor
+        self._transprobfac = transprobfac
+        self._states, self._actions_dict, self._indrewardfcn = self._transprobfac.generate_mdp_objects()
+        self._state_names = self._states.get_all_possible_state_values()
+        self._states_to_indices = {state: idx for idx, state in enumerate(self._state_names)}
+        self._action_names = list(self._actions_dict.keys())
+        self._actions_to_indices = {action_value: idx for idx, action_value in enumerate(self._action_names)}
+
+        self._evaluated = False
+        self._improved = False
+
+        self._theta = 1e-10
+        self._policy_evaluation_maxiter = 10000
+
+    def _policy_evaluation(self, policy: DiscreteDeterminsticPolicy) -> np.ndarray:
+        prev_V = np.zeros(len(self._states_to_indices))
+
+        for _ in range(self._policy_evaluation_maxiter):
+            V = np.zeros(len(self._states_to_indices))
+            for state_value in self._state_names:
+                state_index = self._states_to_indices[state_value]
+                action_value = policy.get_action_value(state_value)
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    V[state_index] += prob * (reward + (self._gamma*prev_V[next_state_index] if not terminal else 0.))
+
+            if np.max(np.abs(V-prev_V)) < self._theta:
+                break
+
+            prev_V = V.copy()
+
+        return V
+
+    def _policy_improvement(self, V: np.ndarray) -> DiscreteDeterminsticPolicy:
+        Q = np.zeros((len(self._states_to_indices), len(self._actions_to_indices)))
+
+        for state_value in self._state_names:
+            state_index = self._states_to_indices[state_value]
+            for action_value in self._action_names:
+                action_index = self._actions_to_indices[action_value]
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    Q[state_index, action_index] += prob * (reward + (self._gamma*V[next_state_index] if not terminal else 0.))
+
+        optimal_policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        optimal_action_indices = np.argmax(Q, axis=1)
+        for state_value, action_index in zip(self._state_names, optimal_action_indices):
+            action_value = self._action_names[action_index]
+            optimal_policy.add_deterministic_rule(state_value, action_value)
+        return optimal_policy
+
+    def _policy_iteration(self) -> Tuple[np.ndarray, DiscreteDeterminsticPolicy]:
+        policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        for state_value in self._state_names:
+            policy.add_deterministic_rule(state_value, random.choice(self._action_names))
+        V = None
+
+        done = False
+        while not done:
+            old_policy = copy(policy)
+
+            V = self._policy_evaluation(policy)
+            policy = self._policy_improvement(V)
+
+            if policy == old_policy:
+                done = True
+
+        return V, policy
+
+
+    def _value_iteration(self) -> Tuple[np.ndarray, DiscreteDeterminsticPolicy]:
+        V = np.zeros(len(self._state_names))
+
+        for _ in range(self._policy_evaluation_maxiter):
+            Q = np.zeros((len(self._state_names), len(self._action_names)))
+            for state_value, action_value in product(self._state_names, self._action_names):
+                state_index = self._states_to_indices[state_value]
+                action_index = self._actions_to_indices[action_value]
+                for next_state_tuple in self._transprobfac.transition_probabilities[state_value][action_value]:
+                    prob = next_state_tuple.probability
+                    reward = next_state_tuple.reward
+                    next_state_value = next_state_tuple.next_state_value
+                    next_state_index = self._states_to_indices[next_state_value]
+                    terminal = next_state_tuple.terminal
+
+                    Q[state_index, action_index] += prob * (reward + (self._gamma * V[next_state_index] if not terminal else 0.))
+
+            if np.max(np.abs(V-np.max(Q, axis=1))) < self._theta:
+                break
+
+            V = np.max(Q, axis=1)
+
+        Qmaxj = np.argmax(Q, axis=1)
+
+        policy = DiscreteDeterminsticPolicy(self._actions_dict)
+        for state_value, action_index in zip(self._state_names, Qmaxj):
+            policy.add_deterministic_rule(state_value, self._action_names[action_index])
+
+        return V, policy
+
+    def policy_iteration(self) -> Tuple[Dict[DiscreteStateValueType, float], DiscreteDeterminsticPolicy]:
+        V, policy = self._policy_iteration()
+        state_values_dict = {
+            self._state_names[i]: V[i]
+            for i in range(V.shape[0])
+        }
+        return state_values_dict, policy
+
+    def value_iteration(self) -> Tuple[Dict[DiscreteStateValueType, float], DiscreteDeterminsticPolicy]:
+        V, policy = self._value_iteration()
+        state_values_dict = {
+            self._state_names[i]: V[i]
+            for i in range(V.shape[0])
+        }
+        return state_values_dict, policy

pyrlutils-0.0.4.dist-info/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Kwan Yuet Stephen Ho
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyrlutils-0.0.4.dist-info/METADATA

@@ -0,0 +1,42 @@
+Metadata-Version: 2.2
+Name: pyrlutils
+Version: 0.0.4
+Summary: Utility and Helpers for Reinformcement Learning
+Author-email: Kwan Yuet Stephen Ho <stephenhky@yahoo.com.hk>
+License: MIT
+Project-URL: Repository, https://github.com/stephenhky/PyRLUtils
+Project-URL: Issues, https://github.com/stephenhky/PyRLUtils/issues
+Keywords: machine learning,reinforcement leaning,artificial intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Version Control :: Git
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Provides-Extra: openaigym
+Requires-Dist: gymnasium; extra == "openaigym"
+Provides-Extra: test
+Requires-Dist: unittest; extra == "test"
+
+# PyRLUtils 
+
+[![CircleCI](https://circleci.com/gh/stephenhky/PyRLUtils.svg?style=svg)](https://circleci.com/gh/stephenhky/PyRLUtils.svg)
+[![GitHub release](https://img.shields.io/github/release/stephenhky/PyRLUtils.svg?maxAge=3600)](https://github.com/stephenhky/pyqentangle/PyRLUtils)
+[![pypi](https://img.shields.io/pypi/v/PyRLUtils.svg?maxAge=3600)](https://pypi.org/project/pyqentangle/)
+[![download](https://img.shields.io/pypi/dm/PyRLUtils.svg?maxAge=2592000&label=installs&color=%2327B1FF)](https://pypi.org/project/PyRLUtils/)
+[![Updates](https://pyup.io/repos/github/stephenhky/PyRLUtils/shield.svg)](https://pyup.io/repos/github/stephenhky/PyRLUtils/)
+[![Python 3](https://pyup.io/repos/github/stephenhky/PyRLUtils/python-3-shield.svg)](https://pyup.io/repos/github/stephenhky/PyRLUtils/)
+
+
+This is a Python package with utility classes and helper functions for
+that facilitates the development of any reinformecement learning projects.

pyrlutils-0.0.4.dist-info/RECORD

@@ -0,0 +1,17 @@
+pyrlutils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyrlutils/action.py,sha256=2kJqNZxsLOV8yOTl-RcpM8b0zu-WXNREJCrl49uZi2c,437
+pyrlutils/policy.py,sha256=Cx4vsIXzFZi_KEgI06S378Y5E6g-AfK90skDYoGsfOI,2794
+pyrlutils/reward.py,sha256=are0swsobMqI1IbrBVBaPMYXWpJnp6lZwAyfgBEm2zg,1211
+pyrlutils/state.py,sha256=w0YJ50FUyNboPoYduLMX1xaBJJHAOaSlsr3Og1dd0dY,7840
+pyrlutils/transition.py,sha256=lgh4YfOi-YjSIyymWfrXe-ugDWpZYK3MvjdeehgcQhk,5816
+pyrlutils/valuefcns.py,sha256=CJxu0EIFgrdbP0n0x6nzs3X08accFsuJW71tv1rMTkQ,6342
+pyrlutils/bandit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyrlutils/bandit/algo.py,sha256=X2Pn4DOi-RXWz5CNg1h0RJCoV3VlAwEGHRMjkfbckfw,3969
+pyrlutils/bandit/reward.py,sha256=S_uECjMOg3cmK24J-5uPcckLvtxmU4yllR7JEvMwAQE,249
+pyrlutils/openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyrlutils/openai/utils.py,sha256=ehj1cGlDYjQLno3pKMCS3CzZwbZGSTmjxDlU07aSBFo,1033
+pyrlutils-0.0.4.dist-info/LICENSE,sha256=bnQPjIcaeBdr2ZofX-_j-nELs8pAx5fQ4Cdfgeaspew,1063
+pyrlutils-0.0.4.dist-info/METADATA,sha256=7ncLjVrpqIZpdMFMrRjqRNgfZl9LUKc_SZFkw_CoTFc,2228
+pyrlutils-0.0.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+pyrlutils-0.0.4.dist-info/top_level.txt,sha256=gOBuxugE2MA4WDXlLhzkQh_rUonZU6nvJnMuomeHMCU,10
+pyrlutils-0.0.4.dist-info/RECORD,,

pyrlutils-0.0.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pyrlutils-0.0.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+pyrlutils