test-mdp 0.0.1__tar.gz

test_mdp-0.0.1/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: test-mdp
+Version: 0.0.1
+Summary: test for policy evaluation examples
+Home-page: https://github.com/limaries30/test-mdp
+Author: teddylee777
+Author-email: limaries30@kaist.ac.kr
+Keywords: policy evaluation
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Requires-Python: >=3.6
+Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

test_mdp-0.0.1/envs/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.0.1'

test_mdp-0.0.1/envs/baird.py

@@ -0,0 +1,125 @@
+'''
+Baird's counterexample environment
+'''
+from typing import Tuple
+
+import numpy as np
+from nptyping import NDArray
+
+from envs.base_mdp import BaseMDP
+from utils.math_utils import rowwise_kron, compute_stationary_dist
+
+class Baird(BaseMDP):
+
+    NUM_STATES = 7
+    NUM_ACTIONS = 2
+    num_features = 8
+
+    DASH = 0 
+    SOLID = 1
+
+    def __init__(self, gamma:float=0.9):
+
+        self.gamma = gamma
+        self.current_state = None
+        self.num_steps = 0
+
+        self.phi = self.construct_feature()
+        self.pi_beta, self.pi_target = self.construct_target_behavior_policy()
+        self.transition_mat = self.construct_transition()
+
+        self.p_pi = self.pi_target@self.transition_mat
+        self.p_beta = self.pi_beta@self.transition_mat
+        self.d_mu = np.diag(compute_stationary_dist(self.p_beta))
+
+
+        self.rewards = self.construct_reward()
+        self.expected_rewards = np.sum(self.p_pi*self.rewards, axis=1)
+
+
+        self.proj = self.phi @ np.linalg.pinv(self.phi.T@self.d_mu@self.phi) @ self.phi.T @ self.d_mu
+
+
+        self.sol = np.linalg.pinv(self.phi.T @ self.d_mu @ (np.eye(self.NUM_STATES) - self.gamma*self.p_pi) @ self.phi) @ (self.phi.T @ self.d_mu @ self.expected_rewards)
+
+
+    def construct_reward(self)->NDArray:
+        '''Construct reward matrix'''
+        self.rewards = np.zeros((self.NUM_STATES, self.NUM_STATES))
+        return self.rewards
+
+
+    def construct_target_behavior_policy(self)->Tuple[NDArray, NDArray]:
+        '''Construct target and behavior policy'''
+        self.target_policy = np.array([0, 1])
+        self.behavior_policy = np.array([1/6, 5/6])
+        self.target_policy = np.tile(self.target_policy, (self.NUM_STATES, 1))
+        self.behavior_policy = np.tile(self.behavior_policy, (self.NUM_STATES, 1))
+        self.pi_begta = rowwise_kron(np.eye(self.NUM_STATES), self.behavior_policy)
+        self.pi_target = rowwise_kron(np.eye(self.NUM_STATES), self.target_policy)
+
+        return self.pi_begta, self.pi_target
+
+    def construct_transition(self)->NDArray: 
+        '''Construct transition matrix size of SA times S'''
+        self.transition_mat = np.zeros((self.NUM_STATES*self.NUM_ACTIONS, self.NUM_STATES))
+        for s in range(0, self.NUM_STATES):
+            self.transition_mat[s*self.NUM_ACTIONS + self.SOLID, -1] = 1.0
+            self.transition_mat[s*self.NUM_ACTIONS + self.DASH, :-1] = 1/6
+        return self.transition_mat
+               
+    def construct_feature(self,)->NDArray:
+        '''Construct baird feature matrix'''
+        self.phi = np.zeros((self.NUM_STATES, self.num_features))
+        for s in range(self.NUM_STATES):
+            self.phi[s,s] = 2
+            self.phi[s,-1 ] = 1
+        self.phi[-1,-1] = 2
+        self.phi[-1, -2] = 1
+        return self.phi
+    
+
+    def reset(self)->Tuple[NDArray,dict]:
+        '''Return initial state'''
+
+        state = np.random.randint(0, self.NUM_STATES)
+        self.num_steps = 0
+
+        current_phi = self.phi[state,:]
+        self.current_state = state
+
+        info = {'rho': 0} # null value
+
+        return state, current_phi, info
+
+    def sample_action(self, state=None)->int:
+        '''Select action according to behavior policy'''
+
+        action = np.random.choice(np.arange(self.NUM_ACTIONS), 1, p=self.behavior_policy[state,:])[0]
+        return action
+
+    def step(self, state:int, action:int)->Tuple[NDArray,float,bool,bool,dict]:
+        '''Take action, return next state, reward, done, truncated, info'''
+        done = False
+        truncated = False 
+        self.num_steps += 1
+
+        next_state = np.random.choice(np.arange(self.NUM_STATES), p=self.transition_mat[state*self.NUM_ACTIONS+action,:])
+        next_phi = self.phi[next_state,:]
+        reward = self.rewards[state, next_state]
+        
+        info = {'rho':self.target_policy[state,action]/self.behavior_policy[state, action]}  
+
+        self.current_state = next_state
+        return next_state, next_phi, reward, done, truncated, info
+    
+    
+    def get_bellman_error(self, weight: np.ndarray) -> float:
+        be = self.proj @ (self.expected_rewards + self.gamma*self.p_pi @ self.phi @ weight - self.phi @ weight)
+        return np.linalg.norm(np.sqrt(self.d_mu)@be)
+    
+    def get_error(self, weight)->float:
+        error = np.linalg.norm(weight - self.sol )
+        return error
+
+    

test_mdp-0.0.1/envs/base_mdp.py

@@ -0,0 +1,101 @@
+'''
+Abstract Class for MDP
+'''
+
+from abc import ABC, abstractmethod
+from typing import Tuple, Dict, Any
+
+import numpy as np
+from nptyping import NDArray
+
+
+class BaseMDP(ABC):
+    '''
+    Base class for Markov Decision Process (MDP) environments.
+    
+    Subclasses must implement the following methods:
+    - reset()
+    - step()
+    - get_error()
+    '''
+
+
+    @abstractmethod
+    def construct_feature(self) -> NDArray:
+        '''
+        Construct the feature matrix for the MDP.
+        
+        Returns:
+            NDArray: The feature matrix.
+        '''
+    @abstractmethod
+    def construct_transition(self) -> NDArray:
+        '''
+        Construct the transition matrix for the MDP.
+        
+        Returns:
+            NDArray: The transition matrix.
+        '''
+    @abstractmethod
+    def construct_target_behavior_policy(self) -> Tuple[NDArray, NDArray]:
+        '''
+        Construct the behavior and target policy matrices.
+        
+        Returns:
+            Tuple[NDArray, NDArray]: The behavior and target policy matrices.
+        '''
+    
+    @abstractmethod
+    def reset(self) -> Tuple[int, NDArray, dict]:
+        '''
+        Resets the environment to an initial state.
+        
+        Returns:
+            Tuple: (state, feature, info)
+                - state: The initial state (int).
+                - feature: The feature representation of the initial state (NDArray).
+                - info: Additional information (dict).
+        '''
+
+    @abstractmethod
+    def step(self, action: int) -> Tuple[NDArray, float, bool, bool, Dict[str, Any]]:
+        '''
+        Takes an action and returns the next state, reward, and other information.
+
+        Args:
+            action (int): The action taken by the agent.
+
+        Returns:
+            Tuple: (next_state, reward, done, truncated, info)
+                - next_state: State after the action (NDArray).
+                - reward: Reward after the action (float).
+                - done: Boolean indicating whether the episode has ended.
+                - truncated: Boolean indicating if the episode was truncated.
+                - info: Additional information (e.g., importance sampling ratio).
+        '''
+
+    
+    @abstractmethod
+    def get_error(self, weight: np.ndarray) -> float:
+        '''
+        Calculate the error (e.g., in value function approximation).
+        
+        Args:
+            weight (np.ndarray): The weight parameter (e.g., for value function).
+
+        Returns:
+            float: The error based on the weight.
+        '''
+
+
+    @abstractmethod
+    def sample_action(self, state: int) -> int:
+        '''
+        Select an action based on the current state.
+        
+        Args:
+            state (int): The current state.
+
+        Returns:
+            int: The action selected by the agent.
+        '''

test_mdp-0.0.1/envs/env_utils.py

@@ -0,0 +1,17 @@
+'''
+  utils related to environments
+'''
+
+
+def get_env(name:str, **kwargs):
+    if name == "Baird":
+        from envs.baird import Baird
+        return Baird(kwargs.get('gamma', 0.9))
+    elif name == "RandomWalkTabular":
+        from envs.random_walk_tabular import RandomWalkTabular
+        return RandomWalkTabular(gamma=kwargs.get('gamma', 0.9))
+    elif name == "RandomWalkDependent":
+        from envs.random_walk_dependent import RandomWalkDependent
+        return RandomWalkDependent(gamma=kwargs.get('gamma', 0.9))
+    else:
+        raise NotImplementedError(f"Environment {name} not implemented")

test_mdp-0.0.1/envs/random_walk_dependent.py

@@ -0,0 +1,131 @@
+from envs.base_mdp import BaseMDP
+import numpy as np
+from nptyping import NDArray
+from typing import Tuple
+import copy
+from utils.math_utils import rowwise_kron, compute_stationary_dist 
+
+class RandomWalkDependent(BaseMDP):
+
+    NUM_STATES = 7
+    NUM_ACTIONS = 2
+    num_features = 3
+
+    LEFT = 0 
+    RIGHT = 1
+
+    END_0 = 0
+    END_1 = NUM_STATES-1
+    START = int((NUM_STATES-1)/2)
+
+    def __init__(self, gamma):
+
+
+        self.reward = 0
+        self.current_state = None
+        self.num_steps = 0
+        self.phi = self.construct_feature()
+        # Define target and behavior policy
+        # Target policy: go right with prob 0.6, left with prob 0
+
+        self.target_policy = np.array([0.4, 0.6])
+        self.behavior_policy = np.array([0.5, 0.5])
+        self.target_policy = np.tile(self.target_policy, (self.NUM_STATES, 1))
+        self.behavior_policy = np.tile(self.behavior_policy, (self.NUM_STATES, 1))
+
+        self.pi_beta, self.pi_target = self.construct_target_behavior_policy()
+        self.transition_mat = self.construct_transition()
+
+        self.p_pi = self.pi_target@self.transition_mat
+        self.p_beta = self.pi_beta@self.transition_mat
+        self.d_mu = np.diag(compute_stationary_dist(self.p_beta))
+
+
+        self.gamma = gamma
+        self.rewards = self.construct_reward()
+        self.expected_rewards = np.sum(self.p_pi*self.rewards, axis=1)
+
+        self.proj = self.phi @ np.linalg.pinv(self.phi.T@self.d_mu@self.phi) @ self.phi.T @ self.d_mu
+
+
+        self.sol = np.linalg.pinv(self.phi.T @ self.d_mu @ (np.eye(self.NUM_STATES) - self.gamma*self.p_pi) @ self.phi) @ (self.phi.T @ self.d_mu @ self.expected_rewards)
+
+    def construct_reward(self):
+        self.rewards = np.zeros((self.NUM_STATES, self.NUM_STATES))
+        self.rewards[self.END_0+1, self.END_0] = -1.0
+        self.rewards[self.END_1-1, self.END_1] = 1.0  
+        return self.rewards
+
+    def construct_feature(self):
+        self.phi = np.array([[1,0,0], [1,0,0], [1/np.sqrt(2), 1/np.sqrt(2) ,0],
+                             [1/np.sqrt(3), 1/np.sqrt(3), 1/np.sqrt(3)], 
+                             [0,1/np.sqrt(2),1/np.sqrt(2)],[0,0,1],[0,0,1]])
+        return self.phi
+    def construct_target_behavior_policy(self)->Tuple[NDArray, NDArray]:
+
+        self.target_policy = np.array([0.4, 0.6])
+        self.behavior_policy = np.array([0.5, 0.5])
+        self.target_policy = np.tile(self.target_policy, (self.NUM_STATES, 1))
+        self.behavior_policy = np.tile(self.behavior_policy, (self.NUM_STATES, 1))
+        
+        self.pi_beta = rowwise_kron(np.eye(self.NUM_STATES), self.behavior_policy)
+        self.pi_target = rowwise_kron(np.eye(self.NUM_STATES), self.target_policy)
+
+        return self.pi_beta, self.pi_target
+    
+    def construct_transition(self):
+
+        self.transition_mat = np.zeros((self.NUM_STATES*self.NUM_ACTIONS, self.NUM_STATES))
+        for s in range(1, self.NUM_STATES-1):
+            self.transition_mat[s*self.NUM_ACTIONS + self.LEFT, s-1] = 1.0
+            self.transition_mat[s*self.NUM_ACTIONS + self.RIGHT, s+1] = 1.0
+        self.transition_mat[0, self.START] = 1.0
+        self.transition_mat[1, self.START] = 1.0
+        self.transition_mat[(self.NUM_STATES-1)*self.NUM_ACTIONS + self.LEFT, self.START] = 1.0
+        self.transition_mat[(self.NUM_STATES-1)*self.NUM_ACTIONS + self.RIGHT, self.START] = 1.0
+
+        return self.transition_mat
+    
+    def reset(self)->Tuple[NDArray,dict]:
+        '''Return initial state'''
+
+        state = np.random.randint(0, self.NUM_STATES)
+        self.num_steps = 0
+
+        current_phi = self.phi[state,:]
+
+        info = {'rho':0}
+
+        return state, current_phi, info
+
+    def sample_action(self, state=None)->int:
+        '''Select action according to behavior policy'''
+
+        action = np.random.choice(np.arange(self.NUM_ACTIONS), 1, p=self.behavior_policy[state,:])[0]
+        return action
+
+    def step(self, state:int, action:int)->Tuple[NDArray,float,bool,bool,dict]:
+
+        done = False
+        truncated = False 
+        self.num_steps += 1
+      
+      
+        next_state = np.random.choice(np.arange(self.NUM_STATES), p=self.transition_mat[state*self.NUM_ACTIONS+action,:])
+        next_phi = self.phi[next_state,:]
+        reward = self.rewards[state, next_state]
+
+        info = {'rho':self.target_policy[state, action]/self.behavior_policy[state, action]}  
+        self.current_state = next_state
+        return next_state, next_phi, reward, done, truncated,info
+    
+    
+    def get_bellman_error(self, weight: np.ndarray) -> float:
+        be = self.proj @ (self.expected_rewards + self.gamma*self.p_pi @ self.phi @ weight - self.phi @ weight)
+        return np.linalg.norm(np.sqrt(self.d_mu)@be)
+    
+    def get_error(self, weight)->float:
+        error = np.linalg.norm(weight - self.sol )
+        return error
+
+    

test_mdp-0.0.1/envs/random_walk_tabular.py

@@ -0,0 +1,134 @@
+import numpy as np
+from nptyping import NDArray
+from typing import Tuple
+
+from envs.base_mdp import BaseMDP
+from utils.math_utils import rowwise_kron, compute_stationary_dist
+
+class RandomWalkTabular(BaseMDP):
+
+    NUM_STATES = 7
+    NUM_ACTIONS = 2
+    num_features = 7
+
+    LEFT = 0 
+    RIGHT = 1
+
+    END_0 = 0
+    END_1 = NUM_STATES-1
+    START = int((NUM_STATES-1)/2)
+
+    def __init__(self, gamma:float):
+
+        self.gamma = gamma
+        self.reward = 0
+        self.current_state = None
+        self.num_steps = 0
+        self.phi = self.construct_feature()
+
+
+        self.target_policy = np.array([0.4, 0.6])
+        self.behavior_policy = np.array([0.5, 0.5])
+        self.target_policy = np.tile(self.target_policy, (self.NUM_STATES, 1))
+        self.behavior_policy = np.tile(self.behavior_policy, (self.NUM_STATES, 1))
+
+        self.pi_beta , self.pi_target = self.construct_target_behavior_policy()
+        self.transition_mat = self.construct_transition()
+        
+        self.p_pi = self.pi_target@self.transition_mat
+        self.p_beta = self.pi_beta@self.transition_mat
+        self.d_mu = np.diag(compute_stationary_dist(self.p_beta))
+
+
+        self.rewards = self.construct_reward()
+        self.expected_rewards = np.sum(self.p_pi*self.rewards, axis=1)
+        
+        self.proj = self.phi @ np.linalg.pinv(self.phi.T@self.d_mu@self.phi) @ self.phi.T @ self.d_mu
+
+        self.sol = np.linalg.pinv(self.phi.T @ self.d_mu @ (np.eye(self.NUM_STATES) - self.gamma*self.p_pi) @ self.phi) @ (self.phi.T @ self.d_mu @ self.expected_rewards)
+
+
+    def construct_feature(self):
+        self.phi = np.eye(self.NUM_STATES)
+        return self.phi
+    
+    def construct_target_behavior_policy(self)->Tuple[NDArray, NDArray]:
+
+        self.target_policy = np.array([0.4, 0.6])
+        self.behavior_policy = np.array([0.5, 0.5])
+        self.target_policy = np.tile(self.target_policy, (self.NUM_STATES, 1))
+        self.behavior_policy = np.tile(self.behavior_policy, (self.NUM_STATES, 1))
+        
+        self.pi_beta = rowwise_kron(np.eye(self.NUM_STATES), self.behavior_policy)
+        self.pi_target = rowwise_kron(np.eye(self.NUM_STATES), self.target_policy)
+
+        return self.pi_beta, self.pi_target
+    
+    def construct_transition(self):
+
+        self.transition_mat = np.zeros((self.NUM_STATES*self.NUM_ACTIONS, self.NUM_STATES))
+        for s in range(1, self.NUM_STATES-1):
+            self.transition_mat[s*self.NUM_ACTIONS + self.LEFT, s-1] = 1.0
+            self.transition_mat[s*self.NUM_ACTIONS + self.RIGHT, s+1] = 1.0
+        self.transition_mat[0, self.START] = 1.0
+        self.transition_mat[1, self.START] = 1.0
+        self.transition_mat[(self.NUM_STATES-1)*self.NUM_ACTIONS + self.LEFT, self.START] = 1.0
+        self.transition_mat[(self.NUM_STATES-1)*self.NUM_ACTIONS + self.RIGHT, self.START] = 1.0
+
+        return self.transition_mat
+    def construct_reward(self):
+        self.rewards = np.zeros((self.NUM_STATES, self.NUM_STATES))
+        self.rewards[self.END_0+1, self.END_0] = -1.0
+        self.rewards[self.END_1-1, self.END_1] = 1.0  
+        self.expected_rewards = np.sum(self.p_pi*self.rewards, axis=1)
+        return self.rewards
+    
+
+    def reset(self)->Tuple[NDArray,dict]:
+        '''Return initial state'''
+
+        state = np.random.randint(0, self.NUM_STATES)
+        self.num_steps = 0
+
+        current_phi = self.phi[state,:]
+
+        info = {'rho':1}
+
+        self.current_state = state
+        self.current_phi = current_phi
+
+
+        return state, current_phi ,info
+
+    def sample_action(self, state=None)->int:
+        '''Select action according to behavior policy'''
+
+        action = np.random.choice(np.arange(self.NUM_ACTIONS), 1, p=self.behavior_policy[state,:])[0]
+        return action
+
+    def step(self, state:int, action:int)->Tuple[NDArray,float,bool,bool,dict]:
+
+        done = False
+        truncated = False 
+        self.num_steps += 1
+      
+      
+        next_state = np.random.choice(np.arange(self.NUM_STATES),
+                                       p=self.transition_mat[state*self.NUM_ACTIONS+action,:])
+        next_phi = self.phi[next_state,:]
+        reward = self.rewards[state, next_state]
+
+        info = {'rho':self.target_policy[state, action]/self.behavior_policy[state, action]}
+        self.current_state = next_state
+        return next_state, next_phi, reward, done, truncated,info
+    
+    
+    def get_bellman_error(self, weight: np.ndarray) -> float:
+        be = self.proj @ (self.expected_rewards + self.gamma*self.p_pi @ self.phi @ weight - self.phi @ weight)
+        return np.linalg.norm(np.sqrt(self.d_mu)@be)
+    
+    def get_error(self, weight)->float:
+        error = np.linalg.norm(weight - self.sol )
+        return error
+
+    

test_mdp-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

test_mdp-0.0.1/setup.py

@@ -0,0 +1,22 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='test-mdp',
+    version='0.0.1',
+    description='test for policy evaluation examples',
+    author='teddylee777',
+    author_email='limaries30@kaist.ac.kr',
+    url='https://github.com/limaries30/test-mdp',
+    install_requires=['tqdm',],
+    packages=find_packages(exclude=[]),
+    keywords=['policy evaluation'],
+    python_requires='>=3.6',
+    package_data={},
+    zip_safe=False,
+    classifiers=[
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+    ],
+)

test_mdp-0.0.1/test_mdp.egg-info/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: test-mdp
+Version: 0.0.1
+Summary: test for policy evaluation examples
+Home-page: https://github.com/limaries30/test-mdp
+Author: teddylee777
+Author-email: limaries30@kaist.ac.kr
+Keywords: policy evaluation
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Requires-Python: >=3.6
+Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

test_mdp-0.0.1/test_mdp.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+setup.py
+envs/__init__.py
+envs/baird.py
+envs/base_mdp.py
+envs/env_utils.py
+envs/random_walk_dependent.py
+envs/random_walk_tabular.py
+test_mdp.egg-info/PKG-INFO
+test_mdp.egg-info/SOURCES.txt
+test_mdp.egg-info/dependency_links.txt
+test_mdp.egg-info/not-zip-safe
+test_mdp.egg-info/requires.txt
+test_mdp.egg-info/top_level.txt

test_mdp-0.0.1/test_mdp.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

test_mdp-0.0.1/test_mdp.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

test_mdp-0.0.1/test_mdp.egg-info/requires.txt

@@ -0,0 +1 @@
+tqdm

test_mdp-0.0.1/test_mdp.egg-info/top_level.txt

@@ -0,0 +1 @@
+envs