samsara-rl 0__py3-none-any.whl

samsara_rl/mdp/grid_world/grid_world_mdp.py

@@ -0,0 +1,103 @@
+from enum import Enum
+
+import numpy as np
+
+
+class GridWorldActions(Enum):
+    UP = 0
+    DOWN = 1
+    LEFT = 2
+    RIGHT = 3
+
+
+class GridWorldMDP:
+    """
+    Sample 4x4 Grid world as described in David Silvers 3rd RL Lecture.
+    Consists of 4 actions, UP,DOWN,LEFT,RIGHT. Fully deterministic, transition probability is 1.0 for an action (i.e. UP will always
+    move up.). Default initial policy is uniform random.
+    @author ashish juneja
+    """
+
+    ACTION_COUNT = 4
+    STATE_COUNT = 16
+
+    def __init__(self) -> None:
+        self.terminal_states: list[tuple[int, int] | int] = [(0, 0), (3, 3), 0, 15]
+        self.state_action_transition_matrix = np.zeros((16, 4, 16))
+        self.reward_matrix = np.full((16, 4, 16), -1)
+
+        self.actions = {
+            GridWorldActions.UP.value: self.up,
+            GridWorldActions.DOWN.value: self.down,
+            GridWorldActions.LEFT.value: self.left,
+            GridWorldActions.RIGHT.value: self.right,
+        }
+        self.init_transition_probabilities()
+        self.init_reward_matrix()
+
+    def step(self, state: int, action: int) -> tuple[int, int]:
+        curr_state_transitions = self.state_action_transition_matrix[state][action]
+        next_state = np.random.choice(len(curr_state_transitions), p=curr_state_transitions)
+        reward = self.reward_matrix[state][action][next_state]
+        return reward, next_state
+
+    def init_reward_matrix(self) -> None:
+        self.reward_matrix[0, :, 0] = 0
+        self.reward_matrix[15, :, 15] = 0
+
+    def init_transition_probabilities(self) -> None:
+        """
+        Initialize a 16x4x16 numpy matrix that captures the GridWorld
+        Deterministic Dynamics. 16 possible starting states, 4 actions from each of those states
+        and 16 possible states the agent can end up in after invoking one of the 4 UP,DOWN,LEFT,RIGHT
+        actions. All actions are deterministic, ex: UP has 100% probability of upward movement and 0% into any
+        of the other 15 states. At boarder collisions agent ends up in current state before action
+        """
+        for curr_state_next_action in np.ndindex(self.state_action_transition_matrix.shape[:-1]):
+            curr_state, action = curr_state_next_action
+            curr_row, curr_col = self._unflatten(curr_state)
+            next_state = self.actions[action]((curr_row, curr_col))[0]
+            flattened_next_state = self._flatten(next_state[0], next_state[1])
+            self.state_action_transition_matrix[curr_state, action, flattened_next_state] = 1
+
+    def move(self, state: tuple[int, int], x_direction: int, y_direction: int) -> list[tuple[int, int]]:
+        next_step = (state[0] + x_direction, state[1] + y_direction)
+        if state in self.terminal_states or not self.in_bounds(next_step[0], next_step[1]):
+            return [state]
+        else:
+            return [next_step]
+
+    def up(self, state: tuple[int, int]) -> list[tuple[int, int]]:
+        return self.move(state, x_direction=-1, y_direction=0)
+
+    def down(self, state: tuple[int, int]) -> list[tuple[int, int]]:
+        return self.move(state, x_direction=1, y_direction=0)
+
+    def left(self, state: tuple[int, int]) -> list[tuple[int, int]]:
+        return self.move(state, x_direction=0, y_direction=-1)
+
+    def right(self, state: tuple[int, int]) -> list[tuple[int, int]]:
+        return self.move(state, x_direction=0, y_direction=1)
+
+    def in_bounds(self, x: int, y: int) -> bool:
+        return x in range(0, 4) and y in range(0, 4)
+
+    def _unflatten(self, state: int) -> tuple[int, int]:
+        """
+        Folds the tensor addressable 0-15 state into a 4x4 human readable coordinate
+        """
+        row = state // 4
+        col = state % 4
+        return row, col
+
+    def _flatten(self, row: int, col: int) -> int:
+        """
+        Folds the human readable row,col coordinate into a machine addressable 0..15
+        """
+        return 4 * row + col
+
+    def initial_state(self) -> int:
+        return self._flatten(2, 2)
+
+    def is_terminal_state(self, state: int) -> bool:
+        return state in self.terminal_states

samsara_rl/planning/planning.py

@@ -0,0 +1,41 @@
+import math
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+
+
+class Planning(ABC):
+    def __init__(self, mdp: Any, bellman_tolerance: float = 0.01) -> None:
+        self.mdp = mdp
+        self.bellman_tolerance = bellman_tolerance
+        self.policy: np.ndarray = np.zeros((self.mdp.STATE_COUNT, self.mdp.ACTION_COUNT))
+        self.values: np.ndarray = np.zeros(self.mdp.STATE_COUNT)
+
+    @abstractmethod
+    def find_optimal_policy(self) -> np.ndarray:
+        pass
+
+    def _bellman_error(self, value_history: list[np.ndarray]) -> float:
+        """Return the sum of absolute value changes between the last two iterations."""
+        if len(value_history) > 1:
+            return float(np.absolute(value_history[-1] - value_history[-2]).sum())
+        return math.inf
+
+    def improve_policy(self) -> None:
+        """
+        Update the policy greedily with respect to the current state values.
+
+        For each state, assigns probability 1.0 to the action with the highest
+        expected next-state value (ties are broken by np.argmax, i.e. first index).
+
+        Updates self.policy in place.
+        """
+        # Expected next-state value for each (s, a): shape (S, A)
+        expected_next_value = np.dot(self.mdp.state_action_transition_matrix, self.values)
+
+        best_actions = np.argmax(expected_next_value, axis=1)
+        state_indices = np.arange(self.mdp.STATE_COUNT)
+
+        self.policy = np.zeros((self.mdp.STATE_COUNT, self.mdp.ACTION_COUNT))
+        self.policy[state_indices, best_actions] = 1.0

samsara_rl/planning/policy_iteration.py

@@ -0,0 +1,84 @@
+from typing import Any
+
+import numpy as np
+
+from samsara_rl.planning.planning import Planning
+
+
+class PolicyIteration(Planning):
+    """
+    Policy Iteration for finite, discrete MDPs.
+
+    Alternates between two steps until the policy stops changing:
+      1. Policy Evaluation  — compute V(s) for the current policy via iterative
+                              application of the Bellman expectation equation.
+      2. Policy Improvement — update the policy greedily with respect to V(s).
+
+    Follows the formulation in David Silver's RL Lecture 3 (Planning by DP).
+
+    Attributes:
+        mdp: The MDP to solve. Must expose STATE_COUNT, ACTION_COUNT,
+             state_action_transition_matrix, and reward_matrix.
+        bellman_tolerance: Convergence threshold for policy evaluation.
+        policy: Current policy as a (STATE_COUNT, ACTION_COUNT) probability
+                matrix. Initialised to the uniform random policy.
+        values: Current state-value estimate, shape (STATE_COUNT,).
+    """
+
+    def __init__(self, mdp: "Any", bellman_tolerance: float = 0.01) -> None:
+        super().__init__(mdp, bellman_tolerance)
+        self.policy = self._init_policy()
+
+    def find_optimal_policy(self, max_iter: int = 99) -> np.ndarray:
+        """
+        Run policy iteration until the policy converges or max_iter is reached.
+
+        Returns:
+            The converged (optimal) policy array of shape (STATE_COUNT, ACTION_COUNT).
+        """
+        previous_policy: np.ndarray | None = None
+        iteration = 0
+
+        while not self._is_converged(previous_policy) and iteration < max_iter:
+            self.evaluate_policy()
+            previous_policy = np.copy(self.policy)
+            self.improve_policy()
+            iteration += 1
+
+        return self.policy
+
+    def evaluate_policy(self) -> None:
+        """
+        Estimate state values for the current policy via iterative Bellman updates.
+
+        Applies the Bellman expectation equation repeatedly:
+            V(s) = Σ_a π(a|s) Σ_s' T(s,a,s') [R(s,a,s') + V(s')]
+        until the change in values falls below bellman_tolerance.
+
+        Updates self.values in place.
+        """
+        value_history: list[np.ndarray] = []
+
+        while self._bellman_error(value_history) > self.bellman_tolerance:
+            # Expected next-state value for each (s, a): shape (S, A)
+            expected_next_value = np.dot(self.mdp.state_action_transition_matrix, self.values)
+
+            # Expected immediate reward for each (s, a): shape (S, A)
+            expected_reward = (self.mdp.reward_matrix * self.mdp.state_action_transition_matrix).sum(axis=-1)
+
+            # Weight by policy probabilities and collapse over actions: shape (S,)
+            new_values = (self.policy * (expected_next_value + expected_reward)).sum(axis=-1)
+
+            value_history.append(new_values)
+            self.values = new_values
+
+    def _init_policy(self) -> np.ndarray:
+        """Return a uniform random policy of shape (STATE_COUNT, ACTION_COUNT)."""
+        return np.full(
+            (self.mdp.STATE_COUNT, self.mdp.ACTION_COUNT),
+            1.0 / self.mdp.ACTION_COUNT,
+        )
+
+    def _is_converged(self, previous_policy: np.ndarray | None) -> bool:
+        """Return True if the policy is unchanged since the last improvement step."""
+        return previous_policy is not None and np.array_equal(previous_policy, self.policy)

samsara_rl/planning/value_iteration.py

@@ -0,0 +1,55 @@
+from typing import Any
+
+import numpy as np
+
+from samsara_rl.planning.planning import Planning
+
+
+class ValueIteration(Planning):
+    """
+    Value Iteration for finite, discrete MDPs.
+
+    Iteratively applies the Bellman optimality equation until state values
+    converge, then extracts a greedy policy from the converged values.
+
+    Follows the formulation in David Silver's RL Lecture 3 (Planning by DP).
+    """
+
+    def __init__(self, mdp: "Any", bellman_tolerance: float = 0.01) -> None:
+        super().__init__(mdp, bellman_tolerance)
+
+    def find_optimal_policy(self) -> np.ndarray:
+        """
+        Run value iteration to convergence, then extract a greedy policy.
+
+        Returns:
+            The optimal policy as a (STATE_COUNT, ACTION_COUNT) array.
+        """
+        self._evaluate_value_function()
+        self.improve_policy()
+        return self.policy
+
+    def _evaluate_value_function(self) -> None:
+        """
+        Estimate optimal state values via iterative Bellman optimality updates.
+
+        For each iteration, applies:
+            V(s) = max_a Σ_s' T(s,a,s') [R(s,a,s') + V(s')]
+        until the change in values falls below bellman_tolerance.
+
+        Updates self.values in place.
+        """
+        value_history: list[np.ndarray] = []
+
+        while self._bellman_error(value_history) > self.bellman_tolerance:
+            # Expected next-state value for each (s, a): shape (S, A)
+            expected_next_value = np.dot(self.mdp.state_action_transition_matrix, self.values)
+
+            # Expected immediate reward for each (s, a): shape (S, A)
+            expected_reward = (self.mdp.reward_matrix * self.mdp.state_action_transition_matrix).sum(axis=2)
+
+            # Bellman optimality: take max over actions, shape (S,)
+            new_values = np.max(expected_reward + expected_next_value, axis=1)
+
+            self.values = new_values
+            value_history.append(new_values.copy())

samsara_rl/prediction/monte_carlo.py

@@ -0,0 +1,74 @@
+from typing import Any
+
+import numpy as np
+
+from samsara_rl.prediction.policy_evaluation import PolicyEvaluation
+
+
+class MonteCarloPrediction(PolicyEvaluation):
+    """Every-visit Monte Carlo prediction for estimating Q(s, a).
+
+    Generates episodes under a given policy and updates action-value
+    estimates using constant-alpha MC learning. Returns are computed
+    in a fully vectorized manner without explicit loops over time steps.
+
+    Args:
+        mdp: Environment with ``STATE_COUNT``, ``ACTION_COUNT``,
+            ``initial_state()``, ``is_terminal_state()``, and ``step()``
+            methods.
+        policy: A stochastic policy array of shape
+            ``(STATE_COUNT, ACTION_COUNT)`` used to sample actions.
+        alpha: Learning rate for incremental Q updates.
+        gamma: Discount factor.
+    """
+
+    def __init__(self, mdp: Any, policy: np.ndarray, alpha: float = 0.01, gamma: float = 1) -> None:
+        super().__init__(mdp, policy, alpha, gamma)
+
+    def post_visit(self, trajectory: np.ndarray) -> None:
+        return
+
+    def post_episode(self, trajectory: np.ndarray) -> None:
+        """Update Q-table from a single episode trajectory.
+
+        Uses advanced indexing to apply the constant-alpha MC update
+        Q(s,a) <- Q(s,a) + alpha * (G - Q(s,a)) for every visited
+        (state, action) pair in the trajectory.
+
+        Args:
+            trajectory: Array of shape ``(T, 3)`` where each row is
+                ``[state, action, reward]``.
+        """
+        discounted_trajectory = self._discounted_cum_trajectory(trajectory)
+        trajectory = np.column_stack((trajectory, discounted_trajectory))
+        s_a_pairs = trajectory[:, 0:2].astype(np.int16)
+        s = s_a_pairs[:, 0]
+        a = s_a_pairs[:, 1]
+        bellman_error = self.alpha * (discounted_trajectory - self.q_table[s, a])
+        np.add.at(self.q_table, (s, a), bellman_error)
+
+    def _discounted_cum_trajectory(self, trajectory: np.ndarray) -> np.ndarray:
+        """Compute discounted returns for every time step, vectorized.
+
+        Avoids the standard O(T) reverse loop by factoring out discount
+        weights from a cumulative sum:
+
+        1. Divide each reward by its positional gamma power to normalize.
+        2. Reverse and cumsum so earlier states accumulate future rewards.
+        3. Multiply back by gamma powers to restore correct discounting.
+
+        Args:
+            trajectory: Array of shape ``(T, 3)`` where column 2 contains
+                rewards.
+
+        Returns:
+            Array of shape ``(T,)`` with the discounted return G_t for
+            each time step.
+        """
+        discount_ratio = self.gamma ** np.arange(0, len(trajectory[:, 2]))[::-1]
+        reversed_reward = trajectory[:, 2]
+        reversed_reward = reversed_reward / discount_ratio
+        reversed_reward_cum = reversed_reward[::-1].cumsum()
+        reversed_reward_cum = reversed_reward_cum * discount_ratio[::-1]
+        result: np.ndarray = reversed_reward_cum[::-1]
+        return result

samsara_rl/prediction/policy_evaluation.py

@@ -0,0 +1,100 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+
+from samsara_rl.utils.policy.policy_utils import sample
+
+
+class PolicyEvaluation(ABC):
+    """Base class for model-free policy evaluation methods.
+
+    Provides the episode-generation loop (template method pattern) and
+    defines hooks that subclasses override to implement algorithm-specific
+    update logic.
+
+    Args:
+        mdp: Environment with ``STATE_COUNT``, ``ACTION_COUNT``,
+            ``initial_state()``, ``is_terminal_state()``, and ``step()``
+            methods.
+        policy: Stochastic policy array of shape ``(S, A)`` where each
+            row sums to 1.
+        alpha: Learning rate for incremental Q updates.
+        gamma: Discount factor applied to future rewards.
+    """
+
+    def __init__(self, mdp: Any, policy: np.ndarray, alpha: float = 0.01, gamma: float = 0.9) -> None:
+        self.q_table: np.ndarray = np.zeros((mdp.STATE_COUNT, mdp.ACTION_COUNT))
+        self.mdp = mdp
+        self.policy: np.ndarray = policy
+        self.gamma: float = gamma
+        self.alpha: float = alpha
+
+    @abstractmethod
+    def post_visit(self, trajectory: np.ndarray) -> None:
+        """Called after each step within an episode.
+
+        Subclasses use this to perform per-step updates (e.g. TD
+        updates with eligibility traces). No-op for methods that
+        only update at episode end (e.g. Monte Carlo).
+
+        Args:
+            trajectory: Array of shape ``(t, 3)`` containing all steps
+                recorded so far in the current episode, where each row
+                is ``[state, action, reward]``.
+        """
+        pass
+
+    @abstractmethod
+    def post_episode(self, trajectory: np.ndarray) -> None:
+        """Called after a complete episode has been generated.
+
+        Subclasses use this to perform end-of-episode updates
+        (e.g. Monte Carlo return-based Q updates) or to reset
+        per-episode state (e.g. eligibility traces).
+
+        Args:
+            trajectory: Array of shape ``(T, 3)`` containing the full
+                episode, where each row is ``[state, action, reward]``.
+        """
+        pass
+
+    def run_episode(self) -> np.ndarray:
+        """Generate a complete episode under the current policy.
+
+        Samples actions from the policy and steps through the MDP until
+        a terminal state is reached. Calls ``post_visit`` after each
+        step so subclasses can perform online updates.
+
+        Returns:
+            np.ndarray: Array of shape ``(T, 3)`` where each row is
+                ``[state, action, reward]``.
+        """
+        curr_state, curr_step = self.mdp.initial_state(), 0
+
+        trajectory = np.zeros((1000, 3))
+
+        while not self.mdp.is_terminal_state(curr_state):
+            sampled_action = sample(self.policy, curr_state)
+            reward, next_state = self.mdp.step(curr_state, sampled_action)
+            trajectory[curr_step][0] = curr_state
+            trajectory[curr_step][1] = sampled_action
+            trajectory[curr_step][2] = reward
+            curr_step += 1
+            curr_state = next_state
+            self.post_visit(trajectory[0:curr_step])
+
+        return trajectory[0:curr_step]
+
+    def evaluate(self, max_iter: int = 1000) -> None:
+        """Run prediction for a fixed number of episodes.
+
+        Generates episodes and calls ``post_episode`` after each one,
+        allowing subclasses to perform end-of-episode updates.
+
+        Args:
+            max_iter: Number of episodes to sample and learn from.
+        """
+        for _ in range(0, max_iter):
+            trajectory = self.run_episode()
+            self.post_episode(trajectory)

samsara_rl/prediction/td.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+import numpy as np
+
+from samsara_rl.prediction.policy_evaluation import PolicyEvaluation
+
+
+class TemporalDifference(PolicyEvaluation):
+    def __init__(
+        self, mdp: Any, policy: np.ndarray, alpha: float = 0.01, gamma: float = 0.9, _lambda: float = 0.4
+    ) -> None:
+        super().__init__(mdp, policy, alpha, gamma)
+        self.elibility = np.zeros((mdp.STATE_COUNT, self.mdp.ACTION_COUNT))
+        self._lambda = _lambda
+
+    def post_episode(self, trajectory: np.ndarray) -> None:
+        self.elibility = np.zeros((self.mdp.STATE_COUNT, self.mdp.ACTION_COUNT))
+
+    def post_visit(self, trajectory: np.ndarray) -> None:
+        if trajectory.shape[0] < 2:
+            return
+        self.elibility = self.elibility * self._lambda
+        visited_state = trajectory[-2][0].astype(np.int8)
+        el_action = trajectory[-2][1]
+        self.elibility[visited_state][el_action.astype(np.int8)] = 1
+
+        R_prime = trajectory[-2][2]
+        S_prime = trajectory[-1][0].astype(np.int8)  # state we ended up in after A
+        Q_expectation = self.q_table[S_prime].dot(self.policy[S_prime])  # optimal known action from next step
+
+        S = trajectory[-2][0].astype(np.int8)  # state before taking A
+        A = trajectory[-2][1].astype(np.int8)  # action we took to end up in S'
+        Q = self.q_table[S][A]  # Estimate of how good S,A is before taking
+
+        td_error = (R_prime + self.gamma * Q_expectation) - Q
+        self.q_table = self.q_table + self.alpha * self.elibility * td_error

samsara_rl/utils/policy/policy_utils.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+import numpy as np
+
+
+def init_uniform_random(mdp: Any) -> np.ndarray:
+    """Return uniform random policy given an MDP
+    of all actions being uniform"""
+    return np.full(
+        (mdp.STATE_COUNT, mdp.ACTION_COUNT),
+        1.0 / mdp.ACTION_COUNT,
+    )
+
+
+def sample(policy: np.ndarray, s: int) -> int:
+    """Returns a sample action based on an input policy"""
+    action_distribution = policy[s]
+    selected_action = np.random.choice(len(action_distribution), p=action_distribution)
+    return selected_action

samsara_rl-0.dist-info/METADATA

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: samsara-rl
+Version: 0
+Summary: Vectorized Reinforcement Learning Library
+Project-URL: Homepage, https://swordofzeus.github.io/samsara-rl/
+Project-URL: Repository, https://github.com/swordofzeus/samsara-rl
+Project-URL: Documentation, https://swordofzeus.github.io/samsara-rl/
+Author-email: Ashish Juneja <ashishjuneja99@gmail.com>
+License-File: LICENSE
+Keywords: python
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <4.0,>=3.10
+Requires-Dist: numpy>=2.2.6
+Description-Content-Type: text/markdown
+
+# Samsara RL
+
+<p align="center">
+  <img src="img/samsara-rl.png" alt="Samsara RL" width="200">
+</p>
+
+[![CI](https://github.com/swordofzeus/samsara-rl/actions/workflows/main.yml/badge.svg)](https://github.com/swordofzeus/samsara-rl/actions/workflows/main.yml)
+
+A vectorized NumPy implementation of foundational reinforcement learning algorithms, following David Silver's RL lecture series, Sutton and Barto book and other papers cited when referenced. Built for clarity, learning as well as speed.
+
+Applications of RL include robotic manipulation, LLM fine-tuning, financial portfolio management, and control systems.
+
+---
+
+## Table of Contents
+
+1. [Installation](#installation)
+2. [Quick Start](#quick-start)
+3. [Planning](#planning)
+   - [MDP Structure](#mdp-structure)
+   - [Policy Iteration](#policy-iteration)
+   - [Value Iteration](#value-iteration)
+4. [Model-Free Prediction](#model-free-prediction)
+   - [Monte Carlo](#monte-carlo)
+   - [TD Learning](#td-learning)
+5. [Model-Free Control](#model-free-control)
+   - [SARSA](#sarsa)
+   - [Q-Learning](#q-learning)
+6. [Function Approximation](#function-approximation)
+
+---
+
+## Installation
+
+```bash
+pip install samsara-rl
+```
+
+---
+
+## Quick Start
+
+```python
+from samsara_rl.mdp.grid_world.grid_world_mdp import GridWorldMDP
+from samsara_rl.planning.policy_iteration import PolicyIteration
+
+mdp = GridWorldMDP()
+pi = PolicyIteration(mdp)
+policy = pi.find_optimal_policy()
+```
+
+---
+
+## Planning
+
+Planning algorithms assume full knowledge of environment dynamics (transition probabilities and reward function). While not "true RL" — agents never have access to dynamics in practice — planning provides the theoretical foundation all RL algorithms build on.
+
+### MDP Structure
+
+MDPs are represented as NumPy arrays. The included `GridWorldMDP` implements the 4x4 grid world from David Silver's Lecture 3.
+
+| Attribute                        | Shape        | Description                                      |
+|----------------------------------|--------------|--------------------------------------------------|
+| `state_action_transition_matrix` | `(S, A, S')` | T(s, a, s') — transition probabilities           |
+| `reward_matrix`                  | `(S, A, S')` | R(s, a, s') — reward for each transition         |
+
+### Policy Iteration
+
+Alternates between evaluating the current policy using the Bellman expectation equation and improving it greedily until the policy stops changing.
+
+**`PolicyIteration(mdp, bellman_tolerance)`**
+
+| Argument            | Type    | Default | Description                                 |
+|---------------------|---------|---------|---------------------------------------------|
+| `mdp`               | MDP     |         | MDP instance to solve                       |
+| `bellman_tolerance` | `float` | `0.01`  | Convergence threshold for policy evaluation |
+
+**`find_optimal_policy(max_iter)`**
+
+| Argument   | Type  | Default | Description                              |
+|------------|-------|---------|------------------------------------------|
+| `max_iter` | `int` | `99`    | Maximum number of policy iteration steps |
+
+### Value Iteration
+
+Applies the Bellman optimality equation directly each iteration. Equivalent to policy iteration with k=1 evaluation steps per improvement. Policy is extracted once at convergence.
+
+**`ValueIteration(mdp, bellman_tolerance)`**
+
+| Argument            | Type    | Default | Description                               |
+|---------------------|---------|---------|-------------------------------------------|
+| `mdp`               | MDP     |         | MDP instance to solve                     |
+| `bellman_tolerance` | `float` | `0.01`  | Convergence threshold for value iteration |
+
+**Examples**
+
+```python
+from samsara_rl.mdp.grid_world.grid_world_mdp import GridWorldMDP
+from samsara_rl.planning.policy_iteration import PolicyIteration
+from samsara_rl.planning.value_iteration import ValueIteration
+
+mdp = GridWorldMDP()
+
+policy = PolicyIteration(mdp, bellman_tolerance=0.001).find_optimal_policy(max_iter=50)
+policy = ValueIteration(mdp, bellman_tolerance=0.001).find_optimal_policy()
+```
+
+---
+
+## Model-Free Prediction
+
+Model-free methods learn value functions directly from experience (sampled episodes) without access to the MDP's transition or reward dynamics.
+
+### Monte Carlo
+
+Every-visit Monte Carlo prediction estimates Q(s, a) from sampled returns. After each episode, the return G_t (discounted cumulative reward from time step t onward) is computed for every visited state-action pair, and the Q-table is updated using constant-alpha learning:
+
+Q(s, a) ← Q(s, a) + α (G_t − Q(s, a))
+
+If the same (s, a) pair appears multiple times in an episode, each occurrence triggers an update. Returns are computed in a fully vectorized pass using a cumulative-sum trick that avoids the standard reverse loop over time steps.
+
+**`MonteCarloPrediction(mdp, policy, alpha, gamma)`**
+
+| Argument | Type    | Default | Description                              |
+|----------|---------|---------|------------------------------------------|
+| `mdp`    | MDP     |         | MDP instance to sample episodes from     |
+| `policy` | `array` |         | Stochastic policy of shape `(S, A)`      |
+| `alpha`  | `float` | `0.01`  | Learning rate for incremental Q updates  |
+| `gamma`  | `float` | `1`     | Discount factor                          |
+
+**`evaluate(max_iter)`**
+
+| Argument   | Type  | Default | Description                          |
+|------------|-------|---------|--------------------------------------|
+| `max_iter` | `int` | `10000` | Number of episodes to sample from    |
+
+### TD(λ)
+
+TD(λ) learns Q(s, a) online using one-step bootstrapping with eligibility traces. After each step, the TD error is computed against the expected Q-value of the next state under the current policy, and all previously visited state-action pairs are updated proportionally to their eligibility:
+
+δ = R + γ E_π[Q(S', ·)] − Q(S, A)
+
+Q(s, a) ← Q(s, a) + α δ e(s, a)
+
+Eligibility traces use the replacing variant — on each visit to (s, a), the trace is set to 1 rather than incremented. All traces decay by γλ at each time step. Traces are reset to zero between episodes.
+
+**`TemporalDifference(mdp, policy, alpha, gamma, _lambda)`**
+
+| Argument  | Type    | Default | Description                                    |
+|-----------|---------|---------|------------------------------------------------|
+| `mdp`     | MDP     |         | MDP instance to sample episodes from           |
+| `policy`  | `array` |         | Stochastic policy of shape `(S, A)`            |
+| `alpha`   | `float` | `0.01`  | Learning rate for incremental Q updates        |
+| `gamma`   | `float` | `0.9`   | Discount factor                                |
+| `_lambda` | `float` | `0.4`   | Trace decay parameter (0 = TD(0), 1 = TD(1))  |
+
+**`evaluate(max_iter)`**
+
+| Argument   | Type  | Default | Description                          |
+|------------|-------|---------|--------------------------------------|
+| `max_iter` | `int` | `1000`  | Number of episodes to sample from    |
+
+**Examples**
+
+```python
+from samsara_rl.mdp.grid_world.grid_world_mdp import GridWorldMDP
+from samsara_rl.prediction.monte_carlo import MonteCarloPrediction
+from samsara_rl.prediction.td import TemporalDifference
+from samsara_rl.utils.policy.policy_utils import init_uniform_random
+
+mdp = GridWorldMDP()
+policy = init_uniform_random(mdp)
+
+mc = MonteCarloPrediction(mdp, policy, alpha=0.01, gamma=0.9)
+mc.evaluate(max_iter=10000)
+
+td = TemporalDifference(mdp, policy, alpha=0.01, gamma=0.9, _lambda=0.4)
+td.evaluate(max_iter=10000)
+
+# V(s) for the random policy (expected value over actions)
+v_mc = mc.q_table.mean(axis=1).reshape(4, 4)
+v_td = td.q_table.mean(axis=1).reshape(4, 4)
+```
+
+---
+
+## Model-Free Control
+
+TODO
+
+---
+
+## Function Approximation
+
+TODO

samsara_rl-0.dist-info/RECORD

@@ -0,0 +1,18 @@
+samsara_rl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/mdp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/mdp/grid_world/grid_world_mdp.py,sha256=c9UzPNZaNY451gjWSTios39WZDL7rKXQ1CPxAFXhRkk,4106
+samsara_rl/planning/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/planning/planning.py,sha256=GZ3TDxz0n_hdXtHsJ0-zMM-2ZGRAiQHRaxm57IDWy0w,1557
+samsara_rl/planning/policy_iteration.py,sha256=7vAZq2v9-WHpSTV4mEkfwVdKyY_GnGhR6CaUCdCqDfU,3412
+samsara_rl/planning/value_iteration.py,sha256=qV-L_40DGu0y65YsXbO9QrLB4vSl00RLnjlLvV0mFYI,1938
+samsara_rl/prediction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/prediction/monte_carlo.py,sha256=HUAct1tKEdTyTVjiedAB8ZqWtk7twnJ2hDegksxpvqg,3020
+samsara_rl/prediction/policy_evaluation.py,sha256=y9MMHfqH_bG4j5dIBWp66ldzInc81H85IoI7rHhAJeo,3665
+samsara_rl/prediction/td.py,sha256=HbpIgoJeVabxcAAth1r7WRm-fRYujKxwmQ8b-6ThNHI,1533
+samsara_rl/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/utils/policy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+samsara_rl/utils/policy/policy_utils.py,sha256=9uFu_1tpn08Nnue4aZ7zXzTusjqw_w7NKrTEdz6lu1A,541
+samsara_rl-0.dist-info/METADATA,sha256=goJw0yMZxxq7CPOVDM5sDEE3gF05z0ye8tyzXb5FXSc,8822
+samsara_rl-0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+samsara_rl-0.dist-info/licenses/LICENSE,sha256=WsVkxEenhuQ7iKmUhnb9fyyCESIuJTMFwAibGMxyXL0,1070
+samsara_rl-0.dist-info/RECORD,,

samsara_rl-0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

samsara_rl-0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ashish Juneja
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.