rl-template 0.1.0__py3-none-any.whl

rl_template/agent.py

@@ -0,0 +1,69 @@
+"""Abstract base agent interface for reinforcement learning.
+
+Provides BaseAgent, an ABC + nn.Module hybrid that enforces a consistent
+policy/value interface for all RL agent implementations.
+"""
+
+import numpy as np
+import torch.nn as nn
+from torch.distributions import Distribution
+from torch import Tensor
+from abc import ABC, abstractmethod
+
+
+class BaseAgent(ABC, nn.Module):
+    """Abstract base class for all RL agents.
+
+    Subclasses must implement forward() and get_distribution(). The
+    get_action() template method composes those two to sample actions,
+    compute log probabilities, and return critic values.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def forward(self, state: np.ndarray) -> tuple[Tensor, Tensor]:
+        """Compute raw policy logits and value estimate.
+
+        Args:
+            state: Environment observation.
+
+        Returns:
+            Tuple of (policy_logits, value) tensors.
+        """
+        pass
+
+    @abstractmethod
+    def get_distribution(self, state: np.ndarray) -> tuple[Distribution, Tensor]:
+        """Build a distribution over actions for the given state.
+
+        Args:
+            state: Environment observation.
+
+        Returns:
+            Tuple of (distribution, value) where distribution is a
+            torch.distributions.Distribution instance.
+        """
+        pass
+
+    def get_action(self, state: np.ndarray, action: int | None = None) -> tuple[Tensor, Tensor, Tensor, Tensor]:
+        """Sample or evaluate an action under the current policy.
+
+        Template method: calls get_distribution(), then samples if no
+        action is provided.
+
+        Args:
+            state: Environment observation.
+            action: Optional pre-selected action. When None, samples from
+                    the policy distribution.
+
+        Returns:
+            Tuple of (action, log_prob, entropy, value).
+        """
+        dist, value = self.get_distribution(state)
+        if action is None:
+            action = dist.sample()
+        log_prob = dist.log_prob(action)
+        dist_entropy = dist.entropy()
+        return (action, log_prob, dist_entropy, value)

rl_template/algorithms/ppo/ppo.py

@@ -0,0 +1,183 @@
+"""Proximal Policy Optimization (PPO) trainer implementation.
+
+Provides PPOTrainer, which computes GAE advantages and runs the clipped
+surrogate loss optimization with linear learning rate decay.
+
+Reference: Schulman et al., "Proximal Policy Optimization Algorithms" (2017)
+"""
+
+import torch
+import torch.nn as nn
+from torch import optim
+import numpy as np
+from ...common import Buffer
+
+
+class PPOTrainer:
+    """PPO trainer handling advantage computation and policy updates.
+
+    Maintains an Adam optimizer and applies linear LR decay over training.
+
+    Args:
+        model: Neural network with a get_action(state, action) method.
+        lr: Initial learning rate.
+        gamma: Discount factor.
+        gae_lambda: GAE lambda (bias-variance tradeoff).
+        clip_eps: PPO clipping parameter.
+        value_coef: Value loss coefficient.
+        ent_coef: Entropy bonus coefficient.
+    """
+
+    def __init__(self,
+                 model: nn.Module,
+                 lr: float = 3e-4,
+                 gamma: float = 0.99,
+                 gae_lambda: float = 0.95,
+                 clip_eps: float = 0.2,
+                 value_coef: float = 0.5,
+                 ent_coef: float = 0.01):
+       
+        self.model = model
+        self.lr = lr
+        self.optimizer = optim.Adam(model.parameters(), lr=self.lr)
+        self.gamma = gamma
+        self.gae_lambda = gae_lambda
+        self.clip_eps = clip_eps
+        self.value_coef = value_coef
+        self.ent_coef = ent_coef
+        self.mse_loss = nn.MSELoss()
+
+    def compute_gae(self,
+                    rewards: np.ndarray,
+                    values: np.ndarray,
+                    last_value: float,
+                    dones: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+        """Compute Generalized Advantage Estimation.
+
+        Works backwards through the trajectory, accumulating TD errors
+        with exponentially decaying weights.
+
+        Args:
+            rewards: Rewards for each timestep, shape (T,).
+            values: Value estimates for each timestep, shape (T,).
+            last_value: Bootstrap value for the state after the last step.
+            dones: Episode termination flags, shape (T,). 1.0 = done.
+
+        Returns:
+            Tuple of (returns, advantages, deltas), each shape (T,).
+        """
+        gae = 0.0
+        # Mask: 0.0 at episode boundaries (no bootstrapping across episodes)
+        mask = 1.0 - dones
+        next_values = np.concatenate((values[1:], [last_value]), axis=0)
+        total_size = rewards.shape[0]
+        advantages = np.zeros_like(rewards)
+
+        delta = rewards + self.gamma * next_values * mask - values
+
+        for step in reversed(range(total_size)):
+            gae = delta[step] + self.gamma * self.gae_lambda * mask[step] * gae
+            advantages[step] = gae
+
+        returns = advantages + values
+        return (returns, advantages, delta)
+
+    def lr_decay(self, lr: float, total_steps: int, step: int):
+        """Apply linear learning rate decay from lr to 0.
+
+        Args:
+            lr: Base learning rate.
+            total_steps: Total number of training steps.
+            step: Current training step.
+        """
+        frac = 1.0 - (step / total_steps)
+        current_lr = lr * frac
+        for param_group in self.optimizer.param_groups:
+            param_group["lr"] = current_lr
+
+    def update(self, memory: Buffer, total_steps: int, step: int,
+               batch_size: int = 64, epochs: int = 10) -> tuple:
+        """Run a PPO update on collected rollout data.
+
+        Normalizes advantages and returns, then runs multiple epochs of
+        minibatch SGD with the clipped surrogate loss.
+
+        Args:
+            memory: Buffer containing rollout data.
+            total_steps: Total training steps (LR decay denominator).
+            step: Current training step (LR decay numerator).
+            batch_size: Minibatch size.
+            epochs: Number of passes over the data.
+
+        Returns:
+            Tuple of (total_loss, policy_loss, value_loss, entropy),
+            each averaged over all minibatch updates.
+        """
+        self.lr_decay(self.lr, total_steps, step)
+
+        states, actions, old_log_probs, returns, adv, _, _, _ = memory.get_all()
+
+        advantages = (adv - adv.mean()) / (adv.std() + 1e-8)
+        returns = (returns - returns.mean()) / (returns.std() + 1e-8)
+
+        dataset_size = actions.size(0)
+        num_batch = dataset_size // batch_size
+
+        size_total = int((dataset_size / batch_size) * epochs)
+        epoch_losses = torch.zeros((size_total), dtype=torch.float32)
+        epoch_pi_losses = torch.zeros((size_total))
+        epoch_v_losses = torch.zeros((size_total))
+        epoch_entropies = torch.zeros((size_total))
+        index_loss = 0
+
+        batch_rollout = torch.arange(0, dataset_size, batch_size)
+
+        for _ in range(epochs):
+            shuffle_index = batch_rollout[torch.randperm(num_batch)]
+
+            for start in shuffle_index:
+                end = start + batch_size
+                idx = torch.arange(start, end)
+                if idx.numel() == 0:
+                    continue  # Skip empty batches
+
+                _, new_log_probs, dist_entropy, new_values = self.model.get_action(
+                    states[idx],
+                    actions[idx]
+                )
+
+                logratio = new_log_probs - old_log_probs[idx]
+                ratio = torch.exp(logratio)
+
+                idx_adv = advantages[idx].flatten()
+                surr1 = ratio * idx_adv
+                surr2 = torch.clamp(ratio, 1.0 - self.clip_eps,
+                                    1.0 + self.clip_eps) * idx_adv
+
+                policy_loss = -torch.min(surr1, surr2).mean()
+
+                value_loss = self.mse_loss(new_values.flatten(), returns[idx].flatten())
+
+                entropy_loss = dist_entropy.mean()
+
+                loss = policy_loss + \
+                        (self.value_coef * value_loss) + \
+                        (self.ent_coef * entropy_loss)
+
+                self.optimizer.zero_grad(set_to_none=True)
+                loss.backward()
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
+                self.optimizer.step()
+
+                epoch_losses[index_loss] = loss.detach()
+                epoch_pi_losses[index_loss] = policy_loss.detach()
+                epoch_v_losses[index_loss] = value_loss.detach()
+                epoch_entropies[index_loss] = entropy_loss.detach()
+                index_loss += 1
+
+        # Return average losses over all actual updates ([:index_loss] excludes
+        # any unused pre-allocated entries)
+        return (epoch_losses[:index_loss].mean().item(),
+                epoch_pi_losses[:index_loss].mean().item(),
+                epoch_v_losses[:index_loss].mean().item(),
+                epoch_entropies[:index_loss].mean().item())

rl_template/common.py

@@ -0,0 +1,119 @@
+"""Pre-allocated numpy rollout buffer for RL training.
+
+Provides Buffer, which stores trajectory data in fixed-size numpy arrays
+and converts them to PyTorch tensors for the PPO update step.
+"""
+
+import torch
+import numpy as np
+
+
+class Buffer:
+    """Pre-allocated rollout buffer for collecting RL trajectory data.
+
+    Stores 8 arrays (states, actions, old_log_probs, returns, advantages,
+    rewards, values, dones) in pre-allocated numpy arrays with a slice
+    pointer for O(1) insertion. After a full rollout, insert_returns()
+    fills in GAE-computed returns and advantages, and get_all() converts
+    everything to PyTorch tensors for the PPO update step.
+
+    Example:
+        buf = Buffer(step=2048, state_shape=(4,), action_shape=())
+        for _ in range(2048):
+            buf.insert(state, action, log_prob, reward, value, done)
+        buf.insert_returns(returns, advantages)
+        tensors = buf.get_all()
+        buf.clear()
+    """
+
+    def __init__(self, step: int, state_shape: tuple, action_shape: tuple = ()):
+        """Initialize pre-allocated arrays.
+
+        Args:
+            step: Maximum number of timesteps (capacity).
+            state_shape: Shape of a single observation (excluding batch dim).
+            action_shape: Shape of a single action (excluding batch dim).
+                          Use () for scalar discrete actions.
+        """
+        self.step = step
+        self.slice: int = 0
+        self.states = np.zeros((step, *state_shape), dtype=np.float32)
+        self.actions = np.zeros((step, *action_shape), dtype=np.float32)
+        self.old_log_probs = np.zeros(self.step, dtype=np.float32)
+        self.returns = np.zeros(self.step, dtype=np.float32)
+        self.adv = np.zeros(self.step, dtype=np.float32)
+        self.rewards = np.zeros(self.step, dtype=np.float32)
+        self.values = np.zeros(self.step, dtype=np.float32)
+        self.dones = np.zeros(self.step, dtype=np.float32)
+
+    @property
+    def size(self) -> int:
+        """Return the current number of stored elements."""
+        return self.slice
+
+    def insert(self,
+               state: np.ndarray,
+               action: int,
+               old_log_prob: float,
+               reward: float,
+               value: float,
+               dones: int):
+        """Store a single timestep of rollout data.
+
+        Args:
+            state: Environment observation.
+            action: Action taken by the agent.
+            old_log_prob: Log probability of the action under the old policy.
+            reward: Reward received from the environment.
+            value: Critic value estimate for this state.
+            dones: 1 if the episode ended, 0 otherwise.
+
+        Raises:
+            ValueError: If the buffer is already at capacity.
+        """
+        if self.slice >= self.step:
+            raise ValueError(f"Buffer is full (size={self.step}). Cannot insert more data.")
+        self.states[self.slice] = state
+        self.actions[self.slice] = action
+        self.old_log_probs[self.slice] = old_log_prob
+        self.rewards[self.slice] = reward
+        self.values[self.slice] = value
+        self.dones[self.slice] = dones
+        self.slice += 1
+
+    def insert_returns(self, returns: np.ndarray, adv: np.ndarray):
+        """Write GAE-computed returns and advantages into the buffer.
+
+        Called after the rollout phase when GAE has been computed.
+
+        Args:
+            returns: Discounted return values for each timestep.
+            adv: Generalized Advantage Estimation values for each timestep.
+        """
+        self.returns[:] = returns
+        self.adv[:] = adv
+
+    def get_all(self) -> tuple:
+        """Convert buffer data to PyTorch tensors for the PPO update.
+
+        Returns:
+            Tuple of 8 tensors: (states, actions, old_log_probs, returns,
+            advantages, rewards, values, dones). States are float32,
+            dones are long (integer), everything else is float32.
+        """
+        return (torch.tensor(self.states, dtype=torch.float32),
+                torch.tensor(self.actions, dtype=torch.float32),
+                torch.tensor(self.old_log_probs, dtype=torch.float32),
+                torch.tensor(self.returns, dtype=torch.float32),
+                torch.tensor(self.adv, dtype=torch.float32),
+                torch.tensor(self.rewards, dtype=torch.float32),
+                torch.tensor(self.values, dtype=torch.float32),
+                torch.tensor(self.dones, dtype=torch.long))
+
+    def clear(self):
+        """Reset the buffer for reuse.
+
+        Resets the slice pointer to 0. Underlying arrays are not zeroed;
+        old data is overwritten by subsequent insert() calls.
+        """
+        self.slice = 0

rl_template/config.py

@@ -0,0 +1,73 @@
+"""Configuration dataclasses for PPO training.
+
+Provides PPOConfig (immutable hyperparameters), TrainConfig (mutable
+training settings with computed fields), and WandbConfig (logging).
+"""
+
+import torch
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class PPOConfig:
+    """Immutable PPO hyperparameters.
+
+    Frozen to prevent accidental modification during training.
+
+    Attributes:
+        lr: Adam optimizer learning rate.
+        gamma: Discount factor for future rewards.
+        gae_lambda: GAE lambda parameter (bias-variance tradeoff).
+        clip_eps: Clipping range for the PPO surrogate loss.
+        ent_coef: Entropy bonus coefficient.
+        value_coef: Value loss coefficient.
+    """
+    lr: float = 3e-5
+    gamma: float = 0.999
+    gae_lambda: float = 0.95
+    clip_eps: float = 0.1
+    ent_coef: float = 0.01
+    value_coef: float = 0.5
+
+
+@dataclass
+class TrainConfig:
+    """Training configuration with computed fields.
+
+    model_path and num_update are derived in __post_init__(); do not
+    pass them to the constructor.
+
+    Attributes:
+        model_name: Model name (used in the saved file path).
+        model_saved_path: Directory for model checkpoints.
+        device: PyTorch device string (auto-detects CUDA).
+        model_path: Computed as "{model_saved_path}/{model_name}.pt".
+        timestamp: Total environment timesteps for training.
+        batch_size: Minibatch size for PPO updates.
+        rollout_steps: Steps collected before each PPO update.
+        num_update: Computed as timestamp // rollout_steps.
+    """
+    model_name: str
+    model_saved_path: str
+    device: str = "cuda:0" if torch.cuda.is_available() else "cpu"
+    model_path: str = field(init=False)
+    timestamp: int = 6_000_000
+    batch_size: int = 64
+    rollout_steps: int = 2048
+    num_update: int = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.model_path = f"{self.model_saved_path}/{self.model_name}.pt"
+        self.num_update = self.timestamp // self.rollout_steps
+
+
+@dataclass
+class WandbConfig:
+    """Weights & Biases logging configuration.
+
+    Attributes:
+        name: Display name for the W&B run.
+        logs: Metric name-to-value pairs logged to W&B.
+    """
+    name: str
+    logs: dict[str, float] = field(default_factory=dict)

rl_template/env.py

@@ -0,0 +1,52 @@
+"""Abstract base environment interface for reinforcement learning.
+
+Provides BaseEnv, a Gymnasium v1 API wrapper that all environment
+implementations must extend.
+"""
+
+import numpy as np
+from gymnasium import spaces
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseEnv(ABC):
+    """Abstract base class for all RL environments.
+
+    Follows the Gymnasium v1 API where step() returns 5 values:
+    (obs, reward, terminated, truncated, info). Subclasses must set
+    observation_space and action_space in their __init__().
+    """
+
+    def __init__(self):
+        self.observation_space: spaces.Space = None
+        self.action_space: spaces.Space = None
+
+    @abstractmethod
+    def reset(self, seed: int | None = None) -> tuple[np.ndarray, dict[str, Any]]:
+        """Reset the environment to its initial state.
+
+        Args:
+            seed: Optional random seed for reproducibility.
+
+        Returns:
+            Tuple of (observation, info).
+        """
+        pass
+
+    @abstractmethod
+    def step(self, action: np.ndarray) -> tuple[np.ndarray, float, bool, bool, dict[str, Any]]:
+        """Execute one step in the environment.
+
+        Args:
+            action: The action to take.
+
+        Returns:
+            5-tuple of (observation, reward, terminated, truncated, info).
+        """
+        pass
+
+    @abstractmethod
+    def close(self):
+        """Release environment resources (render windows, connections, etc.)."""
+        pass

rl_template/errors.py

@@ -0,0 +1,26 @@
+"""Custom exceptions for the rl_template project."""
+
+
+class EmptyBufferError(Exception):
+    """Raised when an update is attempted on an insufficiently filled buffer.
+
+    Attributes:
+        current_size: Number of entries currently in the buffer.
+        require_buffer_size: Minimum entries required for an update.
+    """
+
+    def __init__(self, current_size: int, require_buffer_size: int):
+        self.current_size = current_size
+        self.require_buffer_size = require_buffer_size
+
+        self.message = "Training agent flow is incorrect: the buffer is empty"
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        """Return a detailed error message with context and suggestion."""
+        suggestion = "Call Rollout before update weights"
+        details = f"[Crash Workflow] {self.message}\n"
+        details += f"the current buffer size: {self.current_size}\n"
+        details += f" the minimal size required is: {self.require_buffer_size}\n"
+        details += f"{suggestion}"
+        return details

rl_template/train.py

@@ -0,0 +1,127 @@
+"""Abstract training loop for RL agents.
+
+Provides BaseTrain, which orchestrates the rollout-update cycle:
+collect experience, compute GAE, run PPO, and repeat.
+"""
+
+import os
+import torch
+import numpy as np
+from .env import BaseEnv
+from .agent import BaseAgent
+from .algorithms.ppo.ppo import PPOTrainer
+from .common import Buffer
+from .config import TrainConfig
+from abc import ABC, abstractmethod
+from .errors import EmptyBufferError
+
+
+class BaseTrain(ABC):
+    """Abstract training loop coordinating agent, environment, and PPO.
+
+    Subclasses must implement rollout_phase(), update_weights(), and
+    save_model(). The provided implementations are template methods;
+    subclasses may override or call super() to reuse them.
+    """
+
+    def __init__(self,
+                 agent: BaseAgent,
+                 env: BaseEnv,
+                 buffer: Buffer,
+                 train_config: TrainConfig,
+                 ppo_trainer: PPOTrainer):
+        """Initialize the training loop.
+
+        Args:
+            agent: Neural network policy to train.
+            env: Environment to collect experience from.
+            buffer: Pre-allocated buffer for rollout data.
+            train_config: Training configuration (device, paths, hyperparams).
+            ppo_trainer: PPO trainer handling GAE and weight updates.
+        """
+        super().__init__()
+        self.agent = agent
+        self.env = env
+        self.buffer = buffer
+        self.train_config = train_config
+        self.ppo_trainer = ppo_trainer
+        self.last_value = 0.0
+        self.cumulative_reward = 0.0
+        self.require_buffer_size = 10
+
+
+    def rollout_phase(self, state: np.ndarray):
+        """Collect experience by running the agent in the environment.
+
+        Stores each transition in the buffer and resets on episode end.
+        After the rollout, computes the bootstrap value for GAE.
+
+        Args:
+            state: Initial observation to start the rollout from.
+        """
+        for _ in range(self.train_config.rollout_steps):
+            with torch.inference_mode():
+                action_t, log_prob, _, value = self.agent.get_action(state)
+
+            # Convention: truncate = terminated (episode naturally ended)
+            #             done = truncated (episode cut short by time limit)
+            next_state, reward, truncate, done, _ = self.env.step(action_t)
+            done_casted = 1 if done else 0
+
+            self.buffer.insert(
+                state=state,
+                action=action_t.item(),
+                old_log_prob=log_prob,
+                reward=reward,
+                value=value,
+                dones=done_casted,
+            )
+            self.cumulative_reward += reward
+
+            if done or truncate:
+                state = self.env.reset()
+            else:
+                state = next_state
+
+        with torch.inference_mode():
+            _, _, _, next_value = self.agent.get_action(state)
+        self.last_value = next_value.item()
+
+
+    def update_weights(self, step: int):
+        """Compute GAE and update agent weights via PPO.
+
+        Args:
+            step: Current training step (used for learning rate decay).
+
+        Raises:
+            EmptyBufferError: If the buffer has fewer entries than
+                             require_buffer_size.
+        """
+        if self.buffer.size < self.require_buffer_size:
+            raise EmptyBufferError(self.buffer.size, self.require_buffer_size)
+
+        rewards_list = self.buffer.rewards
+        values_list = self.buffer.values
+        dones_list = self.buffer.dones
+
+        with torch.inference_mode():
+            returns, adv, _ = self.ppo_trainer.compute_gae(rewards_list,
+                                                          values_list,
+                                                          self.last_value,
+                                                          dones_list)
+            self.buffer.insert_returns(returns, adv)
+
+        loss, policy_loss, value_loss, entropy = self.ppo_trainer.update(
+            self.buffer,
+            self.train_config.timestamp,
+            step,
+            self.train_config.batch_size
+        )
+        self.buffer.clear()
+
+
+    def save_model(self):
+        """Save agent weights to the path in train_config.model_path."""
+        os.makedirs(os.path.dirname(self.train_config.model_path), exist_ok=True)
+        torch.save(self.agent.state_dict(), self.train_config.model_path)

rl_template-0.1.0.dist-info/METADATA

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: rl-template
+Version: 0.1.0
+Summary: A type-safe, algorithm-agnostic Reinforcement Learning framework
+Author-email: Mohamed Tine <mohamedtine17@gmail.com>
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: torch>=2.0.0
+Description-Content-Type: text/markdown
+
+# RL Template
+
+A modular reinforcement learning training framework built on PyTorch and Gymnasium, providing abstract base classes and a concrete PPO implementation.
+
+## Features
+
+- **Abstract base classes** — `BaseAgent`, `BaseEnv`, `BaseTrain` with template method pattern for easy extension
+- **PPO implementation** — Clipped surrogate loss, GAE, linear LR decay, entropy bonus, gradient clipping
+- **Pre-allocated buffer** — Zero-allocation rollout storage with bounds checking, supports discrete and continuous actions
+- **Frozen configs** — Immutable `PPOConfig` and computed `TrainConfig` dataclasses
+- **Auto GPU detection** — Automatic CUDA/CPU device selection
+- **107 tests** — Full test suite covering all components
+
+## Project Structure
+
+```
+rl_template/
+  __init__.py
+  agent.py               # BaseAgent — abstract policy/value interface
+  env.py                 # BaseEnv — Gymnasium v1 API wrapper
+  train.py               # BaseTrain — rollout/update training loop
+  common.py              # Buffer — pre-allocated numpy rollout buffer
+  config.py              # PPOConfig, TrainConfig, WandbConfig
+  errors.py              # EmptyBufferError
+  algorithms/
+    ppo/
+      ppo.py             # PPOTrainer — GAE + clipped surrogate loss
+tests/
+  test_agent.py          # BaseAgent tests (12)
+  test_env.py            # BaseEnv tests (11)
+  test_train.py          # BaseTrain tests (6)
+  test_common.py         # Buffer tests (21)
+  test_config.py         # Config tests (13)
+  test_errors.py         # Error tests (7)
+  test_ppo.py            # PPOTrainer tests (15)
+  test_buffer_integration.py  # Buffer integration tests (6)
+  test_continuous_actions.py  # Continuous action tests (16)
+```
+
+## Installation
+
+Requires Python 3.11+.
+
+```bash
+# With uv (recommended)
+uv sync
+
+# With pip
+pip install -r requirements.txt
+```
+
+## Usage
+
+```python
+import numpy as np
+from rl_template.agent import BaseAgent
+from rl_template.env import BaseEnv
+from rl_template.train import BaseTrain
+from rl_template.common import Buffer
+from rl_template.config import TrainConfig, PPOConfig
+from rl_template.algorithms.ppo.ppo import PPOTrainer
+
+
+# 1. Implement your agent
+class MyAgent(BaseAgent):
+    # Implement forward(), get_distribution()
+    ...
+
+# 2. Implement your environment
+class MyEnv(BaseEnv):
+    # Implement reset(), step(), close()
+    ...
+
+# 3. Configure training
+config = TrainConfig(model_name="my_agent", model_saved_path="./checkpoints")
+ppo_config = PPOConfig(lr=3e-4, gamma=0.99, clip_eps=0.2)
+
+# 4. Wire everything together
+agent = MyAgent()
+env = MyEnv()
+buffer = Buffer(step=config.rollout_steps, state_shape=(4,))
+ppo_trainer = PPOTrainer(agent, lr=ppo_config.lr, gamma=ppo_config.gamma)
+
+# 5. Create trainer and run
+class MyTrain(BaseTrain):
+    def rollout_phase(self, state): super().rollout_phase(state)
+    def update_weights(self, step): super().update_weights(step)
+    def save_model(self): super().save_model()
+
+trainer = MyTrain(agent, env, buffer, config, ppo_trainer)
+state, _ = env.reset()
+
+for step in range(config.num_update):
+    trainer.rollout_phase(state)
+    trainer.update_weights(step)
+    trainer.save_model()
+```
+
+## Key Classes
+
+| Class | Description |
+|-------|-------------|
+| `BaseAgent` | Abstract agent combining ABC and nn.Module. Subclasses implement `forward()` and `get_distribution()`. The `get_action()` template method handles sampling and log-probability computation. |
+| `BaseEnv` | Abstract Gymnasium v1 environment wrapper. Subclasses implement `reset()`, `step()`, and `close()`. |
+| `BaseTrain` | Abstract training loop orchestrating rollout collection, GAE computation, and PPO updates. |
+| `Buffer` | Pre-allocated numpy buffer for rollout data. Supports discrete and continuous actions with automatic dtype handling. |
+| `PPOTrainer` | PPO algorithm implementation with GAE, clipped surrogate loss, value loss, entropy bonus, and linear LR decay. |
+| `PPOConfig` | Frozen (immutable) PPO hyperparameters. |
+| `TrainConfig` | Training configuration with auto-computed `model_path` and `num_update`. |
+
+## PPO Algorithm
+
+The PPO trainer implements:
+
+- **Generalized Advantage Estimation (GAE)** — Blended n-step returns for low-variance advantage estimates
+- **Clipped surrogate loss** — Prevents destructive policy updates
+- **Value function loss** — MSE loss for the critic network
+- **Entropy bonus** — Encourages exploration
+- **Linear LR decay** — Learning rate anneals to zero over training
+- **Gradient clipping** — Max norm of 1.0 for stability
+
+## Testing
+
+```bash
+python -m pytest tests/ -v
+```
+
+All 107 tests cover unit tests for each component, integration tests for buffer workflows, and continuous action space validation.
+
+## License
+
+MIT

rl_template-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+rl_template/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rl_template/agent.py,sha256=D4f7ntcyjRGjziYTxH_r5eerXhqERj-wPXzVVQY9JR0,2166
+rl_template/common.py,sha256=vkSA4Iy8pBWbeWsdrtPUlKQC_DLomCOia0YN4UgOPOY,4706
+rl_template/config.py,sha256=IN_YFy1tEJfSgH4XZY-FwjwqRY5N1RkdYJ4739GyXZs,2308
+rl_template/env.py,sha256=IVCesOGfkYQC42Grrchur3rNWu1ZgagI5N6kP5wIgys,1467
+rl_template/errors.py,sha256=rEV8MTcvSukMTu5NV9wBl4jQI-aKeGz60g_kstCeJgE,1040
+rl_template/train.py,sha256=fEi7wVFOVPD8dRLZItBw2HdZLsielihcj0tcQSGh58Q,4456
+rl_template/algorithms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rl_template/algorithms/ppo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rl_template/algorithms/ppo/ppo.py,sha256=rYQpW-Exhpof1A5bX8TrwHUZqPK5hJy8ZEIdbT1e_mc,6886
+rl_template-0.1.0.dist-info/METADATA,sha256=BI90PwOXxX0BlHDHFSbkH5Wd-hYAokk5iovqMpRzwTU,5234
+rl_template-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+rl_template-0.1.0.dist-info/RECORD,,

rl_template-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any