ezppo 0.0.0__tar.gz

ezppo-0.0.0/.github/workflows/python-publish.yml

@@ -0,0 +1,70 @@
+# This workflow will upload a Python Package to PyPI when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          # NOTE: put your own distribution build steps here.
+          python -m pip install build
+          python -m build
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+  pypi-publish:
+    runs-on: ubuntu-latest
+    needs:
+      - release-build
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+
+    # Dedicated environments with protections for publishing are strongly recommended.
+    # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
+    environment:
+      name: pypi
+      # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
+      # url: https://pypi.org/p/YOURPROJECT
+      #
+      # ALTERNATIVE: if your GitHub Release name is the PyPI project version string
+      # ALTERNATIVE: exactly, uncomment the following line instead:
+      url: https://pypi.org/project/ezppo//${{ github.event.release.name }}
+
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+      - name: Publish release distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

ezppo-0.0.0/.gitignore

@@ -0,0 +1,27 @@
+# Virtual environments
+.venv/
+venv/
+env/
+ppoVenv/
+testingVenv/
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Checkpoints
+checkpoints/
+
+# Old files backup
+old/

ezppo-0.0.0/PKG-INFO

@@ -0,0 +1,149 @@
+Metadata-Version: 2.4
+Name: ezppo
+Version: 0.0.0
+Summary: A simple PPO library for Discrete and Continuous environments
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: torch
+Requires-Dist: gymnasium
+Requires-Dist: numpy
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+
+# EzPPO
+
+A simple, unified PPO (Proximal Policy Optimization) library for PyTorch. Supports both **discrete** and **continuous** action spaces.
+
+## Features
+
+- **Unified API**: Same training loop for discrete and continuous
+- **Observation normalization**: Online running statistics
+- **GAE (Generalized Advantage Estimation)**: For stable advantage estimation
+- **Clipped surrogate objective**: Standard PPO clipping
+- **Mini-batch training**: With random shuffling
+- **Gradient clipping**: Prevents exploding gradients
+- **Checkpoint save/load**: Resume training easily
+- **Multi-environment support**: Parallel environments built-in
+- **GPU support**: Automatic device management
+
+## Quick Start
+
+### Discrete Actions (CartPole)
+
+```python
+from ezppo import DiscretePPOAgent
+import gymnasium as gym
+
+agent = DiscretePPOAgent(state_dim=4, action_dim=2)
+env = gym.make("CartPole-v1")
+
+state, _ = env.reset()
+action, log_prob = agent.act(state)  # Stochastic action
+# action = agent.act_deterministic(state)  # Greedy action (for eval)
+```
+
+### Continuous Actions (Pendulum)
+
+```python
+from ezppo import ContinuousPPOAgent
+import gymnasium as gym
+
+agent = ContinuousPPOAgent(state_dim=3, action_dim=1)
+env = gym.make("Pendulum-v1")
+
+state, _ = env.reset()
+action, log_prob, raw_action = agent.act(state)
+# action = agent.act_deterministic(state)[0]  # Mean action (for eval)
+```
+
+## Installation
+
+```bash
+pip install torch gymnasium numpy
+```
+
+## Architecture
+
+```
+ezppo/
+├── __init__.py          # Public API
+├── networks.py          # DiscretePPO, ContinuousPPO, BasePPO
+├── normalizer.py        # RunningNormalizer
+└── agents.py            # DiscretePPOAgent, ContinuousPPOAgent
+
+examples/
+├── discrete_cartpole.py
+└── continuous_pendulum.py
+```
+
+## Training Loop Template
+
+```python
+trajectory = []
+max_trajectory = 2048
+
+for episode in range(num_episodes):
+    state, _ = env.reset()
+    done = False
+
+    while not done:
+        action, log_prob = agent.act(state)
+        next_state, reward, terminated, truncated, _ = env.step(action.item())
+        done = terminated or truncated
+
+        trajectory.append((state, action.item(), reward, log_prob.item(), 
+                           next_state, terminated))
+        state = next_state
+
+    if len(trajectory) >= max_trajectory:
+        agent.train(num_epochs=4, batch_size=64)
+        trajectory.clear()
+```
+
+## API Reference
+
+### Networks
+
+| Class | Purpose |
+|-------|---------|
+| `BasePPO` | Base class with configurable architecture |
+| `DiscretePPO` | Discrete actions → logits output |
+| `ContinuousPPO` | Continuous actions → mean + learnable log_std |
+
+### Agents
+
+| Class | Purpose |
+|-------|---------|
+| `DiscretePPOAgent` | Discrete action space |
+| `ContinuousPPOAgent` | Continuous action space |
+
+### Key Methods
+
+**Agent.act(state)** → `(action, log_prob)` or `(action, log_prob, raw_action)`
+**Agent.act_deterministic(state)** → Deterministic action (for evaluation)
+**Agent.train(num_epochs=4, batch_size=64)** → Train on trajectory
+**Agent.store(packet)** → Add transition to trajectory
+**Agent.clear()** → Clear trajectory buffer
+**Agent.save(path)** → Save checkpoint
+**Agent.load(path)** → Load checkpoint
+
+## Configuration
+
+```python
+agent = DiscretePPOAgent(
+    state_dim=4, action_dim=2,
+    architecture={                    # Custom network architecture
+        "actor": [state_dim, 64, 64, action_dim],
+        "critic": [state_dim, 64, 64, 1]
+    },
+    lr=3e-4, gamma=0.99, 
+    gae_lambda=0.95, epsilon=0.2,
+    critic_coef=0.5, entropy_coef=0.01,
+    num_envs=1, trajectory_size=2048,
+    device="cpu"
+)
+```
+
+## License
+
+MIT

ezppo-0.0.0/README.md

@@ -0,0 +1,137 @@
+# EzPPO
+
+A simple, unified PPO (Proximal Policy Optimization) library for PyTorch. Supports both **discrete** and **continuous** action spaces.
+
+## Features
+
+- **Unified API**: Same training loop for discrete and continuous
+- **Observation normalization**: Online running statistics
+- **GAE (Generalized Advantage Estimation)**: For stable advantage estimation
+- **Clipped surrogate objective**: Standard PPO clipping
+- **Mini-batch training**: With random shuffling
+- **Gradient clipping**: Prevents exploding gradients
+- **Checkpoint save/load**: Resume training easily
+- **Multi-environment support**: Parallel environments built-in
+- **GPU support**: Automatic device management
+
+## Quick Start
+
+### Discrete Actions (CartPole)
+
+```python
+from ezppo import DiscretePPOAgent
+import gymnasium as gym
+
+agent = DiscretePPOAgent(state_dim=4, action_dim=2)
+env = gym.make("CartPole-v1")
+
+state, _ = env.reset()
+action, log_prob = agent.act(state)  # Stochastic action
+# action = agent.act_deterministic(state)  # Greedy action (for eval)
+```
+
+### Continuous Actions (Pendulum)
+
+```python
+from ezppo import ContinuousPPOAgent
+import gymnasium as gym
+
+agent = ContinuousPPOAgent(state_dim=3, action_dim=1)
+env = gym.make("Pendulum-v1")
+
+state, _ = env.reset()
+action, log_prob, raw_action = agent.act(state)
+# action = agent.act_deterministic(state)[0]  # Mean action (for eval)
+```
+
+## Installation
+
+```bash
+pip install torch gymnasium numpy
+```
+
+## Architecture
+
+```
+ezppo/
+├── __init__.py          # Public API
+├── networks.py          # DiscretePPO, ContinuousPPO, BasePPO
+├── normalizer.py        # RunningNormalizer
+└── agents.py            # DiscretePPOAgent, ContinuousPPOAgent
+
+examples/
+├── discrete_cartpole.py
+└── continuous_pendulum.py
+```
+
+## Training Loop Template
+
+```python
+trajectory = []
+max_trajectory = 2048
+
+for episode in range(num_episodes):
+    state, _ = env.reset()
+    done = False
+
+    while not done:
+        action, log_prob = agent.act(state)
+        next_state, reward, terminated, truncated, _ = env.step(action.item())
+        done = terminated or truncated
+
+        trajectory.append((state, action.item(), reward, log_prob.item(), 
+                           next_state, terminated))
+        state = next_state
+
+    if len(trajectory) >= max_trajectory:
+        agent.train(num_epochs=4, batch_size=64)
+        trajectory.clear()
+```
+
+## API Reference
+
+### Networks
+
+| Class | Purpose |
+|-------|---------|
+| `BasePPO` | Base class with configurable architecture |
+| `DiscretePPO` | Discrete actions → logits output |
+| `ContinuousPPO` | Continuous actions → mean + learnable log_std |
+
+### Agents
+
+| Class | Purpose |
+|-------|---------|
+| `DiscretePPOAgent` | Discrete action space |
+| `ContinuousPPOAgent` | Continuous action space |
+
+### Key Methods
+
+**Agent.act(state)** → `(action, log_prob)` or `(action, log_prob, raw_action)`
+**Agent.act_deterministic(state)** → Deterministic action (for evaluation)
+**Agent.train(num_epochs=4, batch_size=64)** → Train on trajectory
+**Agent.store(packet)** → Add transition to trajectory
+**Agent.clear()** → Clear trajectory buffer
+**Agent.save(path)** → Save checkpoint
+**Agent.load(path)** → Load checkpoint
+
+## Configuration
+
+```python
+agent = DiscretePPOAgent(
+    state_dim=4, action_dim=2,
+    architecture={                    # Custom network architecture
+        "actor": [state_dim, 64, 64, action_dim],
+        "critic": [state_dim, 64, 64, 1]
+    },
+    lr=3e-4, gamma=0.99, 
+    gae_lambda=0.95, epsilon=0.2,
+    critic_coef=0.5, entropy_coef=0.01,
+    num_envs=1, trajectory_size=2048,
+    device="cpu"
+)
+```
+
+## License
+
+MIT

ezppo-0.0.0/examples/continuous_pendulum.py

@@ -0,0 +1,98 @@
+"""
+Continuous PPO example: Pendulum-v1
+
+Trains a PPO agent to swing up and balance the pendulum.
+"""
+
+import gymnasium as gym
+import numpy as np
+from ezppo import ContinuousPPOAgent
+
+
+def main():
+    # Environment setup
+    env_id = "Pendulum-v1"
+    env = gym.make(env_id, render_mode="rgb_array")
+
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+
+    print(f"Training on {env_id}")
+    print(f"State dim: {state_dim}, Action dim: {action_dim}")
+    print(f"Action range: [{env.action_space.low[0]}, {env.action_space.high[0]}]")
+
+    # Create agent — performant hyperparams, normalization off for stability
+    agent = ContinuousPPOAgent(
+        state_dim=state_dim,
+        action_dim=action_dim,
+        architecture={
+            "actor": [state_dim, 256, 256, action_dim],
+            "critic": [state_dim, 256, 256, 1]
+        },
+        lr=3e-4,
+        gamma=0.99,
+        gae_lambda=0.95,
+        epsilon=0.2,
+        critic_coef=0.5,
+        entropy_coef=0.01,
+        num_envs=1,
+        trajectory_size=512,
+        device="cpu",
+        normalize_observations=False,
+        default_num_epochs=10,
+        default_batch_size=64
+    )
+
+    # Training loop
+    max_episodes = 5000
+    best_reward = -float("inf")
+
+    for episode in range(max_episodes):
+        state, info = env.reset()
+        total_reward = 0
+        done = False
+
+        while not done:
+            action, log_prob, raw_action = agent.act(state)
+            # Scale action to env range: tanh gives [-1,1], scale to action space
+            action_scaled = action * env.action_space.high[0]
+            next_state, reward, terminated, truncated, info = env.step(action_scaled)
+            done = terminated or truncated
+            total_reward += reward
+
+            agent.store((state, raw_action, reward, log_prob, next_state, terminated))
+            state = next_state
+
+        # Train when trajectory is full
+        if len(agent.trajectory) >= agent._trajectory_size:
+            agent.train()
+            agent.clear()
+
+        if total_reward > best_reward:
+            best_reward = total_reward
+
+        if (episode + 1) % 100 == 0:
+            print(f"Episode {episode+1}/{max_episodes} | Reward: {total_reward:.1f} | Best: {best_reward:.1f}")
+
+    # Save checkpoint
+    agent.save("checkpoints/pendulum_ppo", None)
+    print("Checkpoint saved!")
+
+    # Evaluation
+    print("\n--- Evaluation ---")
+    eval_env = gym.make(env_id, render_mode="human")
+    for eval_ep in range(5):
+        state, info = eval_env.reset()
+        total_reward = 0
+        done = False
+        while not done:
+            action = agent.act_deterministic(state)[0]  # Get deterministic action
+            action_scaled = action * eval_env.action_space.high[0]
+            state, reward, terminated, truncated, info = eval_env.step(action_scaled)
+            done = terminated or truncated
+            total_reward += reward
+        print(f"Eval episode: {total_reward}")
+
+
+if __name__ == "__main__":
+    main()

ezppo-0.0.0/examples/discrete_cartpole.py

@@ -0,0 +1,94 @@
+"""
+Discrete PPO example: CartPole-v1
+
+Trains a PPO agent to balance the pole on a cart.
+"""
+
+import gymnasium as gym
+from ezppo import DiscretePPOAgent
+
+
+def main():
+    # Environment setup
+    env_id = "CartPole-v1"
+    env = gym.make(env_id, render_mode="rgb_array")
+
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.n
+
+    print(f"Training on {env_id}")
+    print(f"State dim: {state_dim}, Action dim: {action_dim}")
+
+    # Create agent — performant hyperparams based on old reference impl
+    agent = DiscretePPOAgent(
+        state_dim=state_dim,
+        action_dim=action_dim,
+        architecture={
+            "actor": [state_dim, 256, 256, action_dim],
+            "critic": [state_dim, 256, 256, 1]
+        },
+        lr=3e-4,
+        gamma=0.99,
+        gae_lambda=0.95,
+        epsilon=0.2,
+        critic_coef=0.5,
+        entropy_coef=0.001,
+        num_envs=1,
+        trajectory_size=512,
+        device="cpu",
+        normalize_observations=False,
+        default_num_epochs=10,
+        default_batch_size=64
+    )
+
+    # Training loop
+    max_episodes = 2000
+    best_reward = -float("inf")
+
+    for episode in range(max_episodes):
+        state, info = env.reset()
+        total_reward = 0
+        done = False
+
+        while not done:
+            action, log_prob = agent.act(state)
+            next_state, reward, terminated, truncated, info = env.step(action.item())
+            done = terminated or truncated
+            total_reward += reward
+
+            agent.store((state, action.item(), reward, log_prob.item(), next_state, terminated))
+            state = next_state
+
+        # Train when trajectory is full
+        if len(agent.trajectory) >= agent._trajectory_size:
+            agent.train()
+            agent.clear()
+
+        # Track best
+        if total_reward > best_reward:
+            best_reward = total_reward
+
+        if (episode + 1) % 100 == 0:
+            print(f"Episode {episode+1}/{max_episodes} | Reward: {total_reward:.1f} | Best: {best_reward:.1f}")
+
+    # Save checkpoint
+    agent.save("checkpoints/cartpole_ppo", "checkpoints/cartpole_ppo_norm")
+    print("Checkpoint saved!")
+
+    # Evaluation
+    print("\n--- Evaluation ---")
+    eval_env = gym.make(env_id, render_mode="human")
+    for eval_ep in range(5):
+        state, info = eval_env.reset()
+        total_reward = 0
+        done = False
+        while not done:
+            action = agent.act_deterministic(state)
+            state, reward, terminated, truncated, info = eval_env.step(action)
+            done = terminated or truncated
+            total_reward += reward
+        print(f"Eval episode: {total_reward}")
+
+
+if __name__ == "__main__":
+    main()

ezppo-0.0.0/examples/eval_cartpole.py

@@ -0,0 +1,86 @@
+"""
+Evaluate a trained Discrete PPO checkpoint on CartPole-v1.
+
+Usage:
+    python eval_cartpole.py --checkpoint checkpoints/cartpole_ppo
+"""
+
+import argparse
+import gymnasium as gym
+import numpy as np
+from ezppo import DiscretePPOAgent
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Evaluate a trained PPO checkpoint on CartPole-v1")
+    parser.add_argument("--checkpoint", type=str, default="checkpoints/cartpole_ppo",
+                        help="Path to the saved model checkpoint (without extension)")
+    parser.add_argument("--episodes", type=int, default=10,
+                        help="Number of evaluation episodes")
+    args = parser.parse_args()
+
+    # Environment setup
+    env_id = "CartPole-v1"
+    env = gym.make(env_id, render_mode="human")
+
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.n
+
+    print(f"Evaluating on {env_id}")
+    print(f"State dim: {state_dim}, Action dim: {action_dim}")
+
+    # Create agent — matches the hyperparams used in training
+    agent = DiscretePPOAgent(
+        state_dim=state_dim,
+        action_dim=action_dim,
+        architecture={
+            "actor": [state_dim, 256, 256, action_dim],
+            "critic": [state_dim, 256, 256, 1]
+        },
+        lr=3e-4,
+        gamma=0.99,
+        gae_lambda=0.95,
+        epsilon=0.2,
+        critic_coef=0.5,
+        entropy_coef=0.001,
+        num_envs=1,
+        trajectory_size=512,
+        device="cpu",
+        normalize_observations=False,
+        default_num_epochs=10,
+        default_batch_size=64
+    )
+
+    # Load checkpoint — normalizer path only needed if normalization was enabled during training
+    agent.load(args.checkpoint)
+    print(f"Loaded checkpoint from: {args.checkpoint}\n")
+
+    # Evaluation
+    rewards = []
+    lengths = []
+    print(f"--- Evaluation ({args.episodes} episodes) ---")
+    for ep in range(args.episodes):
+        state, info = env.reset()
+        total_reward = 0
+        done = False
+        steps = 0
+        while not done:
+            action = agent.act_deterministic(state)
+            state, reward, terminated, truncated, info = env.step(action)
+            done = terminated or truncated
+            total_reward += reward
+            steps += 1
+        rewards.append(total_reward)
+        lengths.append(steps)
+        print(f"  Episode {ep+1}/{args.episodes}: reward={total_reward:.1f}, steps={steps}")
+
+    print(f"\nAverage reward: {np.mean(rewards):.1f} +/- {np.std(rewards):.1f}")
+    print(f"Average length: {np.mean(lengths):.1f} +/- {np.std(lengths):.1f}")
+    print(f"Best reward: {np.max(rewards):.1f}")
+    print(f"Worst reward: {np.min(rewards):.1f}")
+    success_rate = np.mean([r >= 500 for r in rewards]) * 100
+    print(f"Success rate (reward >= 500): {success_rate:.0f}%")
+
+
+if __name__ == "__main__":
+    main()