synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

examples/task_apps/pokemon_red/README_IMAGE_ONLY_EVAL.md

@@ -1,415 +0,0 @@
-# Pokemon Red Image-Only Evaluation Guide
-
-This guide shows you how to run Pokemon Red evaluations with **image-only input** (no text observations) and save traces + rewards to **Turso database**.
-
-## Prerequisites
-
-1. **OpenAI API Key**: Set in your `.env` file
-2. **UV Package Manager**: Already installed if you can run `uv run`
-3. **Pokemon Red ROM**: Place in `synth_ai/environments/examples/red/roms/pokemon_red.gb`
-4. **Synth AI Repository**: Clone and set up per main README
-
-## Quick Start
-
-### 1. Run Image-Only Evaluation (10 Rollouts)
-
-```bash
-cd /Users/joshpurtell/Documents/GitHub/synth-ai
-
-# Set up environment for Turso tracing
-export TASKAPP_TRACING_ENABLED=1
-export TURSO_NATIVE=1
-export SQLD_DB_PATH="traces/v3/pokemon_red_eval.db"
-
-# Run evaluation with image-only input
-uv run synth-ai eval pokemon_red \
-  --config examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml
-```
-
-**Expected output**:
-- 10 rollouts complete
-- Most will stay in Red's bedroom (challenging task!)
-- All traces and rewards saved to `traces/v3/pokemon_red_eval.db`
-
-### 2. Check Results
-
-```bash
-# View database
-ls -lh traces/v3/pokemon_red_eval.db  # Should be ~192KB
-
-# Count sessions
-sqlite3 traces/v3/pokemon_red_eval.db \
-  "SELECT COUNT(*) FROM session_traces;"
-
-# View all rollouts
-sqlite3 -header -column traces/v3/pokemon_red_eval.db \
-  "SELECT 
-    session_id,
-    total_reward, 
-    achievements_count,
-    json_extract(reward_metadata, '\$.final_map') as map,
-    json_extract(reward_metadata, '\$.party_count') as party
-   FROM outcome_rewards 
-   ORDER BY total_reward DESC;"
-```
-
-### 3. Query Statistics
-
-```bash
-# Get summary stats
-sqlite3 traces/v3/pokemon_red_eval.db \
-  "SELECT 
-    'Total rollouts' as metric, COUNT(*) as value FROM outcome_rewards
-   UNION ALL
-   SELECT 
-    'With rewards', COUNT(*) FROM outcome_rewards WHERE total_reward > 0
-   UNION ALL
-   SELECT 
-    'Average reward', ROUND(AVG(total_reward), 2) FROM outcome_rewards;"
-```
-
-## Configuration File
-
-**Location**: `examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml`
-
-```toml
-[eval]
-app_id = "pokemon_red"
-model = "gpt-4o-mini-2024-07-18"
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]  # 10 rollouts
-max_turns = 10
-concurrency = 1
-env_name = "pokemon_red"
-policy_name = "pokemon_red_policy"
-trace_format = "full"
-return_trace = true
-
-[eval.env_config]
-max_steps_per_episode = 10
-
-[eval.policy_config]
-provider = "openai"
-model = "gpt-4o-mini-2024-07-18"
-inference_url = "https://api.openai.com"
-temperature = 0.7
-top_p = 0.95
-max_tokens = 512
-use_vision = true           # Enable vision mode
-image_only_mode = true      # Send ONLY images (no text)
-max_llm_calls = 10
-```
-
-### Key Configuration Options
-
-| Option | Description | Values |
-|--------|-------------|--------|
-| `use_vision` | Enable vision/image input | `true` / `false` |
-| `image_only_mode` | Send only images (no text) | `true` / `false` |
-| `seeds` | Which seeds to run | Array of integers |
-| `max_turns` | Max policy calls per rollout | Integer (10-100) |
-| `concurrency` | Parallel rollouts | 1-3 recommended |
-
-## Customization
-
-### Run More Steps (Recommended for Pokemon Red)
-
-Pokemon Red needs more steps to make progress:
-
-```toml
-[eval.env_config]
-env_params = {max_steps_per_episode = 500}  # Full Pallet Town sequence
-
-[eval.policy_config]
-max_llm_calls = 100  # Allow more LLM decisions
-```
-
-### Enable Text + Images (Recommended)
-
-Image-only is very challenging for Pokemon Red. Try multimodal:
-
-```toml
-[eval.policy_config]
-use_vision = true
-image_only_mode = false  # Send both text AND images
-```
-
-This gives the model both:
-- Base64-encoded PNG frames (160x144 Game Boy screen)
-- Text state (HP, position, party, inventory, etc.)
-
-### Use Better Model
-
-```toml
-[eval]
-model = "gpt-4o-2024-08-06"  # Full GPT-4o
-
-[eval.policy_config]
-model = "gpt-4o-2024-08-06"
-temperature = 0.7  # Slightly higher for exploration
-```
-
-### Run More Episodes
-
-```toml
-[eval]
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]  # 20 rollouts
-```
-
-## Database Schema
-
-### outcome_rewards Table
-
-```sql
-CREATE TABLE outcome_rewards (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    session_id VARCHAR NOT NULL,
-    total_reward INTEGER NOT NULL,
-    achievements_count INTEGER NOT NULL,  -- Milestone events
-    total_steps INTEGER NOT NULL,
-    created_at DATETIME NOT NULL,
-    reward_metadata TEXT,  -- JSON with map_id, party_count, badges, etc.
-    FOREIGN KEY(session_id) REFERENCES session_traces(session_id)
-);
-```
-
-### Example Queries
-
-```sql
--- Get statistics
-SELECT 
-    COUNT(*) as total,
-    SUM(CASE WHEN total_reward > 0 THEN 1 ELSE 0 END) as with_rewards,
-    AVG(total_reward) as avg_reward,
-    MAX(total_reward) as max_reward,
-    MAX(achievements_count) as max_achievements
-FROM outcome_rewards;
-
--- Find rollouts that made progress
-SELECT 
-    session_id,
-    total_reward,
-    achievements_count,
-    json_extract(reward_metadata, '$.final_map') as final_map,
-    json_extract(reward_metadata, '$.party_count') as party_count,
-    json_extract(reward_metadata, '$.badges') as badges
-FROM outcome_rewards 
-WHERE total_reward > 0 OR achievements_count > 0
-ORDER BY total_reward DESC;
-
--- Join with session traces
-SELECT 
-    st.session_id,
-    st.created_at,
-    st.num_timesteps,
-    orw.total_reward,
-    orw.achievements_count,
-    json_extract(orw.reward_metadata, '$.milestone_events') as milestones
-FROM session_traces st
-INNER JOIN outcome_rewards orw ON st.session_id = orw.session_id
-WHERE orw.total_reward > 0
-ORDER BY orw.total_reward DESC;
-
--- Check which maps were reached
-SELECT 
-    json_extract(reward_metadata, '$.final_map') as map_id,
-    COUNT(*) as count
-FROM outcome_rewards
-GROUP BY map_id
-ORDER BY count DESC;
-```
-
-## Understanding Maps
-
-**Common Map IDs**:
-- `38`: Red's bedroom (starting location)
-- `0`: Pallet Town (outside)
-- `40`: Red's house downstairs
-- `37`: Oak's Lab
-
-**Goal**: Progress from Map 38 → 40 → 0 → 37 (get starter Pokemon)
-
-## Pallet Town Milestones
-
-The `PalletTownProgressionCompositeReward` tracks these milestones:
-
-| Milestone | Reward | Description |
-|-----------|--------|-------------|
-| Leave bedroom | +20 | Go downstairs |
-| Exit house | +30 | Enter Pallet Town |
-| Find Oak's lab | +40 | Discover and enter lab |
-| Talk to Oak | +50 | First dialogue |
-| Get starter | +100 | Receive your first Pokémon |
-| Enter first battle | +75 | Battle rival |
-| Win battle | +150 | Defeat rival |
-
-**Total possible**: ~600+ points
-
-## Typical Results
-
-**Expected Performance** (10 rollouts, 10 steps, image-only):
-
-```
-Total rollouts: 10
-Rollouts with rewards: 0 (0%)  ← Expected! Task is hard
-Average reward: 0.0
-Final map: 38 (Red's bedroom)
-```
-
-**Why Zero Rewards?**
-- 10 steps is too few for Pokemon Red
-- Image-only mode is very challenging (no HP/inventory text)
-- Needs navigation + NPC interaction
-
-**To Get Non-Zero Rewards**:
-1. Increase `max_steps_per_episode` to 100-500
-2. Enable multimodal: `image_only_mode = false`
-3. Use full GPT-4o: `model = "gpt-4o-2024-08-06"`
-
-## Troubleshooting
-
-### No Database Created
-
-**Issue**: `traces/v3/pokemon_red_eval.db` doesn't exist or is 0 bytes
-
-**Fix**: Ensure environment variables are set:
-```bash
-export TASKAPP_TRACING_ENABLED=1
-export TURSO_NATIVE=1
-export SQLD_DB_PATH="traces/v3/pokemon_red_eval.db"
-```
-
-### ROM Not Found
-
-**Issue**: `FileNotFoundError: pokemon_red.gb`
-
-**Fix**: Place ROM at:
-```bash
-synth_ai/environments/examples/red/roms/pokemon_red.gb
-```
-
-Or set environment variable:
-```bash
-export POKEMON_RED_ROM_PATH="/path/to/pokemon_red.gb"
-```
-
-### 401 Unauthorized Error
-
-**Issue**: OpenAI API returns 401
-
-**Fix**: Check your `.env` file:
-```bash
-# .env
-OPENAI_API_KEY=sk-proj-...your-key-here...
-```
-
-### All Rewards are Zero
-
-**Issue**: Agents aren't making progress (expected with image-only + 10 steps)
-
-**Solutions**:
-
-1. **Increase steps**:
-```toml
-[eval.env_config]
-env_params = {max_steps_per_episode = 100}
-
-[eval.policy_config]
-max_llm_calls = 100
-```
-
-2. **Enable text observations**:
-```toml
-[eval.policy_config]
-image_only_mode = false  # Send both image AND text
-```
-
-3. **Use better model**:
-```toml
-[eval]
-model = "gpt-4o-2024-08-06"
-```
-
-### PyBoy Not Installed
-
-**Issue**: `ModuleNotFoundError: No module named 'pyboy'`
-
-**Fix**:
-```bash
-uv add pyboy
-```
-
-## Advanced: Export to CSV
-
-```bash
-# Export all rollouts to CSV
-sqlite3 -header -csv traces/v3/pokemon_red_eval.db \
-  "SELECT 
-    session_id,
-    total_reward,
-    achievements_count,
-    json_extract(reward_metadata, '$.final_map') as final_map,
-    json_extract(reward_metadata, '$.party_count') as party_count,
-    json_extract(reward_metadata, '$.badges') as badges,
-    json_extract(reward_metadata, '$.milestone_events') as milestones
-   FROM outcome_rewards 
-   ORDER BY total_reward DESC" \
-  > pokemon_red_rewards.csv
-```
-
-## Files Overview
-
-```
-examples/task_apps/pokemon_red/
-├── eval_image_only_gpt4o.toml          # Config file
-├── EVAL_IMAGE_ONLY_COMPLETE.md         # Implementation details
-├── EVAL_IMAGE_ONLY_STATUS.md           # Status document
-├── README_IMAGE_ONLY_EVAL.md           # This file
-├── task_app.py                         # Main task app
-│   ├── Image-only mode logic
-│   ├── SessionTracer integration
-│   ├── OpenAI API authentication
-│   └── Reward computation
-└── pallet_town_rl_config.toml          # RL training config (reference)
-```
-
-## Recommended Settings for Success
-
-For best chance of non-zero rewards:
-
-```toml
-[eval]
-model = "gpt-4o-2024-08-06"  # Full GPT-4o
-seeds = [0, 1, 2, 3, 4]       # 5 rollouts
-max_turns = 100               # Allow more decisions
-
-[eval.env_config]
-env_params = {max_steps_per_episode = 500}  # Full episode
-
-[eval.policy_config]
-provider = "openai"
-model = "gpt-4o-2024-08-06"
-inference_url = "https://api.openai.com"
-temperature = 0.7
-max_tokens = 512
-use_vision = true             # Enable vision
-image_only_mode = false       # Send text too (multimodal)
-max_llm_calls = 100
-```
-
-## See Also
-
-- `EVAL_IMAGE_ONLY_COMPLETE.md` - Full implementation details
-- `pallet_town_rl_config.toml` - RL training configuration
-- `../crafter/README_IMAGE_ONLY_EVAL.md` - Crafter version
-
-## Summary
-
-1. ✅ Set environment variables for Turso tracing
-2. ✅ Run `uv run synth-ai eval pokemon_red --config ...`
-3. ✅ Check database: `traces/v3/pokemon_red_eval.db`
-4. ✅ Query rewards: `SELECT * FROM outcome_rewards`
-5. ✅ For non-zero rewards: increase steps + use multimodal + better model
-
-Pokemon Red is challenging - don't be discouraged by zero rewards with image-only + 10 steps! 🎮
-
-

examples/task_apps/pokemon_red/__init__.py

@@ -1,3 +0,0 @@
-"""Pokémon Red task app example package."""
-
-

examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml

@@ -1,29 +0,0 @@
-# Evaluation config for Pokemon Red with image-only input
-# This config uses GPT-4o mini with only image data (no text observations)
-
-[eval]
-app_id = "pokemon_red"
-model = "gpt-4o-mini-2024-07-18"
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-max_turns = 10
-concurrency = 1  # Keep low initially to avoid issues
-env_name = "pokemon_red"
-policy_name = "pokemon_red_policy"
-trace_format = "full"
-return_trace = true
-
-[eval.env_config]
-max_steps_per_episode = 10
-
-[eval.policy_config]
-provider = "openai"
-model = "gpt-4o-mini-2024-07-18"
-inference_url = "https://api.openai.com"
-temperature = 0.7
-top_p = 0.95
-max_tokens = 512
-use_vision = true
-image_only_mode = true
-max_llm_calls = 10
-
-

examples/task_apps/pokemon_red/eval_pokemon_red_policy.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluate GPT-5-nano policy on Pokemon Red Pallet Town progression.
-
-Runs 10 parallel rollouts and reports rewards in a table.
-"""
-import asyncio
-import os
-from typing import Any
-
-import httpx
-from dotenv import load_dotenv
-from tabulate import tabulate
-
-
-# Load environment variables
-load_dotenv()
-
-# Configuration
-TASK_APP_URL = "http://127.0.0.1:8913"
-NUM_EPISODES = 10
-MAX_STEPS_PER_EPISODE = 10  # 10 policy calls per episode (each may return 5-10 actions)
-MODEL = "gpt-5-nano"
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-
-
-async def run_single_rollout(
-    client: httpx.AsyncClient,
-    episode_id: int,
-) -> dict[str, Any]:
-    """Run a single rollout with policy-driven actions."""
-    
-    # Build rollout request with policy actions
-    # Send "policy" for each step to trigger LLM inference
-    rollout_request = {
-        "run_id": f"eval_episode_{episode_id:03d}",
-        "env": {"instance_id": f"pallet_town_{episode_id:03d}"},
-        "ops": ["policy"] * MAX_STEPS_PER_EPISODE,  # Let policy drive all actions
-        "policy": {
-            "type": "llm",
-            "model": MODEL,
-            "config": {
-                "model": MODEL,
-                "temperature": 0.7,
-                "max_tokens": 500,
-            }
-        },
-    }
-    
-    try:
-        response = await client.post(
-            f"{TASK_APP_URL}/rollout",
-            json=rollout_request,
-            timeout=300.0,  # 5 minutes per rollout
-        )
-        response.raise_for_status()
-        result = response.json()
-        
-        # Extract metrics
-        trajectories = result.get("trajectories", [])
-        if not trajectories:
-            return {
-                "episode_id": episode_id,
-                "status": "error",
-                "error": "No trajectories returned",
-            }
-        
-        trajectory = trajectories[0]
-        steps = trajectory.get("steps", [])
-        num_steps = len(steps) - 1  # Subtract initial observation
-        
-        # Get metrics
-        metrics = result.get("metrics", {})
-        total_reward = metrics.get("episode_returns", [0.0])[0]
-        outcome_score = metrics.get("outcome_score", 0.0)
-        details = metrics.get("details", {})
-        
-        # Extract milestone info
-        reward_components = details.get("reward_components", [])
-        milestone_events = details.get("milestone_events", [])
-        final_map = details.get("final_map", -1)
-        party_count = details.get("party_count", 0)
-        badges = details.get("badges", 0)
-        
-        return {
-            "episode_id": episode_id,
-            "status": "success",
-            "total_reward": total_reward,
-            "outcome_score": outcome_score,
-            "num_steps": num_steps,
-            "final_map": final_map,
-            "party_count": party_count,
-            "badges": badges,
-            "num_milestones": len(milestone_events),
-            "reward_components": reward_components,
-            "milestone_events": milestone_events,
-        }
-        
-    except httpx.TimeoutException:
-        return {
-            "episode_id": episode_id,
-            "status": "timeout",
-            "error": "Rollout timed out after 5 minutes",
-        }
-    except Exception as e:
-        return {
-            "episode_id": episode_id,
-            "status": "error",
-            "error": str(e),
-        }
-
-
-async def main():
-    print("=" * 80)
-    print("POKÉMON RED - POLICY EVALUATION")
-    print("=" * 80)
-    print()
-    print(f"Task: Pallet Town Progression")
-    print(f"Policy: {MODEL}")
-    print(f"Episodes: {NUM_EPISODES}")
-    print(f"Max steps per episode: {MAX_STEPS_PER_EPISODE}")
-    print(f"Server: {TASK_APP_URL}")
-    print()
-    
-    # Check server health
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.get(f"{TASK_APP_URL}/health", timeout=5.0)
-            response.raise_for_status()
-            print("✓ Server is healthy")
-        except Exception as e:
-            print(f"❌ Server not responding: {e}")
-            print(f"   Start it with: uv run -m synth_ai task-app serve pokemon_red --port 8913")
-            return
-        
-        # Check API key
-        if not OPENAI_API_KEY:
-            print("❌ OPENAI_API_KEY not found in environment")
-            print("   Make sure .env file contains OPENAI_API_KEY")
-            return
-        print(f"✓ API key loaded (sk_env...{OPENAI_API_KEY[-4:]})")
-        print()
-        
-        # Run rollouts in parallel
-        print(f"🎮 Running {NUM_EPISODES} episodes in parallel...")
-        print()
-        
-        tasks = [
-            run_single_rollout(client, episode_id)
-            for episode_id in range(1, NUM_EPISODES + 1)
-        ]
-        
-        results = await asyncio.gather(*tasks)
-        
-        # Separate successful and failed results
-        successful = [r for r in results if r.get("status") == "success"]
-        failed = [r for r in results if r.get("status") != "success"]
-        
-        # Print summary table
-        print()
-        print("=" * 80)
-        print("RESULTS SUMMARY")
-        print("=" * 80)
-        print()
-        
-        if successful:
-            table_data = []
-            for r in successful:
-                table_data.append([
-                    r["episode_id"],
-                    f"{r['total_reward']:.1f}",
-                    r["num_steps"],
-                    f"Map{r['final_map']}",
-                    r["party_count"],
-                    r["badges"],
-                    r["num_milestones"],
-                    f"{r['outcome_score']:.3f}",
-                ])
-            
-            headers = [
-                "Episode",
-                "Reward",
-                "Steps",
-                "Final Map",
-                "Party",
-                "Badges",
-                "Milestones",
-                "Outcome Score",
-            ]
-            
-            print(tabulate(table_data, headers=headers, tablefmt="grid"))
-            print()
-            
-            # Print statistics
-            rewards = [r["total_reward"] for r in successful]
-            steps = [r["num_steps"] for r in successful]
-            outcome_scores = [r["outcome_score"] for r in successful]
-            
-            print("Statistics:")
-            print(f"  Mean reward: {sum(rewards) / len(rewards):.2f}")
-            print(f"  Max reward: {max(rewards):.2f}")
-            print(f"  Min reward: {min(rewards):.2f}")
-            print(f"  Mean steps: {sum(steps) / len(steps):.1f}")
-            print(f"  Mean outcome score: {sum(outcome_scores) / len(outcome_scores):.4f}")
-            print()
-            
-            # Print milestone breakdown for best episode
-            best_episode = max(successful, key=lambda r: r["total_reward"])
-            print(f"Best Episode (#{best_episode['episode_id']}):")
-            print(f"  Total reward: {best_episode['total_reward']:.1f}")
-            print(f"  Steps taken: {best_episode['num_steps']}")
-            print(f"  Milestones achieved:")
-            for milestone in best_episode["milestone_events"]:
-                print(f"    Step {milestone['step']}: {milestone['description']} (+{milestone['reward']:.1f})")
-            print()
-        
-        if failed:
-            print(f"Failed episodes: {len(failed)}")
-            for r in failed:
-                print(f"  Episode {r['episode_id']}: {r.get('error', 'Unknown error')}")
-            print()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-