synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/task_apps/pokemon_red/test_pallet_town_rewards.py

@@ -0,0 +1,191 @@
+"""
+Test script for Pallet Town Progression Rewards
+
+This script demonstrates the reward function by simulating
+a sequence of states representing the ideal Pallet Town progression.
+"""
+
+import asyncio
+from synth_ai.environments.examples.red.engine_helpers.reward_library.pallet_town_progression import (
+    PalletTownProgressionCompositeReward,
+)
+
+
+async def main():
+    """Simulate a perfect Pallet Town run and show rewards"""
+    
+    reward_fn = PalletTownProgressionCompositeReward()
+    total_reward = 0.0
+    
+    print("=" * 70)
+    print("PALLET TOWN PROGRESSION - REWARD SIMULATION")
+    print("=" * 70)
+    print()
+    
+    # Step 1: Start in bedroom (Map 1)
+    state1 = {
+        "map_id": 1,
+        "player_x": 3,
+        "player_y": 4,
+        "party_count": 0,
+        "in_battle": False,
+        "text_box_active": False,
+        "battle_outcome": 0,
+        "enemy_hp_current": 0,
+        "enemy_hp_max": 0,
+        "enemy_hp_percentage": 0.0,
+    }
+    action1 = {
+        "prev_map_id": 1,
+        "prev_party_count": 0,
+        "prev_in_battle": False,
+        "prev_text_box_active": False,
+        "prev_enemy_hp_current": 0,
+        "prev_enemy_hp_percentage": 0.0,
+    }
+    
+    # Step 2: Go downstairs (Map 1 -> Map 2)
+    state2 = {**state1, "map_id": 2, "player_y": 8}
+    action2 = {**action1, "prev_map_id": 1}
+    
+    r = await reward_fn.score(state2, action2)
+    total_reward += r
+    print(f"✓ Leave bedroom (Map 1→2):                    +{r:.0f} points")
+    
+    # Step 3: Exit house (Map 2 -> Map 0)
+    state3 = {**state2, "map_id": 0, "player_x": 5, "player_y": 7}
+    action3 = {**action2, "prev_map_id": 2}
+    
+    r = await reward_fn.score(state3, action3)
+    total_reward += r
+    print(f"✓ Exit house to Pallet Town (Map 2→0):        +{r:.0f} points")
+    
+    # Step 4: Navigate to and enter Oak's Lab (Map 0 -> Map 3)
+    state4 = {**state3, "map_id": 3, "player_x": 4, "player_y": 11}
+    action4 = {**action3, "prev_map_id": 0}
+    
+    r = await reward_fn.score(state4, action4)
+    total_reward += r
+    print(f"✓ Find and enter Oak's Lab (Map 0→3):         +{r:.0f} points")
+    
+    # Step 5: Talk to Oak (text box appears)
+    state5 = {**state4, "text_box_active": True}
+    action5 = {**action4, "prev_text_box_active": False}
+    
+    r = await reward_fn.score(state5, action5)
+    total_reward += r
+    print(f"✓ Talk to Professor Oak:                      +{r:.0f} points")
+    
+    # Step 6: Receive starter Pokemon (party count 0 -> 1)
+    state6 = {
+        **state5,
+        "party_count": 1,
+        "party_pokemon": [
+            {
+                "species_id": 4,  # Charmander
+                "level": 5,
+                "hp_current": 20,
+                "hp_max": 20,
+                "hp_percentage": 100.0,
+            }
+        ],
+    }
+    action6 = {**action5, "prev_party_count": 0}
+    
+    r = await reward_fn.score(state6, action6)
+    total_reward += r
+    print(f"✓ Receive starter Pokemon:                    +{r:.0f} points")
+    
+    # Step 7: Enter first battle
+    state7 = {**state6, "in_battle": True, "text_box_active": False,
+              "enemy_hp_current": 20, "enemy_hp_max": 20, "enemy_hp_percentage": 100.0}
+    action7 = {**action6, "prev_in_battle": False, "prev_text_box_active": True}
+    
+    r = await reward_fn.score(state7, action7)
+    total_reward += r
+    print(f"✓ Enter first battle with rival:              +{r:.0f} points")
+    
+    # Step 8-12: Deal damage (5 attacks)
+    print()
+    print("Battle sequence:")
+    for i in range(5):
+        prev_hp = 20 - (i * 4)
+        curr_hp = 20 - ((i + 1) * 4)
+        state_dmg = {
+            **state7,
+            "enemy_hp_current": curr_hp,
+            "enemy_hp_percentage": (curr_hp / 20) * 100,
+        }
+        action_dmg = {
+            **action7,
+            "prev_in_battle": True,
+            "prev_enemy_hp_current": prev_hp,
+            "prev_enemy_hp_percentage": (prev_hp / 20) * 100,
+        }
+        
+        r = await reward_fn.score(state_dmg, action_dmg)
+        total_reward += r
+        
+        # Check for half HP and low HP milestones
+        if r > 5:  # Got bonus reward
+            if (prev_hp / 20) >= 0.5 and (curr_hp / 20) < 0.5:
+                print(f"  → Attack {i+1}: Enemy HP {prev_hp}→{curr_hp} (+5) + Half HP bonus (+25) = +{r:.0f}")
+            elif (prev_hp / 20) >= 0.25 and (curr_hp / 20) < 0.25:
+                print(f"  → Attack {i+1}: Enemy HP {prev_hp}→{curr_hp} (+5) + Low HP bonus (+35) = +{r:.0f}")
+        else:
+            print(f"  → Attack {i+1}: Enemy HP {prev_hp}→{curr_hp}              +{r:.0f} points")
+    
+    print()
+    
+    # Step 13: Win battle
+    state13 = {
+        **state7,
+        "in_battle": False,
+        "battle_outcome": 1,  # Win
+        "enemy_hp_current": 0,
+        "enemy_hp_percentage": 0.0,
+        "battle_turn": 4,
+        "party_pokemon": [
+            {
+                "species_id": 4,
+                "level": 5,
+                "hp_current": 15,  # 75% HP
+                "hp_max": 20,
+                "hp_percentage": 75.0,
+            }
+        ],
+    }
+    action13 = {
+        **action7,
+        "prev_in_battle": True,
+        "prev_enemy_hp_current": 0,
+    }
+    
+    r = await reward_fn.score(state13, action13)
+    total_reward += r
+    print(f"✓ Win first battle:                            +{r:.0f} points")
+    
+    # Step 14: Exit lab with Pokemon (Map 3 -> Map 0)
+    state14 = {**state13, "map_id": 0, "player_x": 5, "player_y": 11}
+    action14 = {**action13, "prev_map_id": 3}
+    
+    r = await reward_fn.score(state14, action14)
+    total_reward += r
+    print(f"✓ Exit Oak's Lab with Pokemon (Map 3→0):      +{r:.0f} points")
+    
+    print()
+    print("=" * 70)
+    print(f"TOTAL REWARD: {total_reward:.0f} points")
+    print("=" * 70)
+    print()
+    print("Breakdown by category:")
+    print("  Navigation:      150 points (bedroom, house, lab, exit)")
+    print("  Story:           150 points (talk to Oak, get Pokemon)")
+    print("  Battle:          335 points (enter, damage, milestones, win)")
+    print("  Efficiency:      ~100 points (battle speed, health, navigation)")
+    print()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/task_apps/sokoban/README.md

@@ -0,0 +1,307 @@
+# Sokoban Task App
+
+A task app for training and evaluating LLM agents on Sokoban puzzles.
+
+Sokoban is a classic puzzle game where the player must push boxes onto target locations. It's a good benchmark for spatial reasoning, planning, and sequential decision-making.
+
+## Features
+
+- 🎮 Multiple difficulty levels (easy, medium, hard)
+- 🤖 LLM policy support (GPT-5-mini, Qwen)
+- 📊 Supports both RL training and evaluation rollouts
+- 🎯 Rich observations with ASCII grid visualization
+- ⚡ Batched actions (up to 8 actions per LLM call)
+
+## Quick Start
+
+### 1. Start the Server
+
+```bash
+cd /path/to/synth-ai
+
+# Start the Sokoban task app on port 8911
+uvx synth-ai task-app serve sokoban --port 8911
+```
+
+The server will be available at `http://localhost:8911`.
+
+### 2. Run a Test Rollout
+
+#### Option A: Using GPT-5-mini
+
+```bash
+export OPENAI_API_KEY="your-api-key"
+
+python3 << 'EOF'
+import httpx
+import asyncio
+
+async def test_gpt5mini():
+    async with httpx.AsyncClient(timeout=600.0) as client:  # Longer timeout
+        print("🎮 Testing with GPT-5-mini (slower due to reasoning tokens)...\n")
+        
+        response = await client.post(
+            "http://localhost:8911/rollout",
+            json={
+                "run_id": "test_gpt5mini",
+                "env": {"seed": 123, "config": {"difficulty": "easy", "max_steps": 100}},
+                "ops": ["policy"] * 5,  # Fewer calls due to slowness
+                "policy": {
+                    "config": {
+                        "provider": "openai",
+                        "model": "gpt-5-mini",
+                        "max_actions_per_call": 8
+                    }
+                }
+            },
+            headers={"Authorization": "Bearer sk_env_your_key_here"}
+        )
+        
+        result = response.json()
+        traj = result["trajectories"][0]
+        final = traj["final"]["observation"]
+        
+        print(f"Boxes: {final['boxes_on_target']}/{final['num_boxes']}")
+        print(f"Steps: {final['steps_taken']}")
+
+asyncio.run(test_gpt5mini())
+EOF
+```
+
+#### Option B: Using Qwen via Groq (Fast & Cheap)
+
+```bash
+export GROQ_API_KEY="your-groq-key"
+
+python3 << 'EOF'
+import httpx
+import asyncio
+
+async def test_qwen():
+    async with httpx.AsyncClient(timeout=300.0) as client:
+        response = await client.post(
+            "http://localhost:8911/rollout",
+            json={
+                "run_id": "test_qwen",
+                "env": {"seed": 123, "config": {"difficulty": "easy", "max_steps": 100}},
+                "ops": ["policy"] * 15,
+                "policy": {
+                    "config": {
+                        "provider": "groq",
+                        "model": "qwen-2.5-7b",
+                        "max_actions_per_call": 8
+                    }
+                }
+            },
+            headers={"Authorization": "Bearer sk_env_your_key_here"}
+        )
+        
+        result = response.json()
+        traj = result["trajectories"][0]
+        final = traj["final"]["observation"]
+        
+        print(f"Result: {'✅ SOLVED!' if final['boxes_on_target'] == final['num_boxes'] else '❌ Not solved'}")
+        print(f"Boxes: {final['boxes_on_target']}/{final['num_boxes']}")
+
+asyncio.run(test_qwen())
+EOF
+```
+
+## Configuration Options
+
+### Environment Config
+
+```python
+{
+    "seed": 123,              # Random seed for puzzle generation
+    "config": {
+        "difficulty": "easy",  # "easy", "medium", or "hard"
+        "max_steps": 100      # Maximum steps before truncation
+    }
+}
+```
+
+### Policy Config
+
+```python
+{
+    "provider": "openai",           # "openai" or "groq"
+    "model": "gpt-5-mini",          # Model name
+    "max_actions_per_call": 8,      # Actions per policy call (1-8)
+    "temperature": 0.7,             # Temperature (optional)
+    "max_completion_tokens": 4000   # Max tokens (optional)
+}
+```
+
+## Model Recommendations
+
+| Model | Status | Speed | Notes |
+|-------|--------|-------|-------|
+| **gpt-5-mini** | ✅ Recommended | Slow (30-50s/call) | Uses 1500-2750 reasoning tokens per call |
+| **gpt-5** | ❌ Not supported | N/A | Doesn't support tool calling |
+| **gpt-5-nano** | ❌ Not supported | N/A | Doesn't support tool calling |
+| **qwen-2.5-7b** (Groq) | ✅ Works | Very fast | Cheap and fast alternative |
+
+### Why is GPT-5-mini slow?
+
+GPT-5-mini uses extensive internal reasoning (1500-2750 reasoning tokens per call) before generating actions. While this could lead to better puzzle-solving, it makes each policy call take 30-50 seconds.
+
+Example usage breakdown:
+```json
+{
+  "usage": {
+    "completion_tokens": 2465,
+    "reasoning_tokens": 2432,  // Deep thinking!
+    "prompt_tokens": 470
+  }
+}
+```
+
+## Observation Format
+
+Each observation includes:
+
+```python
+{
+    "room_text": str,           # ASCII visualization of the puzzle
+    "player_position": [x, y],  # Player coordinates
+    "boxes_on_target": int,     # Number of boxes on target squares
+    "num_boxes": int,           # Total number of boxes
+    "steps_taken": int,         # Steps taken so far
+    "max_steps": int,           # Maximum allowed steps
+    "last_action": str,         # Last action taken
+    "reward_last": float,       # Reward from last step
+    "total_reward": float,      # Cumulative reward
+    "terminated": bool,         # Puzzle solved?
+    "truncated": bool           # Max steps reached?
+}
+```
+
+### ASCII Legend
+
+- `P` = Player
+- `O` = Box
+- `X` = Target square
+- `@` = Box on target
+- `+` = Player on target
+- `#` = Wall
+- `_` = Floor
+
+## Action Space
+
+The agent uses the `interact_many` tool to execute multiple actions in sequence:
+
+```python
+{
+    "tool": "interact_many",
+    "args": {
+        "actions": [0, 1, 2, 3]  # 0=left, 1=up, 2=right, 3=down
+    }
+}
+```
+
+Or with string names:
+```python
+{
+    "actions": ["left", "up", "right", "down"]
+}
+```
+
+## Training with RL
+
+The Sokoban task app supports RL training. Example config:
+
+```toml
+# sokoban_rl_config.toml
+[task_app]
+url = "http://localhost:8911"
+auth_token = "sk_env_your_key_here"
+
+[rl]
+algorithm = "grpo"
+num_episodes = 1000
+batch_size = 32
+
+[policy]
+provider = "groq"
+model = "qwen-2.5-7b"
+max_actions_per_call = 8
+
+[env]
+difficulty = "easy"
+max_steps = 100
+```
+
+Run training:
+```bash
+uvx synth-ai train --config sokoban_rl_config.toml
+```
+
+## Debugging
+
+### Check server health
+```bash
+curl http://localhost:8911/health
+```
+
+### View server logs
+```bash
+# If running with nohup
+tail -f nohup_sokoban.log
+
+# Filter for important logs
+tail -f nohup_sokoban.log | grep -E "extract|debug|error"
+```
+
+### Test with explicit actions
+```python
+# Instead of "policy", provide explicit actions
+"ops": [
+    {"button": "right", "count": 3},
+    {"button": "down", "count": 2}
+]
+```
+
+## Troubleshooting
+
+### Empty responses from LLM
+- **GPT-5/GPT-5-nano**: These models don't support tool calling reliably. Use GPT-5-mini instead.
+- **Timeout errors**: GPT-5-mini is slow. Increase client timeout to 600+ seconds or use fewer policy calls.
+
+### Puzzle not solving
+- Try more policy calls (15-30)
+- Use a different seed
+- Try "easy" difficulty first
+- Check if the agent is stuck in a loop (repeating same actions)
+
+### Server won't start
+```bash
+# Check if port is in use
+lsof -i :8911
+
+# Kill existing process
+kill -9 $(lsof -ti :8911)
+
+# Restart
+uvx synth-ai task-app serve sokoban --port 8911
+```
+
+## Examples
+
+See the `examples/workflows/` directory for:
+- RL training scripts
+- Evaluation scripts
+- Multi-episode parallel evaluation
+
+## Contributing
+
+To add new features:
+1. Edit `task_app.py` for core logic
+2. Update `_base_task_info()` for new observation/action specs
+3. Modify `rollout_executor()` for custom rollout behavior
+4. Add tests in `tests/integration/`
+
+## License
+
+MIT
+

examples/task_apps/sokoban/__init__.py

@@ -0,0 +1,3 @@
+"""Sokoban task app example package."""
+
+

examples/task_apps/sokoban/eval_groq_qwen32.toml

@@ -0,0 +1,16 @@
+# Evaluation config for running Groq Qwen/Qwen3-32B against the Sokoban task app.
+
+provider = "groq"
+task_app_url = "http://127.0.0.1:8911"
+model = "qwen/qwen3-32b"
+seeds = [123]
+max_turns = 60
+concurrency = 1
+
+[policy]
+provider = "groq"
+model = "qwen/qwen3-32b"
+temperature = 0.2
+top_p = 0.95
+max_tokens = 8000
+max_actions_per_call = 4

examples/task_apps/sokoban/eval_openai_gpt5.toml

@@ -0,0 +1,16 @@
+# Evaluation config for running OpenAI GPT-5 against the Sokoban task app.
+
+provider = "openai"
+task_app_url = "http://127.0.0.1:8911"
+model = "gpt-5"
+seeds = [123]
+max_turns = 60
+concurrency = 1
+
+[policy]
+provider = "openai"
+model = "gpt-5"
+temperature = 0.2
+top_p = 0.9
+max_completion_tokens = 4000
+max_actions_per_call = 4