synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/blog_posts/pokemon_vl/run_eval_extract_images.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Run pokemon_vl eval with gpt-5-nano and extract images from trajectory response.
+
+This script bypasses the trace validation issue by extracting images directly from
+the trajectory steps in the rollout response.
+"""
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+from pathlib import Path
+
+import httpx
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+async def run_eval_and_extract_images(
+    task_app_url: str,
+    output_dir: Path,
+    seed: int = 0,
+    max_turns: int = 10,
+    model: str = "gpt-5-nano",
+):
+    """Run eval and extract images from trajectory."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    async with httpx.AsyncClient(timeout=300.0) as client:
+        # Build rollout request
+        rollout_request = {
+            "run_id": f"gpt5nano_eval_seed_{seed}",
+            "env": {
+                "env_name": "pokemon_red",
+                "seed": seed,
+                "config": {
+                    "split": "train",
+                    "index": seed,
+                    "env_params": {"max_steps_per_episode": 100},
+                },
+            },
+            "policy": {
+                "policy_name": "pokemon_vl_qwen3_vl",
+                "config": {
+                    "model": model,
+                    "provider": "openai",
+                    "inference_url": "https://api.openai.com/v1",
+                    "temperature": 0.7,
+                    "top_p": 0.95,
+                    "max_tokens": 512,
+                    "use_vision": True,
+                    "image_only_mode": False,
+                    "max_llm_calls": max_turns,
+                },
+            },
+            "ops": ["policy"] * max_turns,
+            "mode": "eval",
+            "record": {
+                "return_trace": True,
+                "trace_format": "full",
+            },
+        }
+        
+        print(f"Running eval with gpt-5-nano (seed={seed})...")
+        response = await client.post(f"{task_app_url}/rollout", json=rollout_request)
+        response.raise_for_status()
+        result = response.json()
+        
+        # Extract trajectory
+        trajectories = result.get("trajectories", [])
+        if not trajectories:
+            print("Error: No trajectories in response")
+            return
+        
+        trajectory = trajectories[0]
+        steps = trajectory.get("steps", [])
+        
+        print(f"✓ Received {len(steps)} steps")
+        print(f"Extracting images (filtering intermediate text box frames)...")
+        
+        # First pass: collect all images with their state
+        image_data = []
+        for idx, step in enumerate(steps):
+            obs = step.get("obs", {})
+            img_b64 = obs.get("observation_image_base64")
+            
+            if not img_b64:
+                continue
+            
+            try:
+                img_data = base64.b64decode(img_b64)
+                map_id = obs.get("map_id", "?")
+                player_x = obs.get("player_x", "?")
+                player_y = obs.get("player_y", "?")
+                text_box_active = obs.get("text_box_active", False)
+                
+                image_data.append({
+                    "idx": idx,
+                    "img_data": img_data,
+                    "map_id": map_id,
+                    "player_x": player_x,
+                    "player_y": player_y,
+                    "text_box_active": text_box_active,
+                })
+            except Exception as e:
+                print(f"  Error decoding step {idx}: {e}")
+                continue
+        
+        # Second pass: filter out intermediate text box frames
+        # Keep: text_box_active=False OR the last frame of a text box sequence
+        filtered_images = []
+        for i, img_info in enumerate(image_data):
+            text_box_active = img_info["text_box_active"]
+            prev_text_box_active = image_data[i - 1]["text_box_active"] if i > 0 else False
+            next_text_box_active = image_data[i + 1]["text_box_active"] if i + 1 < len(image_data) else False
+            
+            # Keep if:
+            # 1. Not in a text box (text_box_active=False)
+            # 2. Last frame of text box sequence (text_box_active=True and next is False)
+            # 3. Last frame overall and in text box (no next frame)
+            if not text_box_active:
+                # Always keep non-text-box frames
+                filtered_images.append(img_info)
+            elif text_box_active and (not next_text_box_active or i + 1 >= len(image_data)):
+                # Keep final frame of text box sequence (transition out or end of trajectory)
+                filtered_images.append(img_info)
+            # Otherwise skip intermediate text box loading frames
+        
+        # Save filtered images
+        image_count = 0
+        for img_info in filtered_images:
+            try:
+                map_id = img_info["map_id"]
+                player_x = img_info["player_x"]
+                player_y = img_info["player_y"]
+                text_box_active = img_info["text_box_active"]
+                idx = img_info["idx"]
+                
+                pos_str = f"Map{map_id}_{player_x},{player_y}"
+                textbox_str = "True" if text_box_active else "False"
+                filename = f"step_{idx:03d}_pos_{pos_str}_textbox_{textbox_str}.png"
+                
+                filepath = output_dir / filename
+                filepath.write_bytes(img_info["img_data"])
+                
+                print(f"  Saved: {filename}")
+                image_count += 1
+            except Exception as e:
+                print(f"  Error saving step {img_info['idx']}: {e}")
+                continue
+        
+        print(f"\n  Filtered: {len(image_data)} -> {len(filtered_images)} images (removed {len(image_data) - len(filtered_images)} intermediate text box frames)")
+        
+        print(f"\n✓ Extracted {image_count} images to {output_dir}/")
+        
+        # Also save metrics
+        metrics = result.get("metrics", {})
+        if metrics:
+            metrics_file = output_dir / "metrics.json"
+            with open(metrics_file, "w") as f:
+                json.dump(metrics, f, indent=2)
+            print(f"✓ Saved metrics to {metrics_file}")
+
+
+async def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--task-app-url",
+        default="http://127.0.0.1:8914",
+        help="Task app URL",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="examples/blog_posts/pokemon_vl/images_gpt5",
+        help="Output directory for images",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=0,
+        help="Random seed",
+    )
+    parser.add_argument(
+        "--max-turns",
+        type=int,
+        default=10,
+        help="Maximum turns",
+    )
+    parser.add_argument(
+        "--model",
+        default="gpt-5-nano",
+        help="Model name",
+    )
+    args = parser.parse_args()
+    
+    await run_eval_and_extract_images(
+        args.task_app_url,
+        Path(args.output_dir),
+        args.seed,
+        args.max_turns,
+        args.model,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/blog_posts/pokemon_vl/run_qwen_eval_extract_images.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Run pokemon_vl eval with Qwen3-VL and extract images from trajectory response.
+
+This script runs a qwen eval and extracts images directly from the trajectory steps
+in the rollout response, similar to run_eval_extract_images.py but for Qwen models.
+"""
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+from pathlib import Path
+
+import httpx
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+async def run_qwen_eval_and_extract_images(
+    task_app_url: str,
+    output_dir: Path,
+    seed: int = 10,
+    max_turns: int = 10,
+    model: str = "Qwen/Qwen3-VL-30B-A3B-Thinking",
+):
+    """Run qwen eval and extract images from trajectory."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    async with httpx.AsyncClient(timeout=600.0) as client:  # Longer timeout for qwen
+        # Build rollout request matching eval_qwen3_vl.toml config
+        rollout_request = {
+            "run_id": f"qwen_eval_seed_{seed}",
+            "env": {
+                "env_name": "pokemon_red",
+                "seed": seed,
+                "config": {
+                    "split": "train",
+                    "index": seed,
+                    "env_params": {"max_steps_per_episode": 100},
+                },
+            },
+            "policy": {
+                "policy_name": "pokemon_vl_qwen3_vl",
+                "config": {
+                    "model": model,
+                    "provider": "synth",
+                    "inference_url": "https://synth-laboratories-dev--learning-v2-service-fastapi-app.modal.run/chat/completions",
+                    "temperature": 1.0,
+                    "top_p": 0.95,
+                    "max_tokens": 2048,
+                    "use_vision": True,
+                    "image_only_mode": False,
+                    "max_llm_calls": max_turns,
+                    "thinking_mode": "think",
+                    "thinking_budget": 3072,
+                },
+            },
+            "ops": ["policy"] * max_turns,
+            "mode": "eval",
+            "record": {
+                "return_trace": True,
+                "trace_format": "full",
+            },
+        }
+        
+        print(f"Running eval with {model} (seed={seed})...")
+        print(f"This may take a while as Qwen models load...")
+        response = await client.post(f"{task_app_url}/rollout", json=rollout_request)
+        response.raise_for_status()
+        result = response.json()
+        
+        # Extract trajectory
+        trajectories = result.get("trajectories", [])
+        if not trajectories:
+            print("Error: No trajectories in response")
+            return
+        
+        trajectory = trajectories[0]
+        steps = trajectory.get("steps", [])
+        
+        print(f"✓ Received {len(steps)} steps")
+        print(f"Extracting images (filtering intermediate text box frames)...")
+        
+        # First pass: collect all images with their state
+        image_data = []
+        for idx, step in enumerate(steps):
+            obs = step.get("obs", {})
+            img_b64 = obs.get("observation_image_base64")
+            
+            if not img_b64:
+                continue
+            
+            try:
+                img_data = base64.b64decode(img_b64)
+                map_id = obs.get("map_id", "?")
+                player_x = obs.get("player_x", "?")
+                player_y = obs.get("player_y", "?")
+                text_box_active = obs.get("text_box_active", False)
+                
+                image_data.append({
+                    "idx": idx,
+                    "img_data": img_data,
+                    "map_id": map_id,
+                    "player_x": player_x,
+                    "player_y": player_y,
+                    "text_box_active": text_box_active,
+                })
+            except Exception as e:
+                print(f"  Error decoding step {idx}: {e}")
+                continue
+        
+        # Second pass: filter out intermediate text box frames
+        # Keep: text_box_active=False OR the last frame of a text box sequence
+        filtered_images = []
+        for i, img_info in enumerate(image_data):
+            text_box_active = img_info["text_box_active"]
+            prev_text_box_active = image_data[i - 1]["text_box_active"] if i > 0 else False
+            next_text_box_active = image_data[i + 1]["text_box_active"] if i + 1 < len(image_data) else False
+            
+            # Keep if:
+            # 1. Not in a text box (text_box_active=False)
+            # 2. Last frame of text box sequence (text_box_active=True and next is False)
+            # 3. Last frame overall and in text box (no next frame)
+            if not text_box_active:
+                # Always keep non-text-box frames
+                filtered_images.append(img_info)
+            elif text_box_active and (not next_text_box_active or i + 1 >= len(image_data)):
+                # Keep final frame of text box sequence (transition out or end of trajectory)
+                filtered_images.append(img_info)
+            # Otherwise skip intermediate text box loading frames
+        
+        # Save filtered images
+        image_count = 0
+        for img_info in filtered_images:
+            try:
+                map_id = img_info["map_id"]
+                player_x = img_info["player_x"]
+                player_y = img_info["player_y"]
+                text_box_active = img_info["text_box_active"]
+                idx = img_info["idx"]
+                
+                pos_str = f"Map{map_id}_{player_x},{player_y}"
+                textbox_str = "True" if text_box_active else "False"
+                filename = f"step_{idx:03d}_pos_{pos_str}_textbox_{textbox_str}_seed{seed}.png"
+                
+                filepath = output_dir / filename
+                filepath.write_bytes(img_info["img_data"])
+                
+                print(f"  Saved: {filename}")
+                image_count += 1
+            except Exception as e:
+                print(f"  Error saving step {img_info['idx']}: {e}")
+                continue
+        
+        print(f"\n  Filtered: {len(image_data)} -> {len(filtered_images)} images (removed {len(image_data) - len(filtered_images)} intermediate text box frames)")
+        
+        print(f"\n✓ Extracted {image_count} images to {output_dir}/")
+        
+        # Also save metrics
+        metrics = result.get("metrics", {})
+        if metrics:
+            metrics_file = output_dir / "metrics.json"
+            with open(metrics_file, "w") as f:
+                json.dump(metrics, f, indent=2)
+            print(f"✓ Saved metrics to {metrics_file}")
+
+
+async def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--task-app-url",
+        default="http://127.0.0.1:8914",
+        help="Task app URL",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="examples/blog_posts/pokemon_vl/images_qwen",
+        help="Output directory for images",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=10,
+        help="Random seed (default matches eval_qwen3_vl.toml)",
+    )
+    parser.add_argument(
+        "--max-turns",
+        type=int,
+        default=10,
+        help="Maximum turns",
+    )
+    parser.add_argument(
+        "--model",
+        default="Qwen/Qwen3-VL-30B-A3B-Thinking",
+        help="Qwen model name",
+    )
+    args = parser.parse_args()
+    
+    await run_qwen_eval_and_extract_images(
+        args.task_app_url,
+        Path(args.output_dir),
+        args.seed,
+        args.max_turns,
+        args.model,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/blog_posts/pokemon_vl/text_box_analysis.md

@@ -0,0 +1,106 @@
+# Pokemon Red Text Box Issue Analysis
+
+## Problem Summary
+The model is getting stuck in text boxes during evaluation, particularly at the starting position `Map26:(3,6)`.
+
+## Key Findings
+
+### Statistics
+- **42 out of 76 states (55%)** have `text_box_active=True`
+- **Position Map26:(3,6) is stuck 18 times** - this is the starting bedroom position
+- The model does eventually escape text boxes, but it takes many steps (50+ steps)
+
+### Visual Issue: Gray Block
+- **Reported**: There's a weird gray block visible in the captured images
+- **Possible causes**:
+  1. PyBoy screen rendering artifact
+  2. Text box background overlay (normal Game Boy behavior)
+  3. Screen capture timing issue (captured during screen transition)
+  4. RGBA→RGB conversion issue in `environment.py` line 295-296
+  
+**Investigation needed**: Check if gray block appears in:
+- All images vs only text_box_active=True images
+- Specific screen regions (bottom half = text box area?)
+- Consistent across all steps or only certain states
+
+### State Progression
+```
+Step   0: pos=Map26:(3,6)          text_box=True  reward=  0.00 map=38
+Step  10: pos=Map26:(3,6)          text_box=True  reward=  0.02 map=38
+Step  16: pos=Map26:(3,6)          text_box=True  reward=  0.02 map=38
+...
+Step  33: pos=Map26:(4,6)          text_box=True  reward=  0.04 map=38
+Step  43: pos=Map26:(5,7)          text_box=True  reward=  0.10 map=38
+Step  52: pos=Map26:(5,7)          text_box=False reward=  0.10 map=38  ← Finally escaped
+```
+
+### Observations
+
+1. **Text box persists across multiple steps** - Even when the model presses B then A (as instructed), the text box doesn't advance immediately
+2. **Position doesn't change when stuck** - The model is stuck at the same position (3,6) for many steps
+3. **Reward stays low** - The model gets minimal reward (0.02-0.04) while stuck
+4. **Eventually breaks free** - After ~50 steps, the model does escape and starts exploring
+
+## Possible Causes
+
+### 1. Game Environment Issue
+- The text box might require a specific button sequence that the model isn't using
+- There might be a timing issue - the model needs to wait longer between button presses
+- The text box might be part of a multi-screen dialogue that requires multiple A presses
+
+### 2. Model Behavior Issue
+- The model might not be pressing buttons correctly (wrong duration/frames)
+- The model might be pressing B too quickly after A, canceling the action
+- The model might need to see the text box advance before understanding it worked
+
+### 3. Reward Function Issue
+- No reward for advancing text boxes means the model doesn't learn this is progress
+- The model might not realize escaping the text box is beneficial
+
+## Recommendations
+
+### Immediate Fixes
+
+1. **Add explicit reward for text box advancement**
+   - Give small reward (+1-2 points) when `text_box_active` transitions from True to False
+   - This signals to the model that escaping text boxes is progress
+
+2. **Improve system prompt**
+   - Be more explicit: "When text_box_active=True, you MUST press A multiple times (5-10 times) to advance through all dialogue screens"
+   - Add: "Each dialogue screen requires pressing A. Continue pressing A until text_box_active becomes False"
+
+3. **Increase button press duration**
+   - Current: `{"button": "A", "frames": 10}` or `{"button": "A", "frames": 30}`
+   - Try: `{"button": "A", "frames": 60}` to ensure the press registers
+
+4. **Add loop detection**
+   - If stuck at same position with text_box_active=True for 3+ turns, force a sequence of 10 A presses
+
+### Longer-term Solutions
+
+1. **Investigate game emulator behavior**
+   - Check if the Pokemon Red emulator handles button presses correctly
+   - Verify text box advancement logic
+
+2. **Add visual feedback**
+   - Show the model screenshots before/after text box advancement
+   - Help it understand the visual change
+
+3. **Pre-training on text box handling**
+   - Create a simple reward for pressing A when text_box_active=True
+   - Let the model learn this basic skill first
+
+## Current Performance
+
+- **Mean outcome score**: 0.010 (very low)
+- **Official mean**: 0.500 (one seed succeeded, one failed)
+- **Total reward**: 0.42-0.50 (milestones give 20-150 points each)
+- **Steps taken**: 105-115 steps (but most spent stuck in text boxes)
+
+## Next Steps
+
+1. Add reward for text box advancement
+2. Update system prompt to be more explicit about text box handling
+3. Test with longer A button press durations
+4. Consider adding loop detection to break out of stuck states
+