synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

examples/blog_posts/pokemon_vl/run_qwen_eval_extract_images.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Run pokemon_vl eval with Qwen3-VL and extract images from trajectory response.
+
+This script runs a qwen eval and extracts images directly from the trajectory steps
+in the rollout response, similar to run_eval_extract_images.py but for Qwen models.
+"""
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+from pathlib import Path
+
+import httpx
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+async def run_qwen_eval_and_extract_images(
+    task_app_url: str,
+    output_dir: Path,
+    seed: int = 10,
+    max_turns: int = 10,
+    model: str = "Qwen/Qwen3-VL-30B-A3B-Thinking",
+):
+    """Run qwen eval and extract images from trajectory."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    async with httpx.AsyncClient(timeout=600.0) as client:  # Longer timeout for qwen
+        # Build rollout request matching eval_qwen3_vl.toml config
+        rollout_request = {
+            "run_id": f"qwen_eval_seed_{seed}",
+            "env": {
+                "env_name": "pokemon_red",
+                "seed": seed,
+                "config": {
+                    "split": "train",
+                    "index": seed,
+                    "env_params": {"max_steps_per_episode": 100},
+                },
+            },
+            "policy": {
+                "policy_name": "pokemon_vl_qwen3_vl",
+                "config": {
+                    "model": model,
+                    "provider": "synth",
+                    "inference_url": "https://synth-laboratories-dev--learning-v2-service-fastapi-app.modal.run/chat/completions",
+                    "temperature": 1.0,
+                    "top_p": 0.95,
+                    "max_tokens": 2048,
+                    "use_vision": True,
+                    "image_only_mode": False,
+                    "max_llm_calls": max_turns,
+                    "thinking_mode": "think",
+                    "thinking_budget": 3072,
+                },
+            },
+            "ops": ["policy"] * max_turns,
+            "mode": "eval",
+            "record": {
+                "return_trace": True,
+                "trace_format": "full",
+            },
+        }
+        
+        print(f"Running eval with {model} (seed={seed})...")
+        print(f"This may take a while as Qwen models load...")
+        response = await client.post(f"{task_app_url}/rollout", json=rollout_request)
+        response.raise_for_status()
+        result = response.json()
+        
+        # Extract trajectory
+        trajectories = result.get("trajectories", [])
+        if not trajectories:
+            print("Error: No trajectories in response")
+            return
+        
+        trajectory = trajectories[0]
+        steps = trajectory.get("steps", [])
+        
+        print(f"✓ Received {len(steps)} steps")
+        print(f"Extracting images (filtering intermediate text box frames)...")
+        
+        # First pass: collect all images with their state
+        image_data = []
+        for idx, step in enumerate(steps):
+            obs = step.get("obs", {})
+            img_b64 = obs.get("observation_image_base64")
+            
+            if not img_b64:
+                continue
+            
+            try:
+                img_data = base64.b64decode(img_b64)
+                map_id = obs.get("map_id", "?")
+                player_x = obs.get("player_x", "?")
+                player_y = obs.get("player_y", "?")
+                text_box_active = obs.get("text_box_active", False)
+                
+                image_data.append({
+                    "idx": idx,
+                    "img_data": img_data,
+                    "map_id": map_id,
+                    "player_x": player_x,
+                    "player_y": player_y,
+                    "text_box_active": text_box_active,
+                })
+            except Exception as e:
+                print(f"  Error decoding step {idx}: {e}")
+                continue
+        
+        # Second pass: filter out intermediate text box frames
+        # Keep: text_box_active=False OR the last frame of a text box sequence
+        filtered_images = []
+        for i, img_info in enumerate(image_data):
+            text_box_active = img_info["text_box_active"]
+            prev_text_box_active = image_data[i - 1]["text_box_active"] if i > 0 else False
+            next_text_box_active = image_data[i + 1]["text_box_active"] if i + 1 < len(image_data) else False
+            
+            # Keep if:
+            # 1. Not in a text box (text_box_active=False)
+            # 2. Last frame of text box sequence (text_box_active=True and next is False)
+            # 3. Last frame overall and in text box (no next frame)
+            if not text_box_active:
+                # Always keep non-text-box frames
+                filtered_images.append(img_info)
+            elif text_box_active and (not next_text_box_active or i + 1 >= len(image_data)):
+                # Keep final frame of text box sequence (transition out or end of trajectory)
+                filtered_images.append(img_info)
+            # Otherwise skip intermediate text box loading frames
+        
+        # Save filtered images
+        image_count = 0
+        for img_info in filtered_images:
+            try:
+                map_id = img_info["map_id"]
+                player_x = img_info["player_x"]
+                player_y = img_info["player_y"]
+                text_box_active = img_info["text_box_active"]
+                idx = img_info["idx"]
+                
+                pos_str = f"Map{map_id}_{player_x},{player_y}"
+                textbox_str = "True" if text_box_active else "False"
+                filename = f"step_{idx:03d}_pos_{pos_str}_textbox_{textbox_str}_seed{seed}.png"
+                
+                filepath = output_dir / filename
+                filepath.write_bytes(img_info["img_data"])
+                
+                print(f"  Saved: {filename}")
+                image_count += 1
+            except Exception as e:
+                print(f"  Error saving step {img_info['idx']}: {e}")
+                continue
+        
+        print(f"\n  Filtered: {len(image_data)} -> {len(filtered_images)} images (removed {len(image_data) - len(filtered_images)} intermediate text box frames)")
+        
+        print(f"\n✓ Extracted {image_count} images to {output_dir}/")
+        
+        # Also save metrics
+        metrics = result.get("metrics", {})
+        if metrics:
+            metrics_file = output_dir / "metrics.json"
+            with open(metrics_file, "w") as f:
+                json.dump(metrics, f, indent=2)
+            print(f"✓ Saved metrics to {metrics_file}")
+
+
+async def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--task-app-url",
+        default="http://127.0.0.1:8914",
+        help="Task app URL",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="examples/blog_posts/pokemon_vl/images_qwen",
+        help="Output directory for images",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=10,
+        help="Random seed (default matches eval_qwen3_vl.toml)",
+    )
+    parser.add_argument(
+        "--max-turns",
+        type=int,
+        default=10,
+        help="Maximum turns",
+    )
+    parser.add_argument(
+        "--model",
+        default="Qwen/Qwen3-VL-30B-A3B-Thinking",
+        help="Qwen model name",
+    )
+    args = parser.parse_args()
+    
+    await run_qwen_eval_and_extract_images(
+        args.task_app_url,
+        Path(args.output_dir),
+        args.seed,
+        args.max_turns,
+        args.model,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

examples/blog_posts/pokemon_vl/text_box_analysis.md

@@ -0,0 +1,106 @@
+# Pokemon Red Text Box Issue Analysis
+
+## Problem Summary
+The model is getting stuck in text boxes during evaluation, particularly at the starting position `Map26:(3,6)`.
+
+## Key Findings
+
+### Statistics
+- **42 out of 76 states (55%)** have `text_box_active=True`
+- **Position Map26:(3,6) is stuck 18 times** - this is the starting bedroom position
+- The model does eventually escape text boxes, but it takes many steps (50+ steps)
+
+### Visual Issue: Gray Block
+- **Reported**: There's a weird gray block visible in the captured images
+- **Possible causes**:
+  1. PyBoy screen rendering artifact
+  2. Text box background overlay (normal Game Boy behavior)
+  3. Screen capture timing issue (captured during screen transition)
+  4. RGBA→RGB conversion issue in `environment.py` line 295-296
+  
+**Investigation needed**: Check if gray block appears in:
+- All images vs only text_box_active=True images
+- Specific screen regions (bottom half = text box area?)
+- Consistent across all steps or only certain states
+
+### State Progression
+```
+Step   0: pos=Map26:(3,6)          text_box=True  reward=  0.00 map=38
+Step  10: pos=Map26:(3,6)          text_box=True  reward=  0.02 map=38
+Step  16: pos=Map26:(3,6)          text_box=True  reward=  0.02 map=38
+...
+Step  33: pos=Map26:(4,6)          text_box=True  reward=  0.04 map=38
+Step  43: pos=Map26:(5,7)          text_box=True  reward=  0.10 map=38
+Step  52: pos=Map26:(5,7)          text_box=False reward=  0.10 map=38  ← Finally escaped
+```
+
+### Observations
+
+1. **Text box persists across multiple steps** - Even when the model presses B then A (as instructed), the text box doesn't advance immediately
+2. **Position doesn't change when stuck** - The model is stuck at the same position (3,6) for many steps
+3. **Reward stays low** - The model gets minimal reward (0.02-0.04) while stuck
+4. **Eventually breaks free** - After ~50 steps, the model does escape and starts exploring
+
+## Possible Causes
+
+### 1. Game Environment Issue
+- The text box might require a specific button sequence that the model isn't using
+- There might be a timing issue - the model needs to wait longer between button presses
+- The text box might be part of a multi-screen dialogue that requires multiple A presses
+
+### 2. Model Behavior Issue
+- The model might not be pressing buttons correctly (wrong duration/frames)
+- The model might be pressing B too quickly after A, canceling the action
+- The model might need to see the text box advance before understanding it worked
+
+### 3. Reward Function Issue
+- No reward for advancing text boxes means the model doesn't learn this is progress
+- The model might not realize escaping the text box is beneficial
+
+## Recommendations
+
+### Immediate Fixes
+
+1. **Add explicit reward for text box advancement**
+   - Give small reward (+1-2 points) when `text_box_active` transitions from True to False
+   - This signals to the model that escaping text boxes is progress
+
+2. **Improve system prompt**
+   - Be more explicit: "When text_box_active=True, you MUST press A multiple times (5-10 times) to advance through all dialogue screens"
+   - Add: "Each dialogue screen requires pressing A. Continue pressing A until text_box_active becomes False"
+
+3. **Increase button press duration**
+   - Current: `{"button": "A", "frames": 10}` or `{"button": "A", "frames": 30}`
+   - Try: `{"button": "A", "frames": 60}` to ensure the press registers
+
+4. **Add loop detection**
+   - If stuck at same position with text_box_active=True for 3+ turns, force a sequence of 10 A presses
+
+### Longer-term Solutions
+
+1. **Investigate game emulator behavior**
+   - Check if the Pokemon Red emulator handles button presses correctly
+   - Verify text box advancement logic
+
+2. **Add visual feedback**
+   - Show the model screenshots before/after text box advancement
+   - Help it understand the visual change
+
+3. **Pre-training on text box handling**
+   - Create a simple reward for pressing A when text_box_active=True
+   - Let the model learn this basic skill first
+
+## Current Performance
+
+- **Mean outcome score**: 0.010 (very low)
+- **Official mean**: 0.500 (one seed succeeded, one failed)
+- **Total reward**: 0.42-0.50 (milestones give 20-150 points each)
+- **Steps taken**: 105-115 steps (but most spent stuck in text boxes)
+
+## Next Steps
+
+1. Add reward for text box advancement
+2. Update system prompt to be more explicit about text box handling
+3. Test with longer A button press durations
+4. Consider adding loop detection to break out of stuck states
+

examples/blog_posts/warming_up_to_rl/ARCHITECTURE.md

@@ -0,0 +1,195 @@
+# Smoke Test Architecture
+
+This document explains how the smoke test works internally, for future maintenance and debugging.
+
+## Component Overview
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                     synth-ai smoke command                       │
+│  (synth_ai/cli/commands/smoke/core.py)                         │
+└────────────┬────────────────────────────────────────────────────┘
+             │
+             ├─► Auto-start sqld (optional)
+             │   ├─ Kill existing process on ports 8080/8081
+             │   ├─ Start: sqld --db-path ... --hrana-listen-addr ... --http-listen-addr ...
+             │   └─ Health check: GET http://127.0.0.1:8081/health
+             │
+             ├─► Auto-start task app (optional)
+             │   ├─ Kill existing process on port 8765
+             │   ├─ Start: nohup uvx synth-ai task-app serve ... (from synth-ai root)
+             │   ├─ Health check: GET http://localhost:8765/health (accepts 200 or 400)
+             │   └─ Output: nohup_task_app.out
+             │
+             ├─► Start mock RL trainer (if use_mock=true)
+             │   ├─ MockRLTrainer(port=0, backend="openai")
+             │   ├─ Forwards requests to OpenAI API
+             │   └─ Logs: [mock-rl] ← request / → response
+             │
+             └─► Execute rollout
+                 ├─ POST /rollout to task app
+                 ├─ Capture response with v3 trace
+                 └─ Extract and display tool calls
+
+```
+
+## Key Implementation Details
+
+### 1. Tool Call Extraction
+
+**Location:** `synth_ai/cli/commands/smoke/core.py` lines ~946-1005
+
+**How it works:**
+1. Request rollout with `return_trace=True` and `trace_format="structured"`
+2. Response includes `trace.event_history[]` - list of policy and environment events
+3. Policy events have `call_records[]` containing LLM call metadata
+4. Each `call_record` has `output_tool_calls[]` with tool call details
+5. Extract `name` and `arguments_json` from each tool call
+6. Display formatted tool calls to user
+
+**Data structure:**
+```python
+response.trace = {
+    "event_history": [
+        {
+            "call_records": [  # Present in policy events
+                {
+                    "output_tool_calls": [
+                        {
+                            "name": "interact_many",
+                            "arguments_json": '{"actions":["move_up","move_up"]}',
+                            "call_id": "call_xyz",
+                            "index": 0
+                        }
+                    ],
+                    "model_name": "gpt-4o-mini",
+                    "provider": "openai",
+                    ...
+                }
+            ],
+            "metadata": {...},
+            ...
+        },
+        {
+            # Environment step event (no call_records)
+            "reward": 1.0,
+            "terminated": false,
+            ...
+        },
+        ...
+    ],
+    "session_id": "...",
+    "markov_blanket_message_history": [...],
+    ...
+}
+```
+
+### 2. Background Service Management
+
+**Task App Startup:**
+- Must run from synth-ai root for task app discovery
+- Uses `nohup` to detach process
+- Redirects output to `nohup_task_app.out`
+- Polls `/health` endpoint (accepts 200 or 400 status)
+- Timeout: 120 seconds with progress updates every 5 seconds
+- Propagates `SYNTH_QUIET=1` to suppress diagnostic messages
+
+**sqld Startup:**
+- Starts with Hrana WebSocket (8080) and HTTP (8081) ports
+- Polls `/health` endpoint for readiness
+- Timeout: 30 seconds
+
+**Port Cleanup:**
+- Uses `lsof -ti :PORT` to find PIDs
+- Kills processes with `kill -9 PID`
+- Waits 2 seconds for port release
+
+### 3. Mock RL Trainer
+
+The mock trainer (`MockRLTrainer`) acts as a proxy:
+- `backend="synthetic"`: Generates fake tool calls deterministically
+- `backend="openai"`: Forwards to real OpenAI API
+- Logs all requests/responses with `[mock-rl]` prefix
+- Auto-assigns port if `port=0`
+
+### 4. Diagnostic Message Suppression
+
+**Permanently disabled (commented out):**
+- `synth_ai/tracing_v3/config.py`: `[TRACING_V3_CONFIG_LOADED]` message
+- `synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py`: All `[PATCH]` messages
+- `synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py`: All `[PATCH]` messages
+- `synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py`: All `[PATCH]` messages
+
+**Reason:** These messages add noise to smoke test output. They're still in the code as comments for documentation.
+
+## Troubleshooting Guide
+
+### No tool calls displayed
+
+**Symptom:** Output shows `⚠ No tool calls found in trace`
+
+**Causes:**
+1. `return_trace=false` in config - **FIX:** Set `return_trace = true`
+2. Trace format mismatch - Check `response.trace.event_history` structure
+3. No LLM calls made - Check for policy errors in task app logs
+
+**Debug:**
+```bash
+# Check task app logs
+cat /path/to/synth-ai/nohup_task_app.out
+
+# Verify trace structure
+# Add debug output in core.py around line 978:
+click.echo(f"DEBUG: trace keys: {list(tr.keys())}")
+click.echo(f"DEBUG: event_history length: {len(event_history)}")
+```
+
+### Task app exits immediately
+
+**Symptom:** `0 steps` in rollout, task app process not running
+
+**Causes:**
+1. Wrong task app name - **FIX:** Use `synth-ai task-app list` to find correct name
+2. Missing .env file - **FIX:** Ensure `task_app_env_file` points to valid .env
+3. Wrong working directory - **FIX:** Task app must be started from synth-ai root
+
+**Debug:**
+```bash
+# Manual test
+cd /path/to/synth-ai
+uvx synth-ai task-app serve grpo-crafter --port 8765 --env-file /path/to/.env --force
+```
+
+### Port conflicts
+
+**Symptom:** `Address already in use` errors
+
+**Fix:** The smoke command auto-kills processes on ports 8080, 8081, 8765. If manual cleanup needed:
+```bash
+lsof -ti :8080 | xargs kill -9
+lsof -ti :8081 | xargs kill -9
+lsof -ti :8765 | xargs kill -9
+```
+
+## Future Improvements
+
+Potential enhancements for future agents:
+
+1. **Streaming tool call display**: Show tool calls as they happen, not just at the end
+2. **Tool call validation**: Verify tool calls match expected format for the environment
+3. **Performance metrics**: Track inference latency per tool call
+4. **Cost tracking**: Display OpenAI API costs for the smoke test
+5. **Parallel rollouts**: Support `--parallel N` to test concurrent execution
+6. **Video/image capture**: For vision-based tasks, save observations
+7. **Interactive mode**: Allow stepping through rollout one action at a time
+
+## Related Files
+
+- `synth_ai/cli/commands/smoke/core.py` - Main smoke command implementation
+- `synth_ai/api/train/configs/rl.py` - `SmokeConfig` Pydantic model
+- `synth_ai/api/train/builders.py` - Removes `[smoke]` section before sending to trainer
+- `synth_ai/task/contracts.py` - `RolloutResponse` with trace field
+- `examples/blog_posts/warming_up_to_rl/SMOKE_TESTING.md` - User-facing documentation
+- `monorepo/docs/cli/smoke.mdx` - Mintlify documentation
+
+

examples/blog_posts/warming_up_to_rl/FINAL_TEST_RESULTS.md

@@ -0,0 +1,127 @@
+# Final Inference Test Results
+
+**Date**: Oct 31, 2025  
+**Endpoint**: `https://synth-laboratories-dev--learning-v2-service-fastapi-app.modal.run/chat/completions`
+
+## Summary
+
+| Model Type | Status | Result |
+|------------|--------|--------|
+| Base Model (Qwen/Qwen3-4B) | ✅ WORKS | Inference successful |
+| PEFT/SFT (Qwen3-0.6B) | ✅ WORKS | Inference successful |
+| RL (Qwen3-4B) | ❌ **BROKEN** | Modal function crashes |
+
+## Detailed Results
+
+### ✅ Test 1: Base Model (No Fine-Tuning)
+
+**Model**: `Qwen/Qwen3-4B`
+
+**Result**: **SUCCESS** ✅
+- **Status**: 200 OK
+- **Tokens**: 31 prompt + 100 completion = 131 total
+- **Response**: Generated successfully
+
+**Notes**:
+- First attempt returned 303 redirect (cold start)
+- Retry succeeded immediately
+- This confirms the endpoint and auth work correctly
+
+---
+
+### ✅ Test 2: PEFT/SFT Model
+
+**Model**: `peft:Qwen/Qwen3-0.6B:job_24faa0fdfdf648b9`
+
+**Result**: **SUCCESS** ✅
+- **Status**: 200 OK (consistent across retries)
+- **Tokens**: 31 prompt + 100 completion = 131 total  
+- **Response**: "Hello, I am working!" (with thinking tokens)
+
+**Notes**:
+- Works reliably
+- No cold start issues
+- This is the expected behavior for all models
+
+---
+
+### ❌ Test 3: RL Model
+
+**Model**: `rl:Qwen/Qwen3-4B:job_19a38041c38f96e638c:checkpoint-epoch-1`
+
+**Result**: **FAILURE** ❌ - Multiple error modes
+
+#### First Attempt:
+```
+Status: 400 Bad Request
+Error: "Device string must not be empty"
+```
+
+#### Retry:
+```
+Status: 500 Internal Server Error
+Error: "modal-http: internal error: function was terminated by signal"
+```
+
+**This is a Modal function crash** - the inference function terminated unexpectedly.
+
+#### Cold Start (from Modal logs):
+```
+RuntimeError: Cannot find any model weights with 
+'/models/rl/Qwen/Qwen3-4B/job_19a38041c38f96e638c/checkpoint-fixed'
+```
+
+**Root Cause**: RL checkpoint contains LoRA adapter files (`adapter_config.json`, `adapter_model.safetensors`), but vLLM expects full merged model weights.
+
+---
+
+## Conclusion
+
+### What Works ✅
+- **Base models**: Standard HuggingFace models load and inference correctly
+- **PEFT/SFT models**: Fine-tuned models with merged weights work perfectly
+
+### What's Broken ❌
+- **RL models**: Crash during model loading because:
+  1. RL checkpoints are stored as LoRA adapters
+  2. vLLM weight loader expects full model weights
+  3. Missing merge step causes vLLM to crash
+  4. Modal function terminates with signal (crash)
+
+### Impact
+- **HIGH SEVERITY**: All RL-trained models cannot be used for inference
+- Users can train RL models but cannot deploy them
+- This blocks the core RL training → inference workflow
+
+### Next Steps
+See `monorepo/RL_INFERENCE_BUG.md` for:
+- Detailed root cause analysis
+- Reproduction script
+- Suggested fix (merge LoRA adapters before vLLM loading)
+- Code locations to modify
+
+---
+
+## Developer Experience Issues Identified
+
+### Issue #1: Confusing Error Messages
+- **400 "Device string must not be empty"** - Not helpful, doesn't indicate RL adapter issue
+- **500 "function was terminated by signal"** - Generic crash, no context
+- **Should be**: "RL checkpoint contains adapter files. Merge required for vLLM loading."
+
+### Issue #2: Inconsistent Behavior
+- Sometimes returns 303 redirect
+- Sometimes returns 400
+- Sometimes crashes with 500
+- **Should be**: Consistent error message explaining the issue
+
+### Issue #3: Not Obvious How to Test Models
+- Had to try 3 different endpoint URLs before finding the right one
+- No documentation on model ID formats
+- **Should be**: `synth-ai inference --model "rl:..." --message "test"` CLI command
+
+---
+
+**Status**: Bug documented and reproduction available.  
+**See**: `monorepo/RL_INFERENCE_BUG.md` for full details.
+