synth-ai 0.2.9.dev3__py3-none-any.whl → 0.2.9.dev5__py3-none-any.whl

examples/evals_old/compare_models.py

@@ -0,0 +1,1037 @@
+#!/usr/bin/env python3
+# ruff: noqa: E402
+"""
+Comprehensive script to run Crafter rollouts for multiple models and compare their performance.
+Updated to use tracing_v3 with async architecture.
+
+Runs experiments for:
+- gpt-4o-mini
+- gpt-4.1-mini
+- gpt-4.1-nano
+- gemini-1.5-flash
+- gemini-2.5-flash-lite
+- qwen3/32b
+
+Analyzes and compares:
+- Invalid action rates
+- Achievement frequencies by step
+- Achievement counts across models
+- Performance metrics
+- Cost analysis
+"""
+import os
+import sys
+from pathlib import Path
+
+# Ensure repository root is on sys.path before importing synth_ai
+synth_ai_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(synth_ai_root))
+
+# Disable v1 logging to see v3 tracing clearly
+os.environ["LANGFUSE_ENABLED"] = "false"
+os.environ["SYNTH_LOGGING"] = "false"
+
+import argparse
+import asyncio
+import contextlib
+import json
+import logging
+import random
+import time
+from collections import defaultdict
+from datetime import datetime
+from typing import Any
+
+import httpx
+import numpy as np
+import pandas as pd
+
+# Import enhanced LM with v3 tracing
+from synth_ai.lm.core.main_v3 import LM
+from synth_ai.tracing_v3.abstractions import (
+    EnvironmentEvent,
+    RuntimeEvent,
+    SessionEventMarkovBlanketMessage,
+    TimeRecord,
+)
+from synth_ai.tracing_v3.db_config import get_default_db_config
+from synth_ai.tracing_v3.decorators import set_turn_number
+from synth_ai.tracing_v3.session_tracer import SessionTracer
+from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
+from tqdm import tqdm
+
+# Disable httpx logging for cleaner output
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+# Import Crafter hooks
+try:
+    from synth_ai.environments.examples.crafter_classic.trace_hooks_v3 import CRAFTER_HOOKS
+
+    print(f"✅ Loaded {len(CRAFTER_HOOKS.hooks)} Crafter achievement hooks (Easy, Medium, Hard)")
+except ImportError:
+    print("Warning: Could not import CRAFTER_HOOKS for v3")
+    from synth_ai.tracing_v3.hooks import HookManager
+
+    CRAFTER_HOOKS = HookManager()
+
+# Global buckets for sessions
+_SESSIONS: dict[str, tuple[str, object]] = {}  # session_id -> (experiment_id, trace)
+
+# Configuration
+MODELS_TO_TEST = [
+    "gpt-5-nano",
+    "gpt-4.1-nano",
+]
+
+# Service URLs (modify these based on your setup)
+CRAFTER_SERVICE_URL = "http://localhost:8901"
+
+# Database configuration - uses the centralized config which matches serve.sh
+db_config = get_default_db_config()
+DATABASE_URL = db_config.database_url
+
+# Retry configuration for HTTP requests
+MAX_RETRIES = 3
+BASE_DELAY = 0.1
+MAX_DELAY = 2.0
+HTTP_TIMEOUT = 30.0
+
+
+class ExperimentConfig:
+    """Configuration for the multi-model experiment."""
+
+    def __init__(self):
+        self.num_episodes = 10  # Number of episodes per model
+        self.max_turns = 100  # Max turns per episode
+        self.difficulty = "easy"
+        self.save_traces = True
+        self.verbose = True
+        self.quiet = False  # Default to verbose mode
+        self.enable_v3_tracing = True
+        self.v3_trace_dir = "./traces/v3/crafter_comparison"
+        self.crafter_service_url = CRAFTER_SERVICE_URL
+        self.database_url = DATABASE_URL
+        self.base_seed = 1000  # Base seed for episode generation
+        self.turn_timeout = 30.0  # Timeout per turn in seconds
+        self.episode_timeout = 300.0  # Total timeout per episode in seconds
+        self.concurrency = 5  # Max concurrent episodes per model
+
+
+async def retry_http_request(client: httpx.AsyncClient, method: str, url: str, **kwargs) -> Any:
+    """Retry HTTP requests with exponential backoff and jitter."""
+    last_exception = None
+
+    for attempt in range(MAX_RETRIES):
+        try:
+            if attempt > 0:
+                delay = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY)
+                jitter = random.uniform(0, 0.1 * delay)
+                total_delay = delay + jitter
+                await asyncio.sleep(total_delay)
+
+            response = await client.request(method, url, timeout=HTTP_TIMEOUT, **kwargs)
+
+            if response.status_code < 500:
+                return response
+
+            last_exception = Exception(f"HTTP {response.status_code}: {response.text}")
+
+        except httpx.ConnectError as e:
+            last_exception = Exception(f"Connection failed to {url}: {e}")
+            if attempt < MAX_RETRIES - 1:
+                await asyncio.sleep(1.0 * (2**attempt))
+        except httpx.ReadError as e:
+            last_exception = e
+            if attempt < MAX_RETRIES - 1:
+                read_error_delay = min(1.0 * (2**attempt), 5.0)
+                await asyncio.sleep(read_error_delay)
+        except Exception as e:
+            last_exception = e
+
+    print(f"    ❌ HTTP request failed after {MAX_RETRIES} attempts: {method} {url}")
+    print(f"    ❌ Error: {type(last_exception).__name__}: {str(last_exception)[:200]}")
+    raise last_exception
+
+
+# Crafter action mapping
+CRAFTER_ACTIONS = {
+    "noop": 0,
+    "move_left": 1,
+    "move_right": 2,
+    "move_up": 3,
+    "move_down": 4,
+    "do": 5,
+    "sleep": 6,
+    "place_stone": 7,
+    "place_table": 8,
+    "place_furnace": 9,
+    "place_plant": 10,
+    "make_wood_pickaxe": 11,
+    "make_stone_pickaxe": 12,
+    "make_iron_pickaxe": 13,
+    "make_wood_sword": 14,
+    "make_stone_sword": 15,
+    "make_iron_sword": 16,
+    "eat_cow": 17,
+    "eat_plant": 18,
+}
+
+# Create reverse mapping for validation
+INT_TO_ACTION_STRING = {v: k for k, v in CRAFTER_ACTIONS.items()}
+
+
+def compress_observation_for_trace(obs: dict[str, Any]) -> str:
+    """Compress observation data for storage in traces."""
+    try:
+        return json.dumps(
+            {
+                "inv": {k: v for k, v in obs.get("inventory", {}).items() if v > 0},
+                "nearby": obs.get("nearby", []),
+                "hp": obs.get("status", {}).get("health", 0),
+                "food": obs.get("status", {}).get("food", 0),
+                "ach": sum(1 for v in obs.get("achievements_status", {}).values() if v),
+            },
+            separators=(",", ":"),
+        )
+    except Exception as e:
+        return f'{{"error": "{str(e)}"}}'
+
+
+def create_message(
+    content: str, message_type: str, system_id: str, turn: int
+) -> SessionEventMarkovBlanketMessage:
+    """Create a SessionEventMarkovBlanketMessage representing cross-boundary communication."""
+    return SessionEventMarkovBlanketMessage(
+        content=content,
+        message_type=message_type,
+        metadata={"system_id": system_id, "turn": turn},
+        time_record=TimeRecord(event_time=time.time(), message_time=turn),
+    )
+
+
+async def run_episode(
+    config: ExperimentConfig,
+    model_name: str,
+    episode_num: int,
+    experiment_id: str,
+    pbar: tqdm | None = None,
+) -> dict[str, Any]:
+    """Run a single episode with a specific model using v3 tracing."""
+    # Create a new session tracer for this episode
+    session_tracer = SessionTracer(hooks=CRAFTER_HOOKS, db_url=config.database_url)
+
+    # Start session with metadata
+    session_id = await session_tracer.start_session(
+        metadata={
+            "model": model_name,
+            "episode": episode_num,
+            "experiment_id": experiment_id,
+            "difficulty": config.difficulty,
+        }
+    )
+
+    # Started tracing session (output disabled for clean UI)
+
+    # Store session in global bucket
+    _SESSIONS[session_id] = (experiment_id, session_tracer)
+
+    # Initialize LM with session tracer
+    lm = LM(
+        vendor="openai",
+        model=model_name,
+        temperature=0.1,  # Low temperature for more consistent gameplay
+        session_tracer=session_tracer,
+        system_id=f"crafter_agent_{model_name}",
+        enable_v3_tracing=True,
+    )
+
+    # Create HTTP client
+    async with httpx.AsyncClient() as client:
+        try:
+            # Initialize environment with consecutive seed
+            seed = (
+                config.base_seed + episode_num
+            )  # Base seed + episode number for consecutive seeds
+            request_data = {"config": {"difficulty": config.difficulty, "seed": seed}}
+            init_response = await retry_http_request(
+                client,
+                "POST",
+                f"{config.crafter_service_url}/env/CrafterClassic/initialize",
+                json=request_data,
+            )
+            init_data = init_response.json()
+
+            # Debug the response format (removed for clean output)
+
+            # Handle different possible response formats
+            if "instance_id" in init_data:
+                instance_id = init_data["instance_id"]
+            elif "env_id" in init_data:
+                instance_id = init_data["env_id"]
+            elif "id" in init_data:
+                instance_id = init_data["id"]
+            else:
+                # If none of the expected keys exist, print the response and raise a clear error
+                print(f"❌ Unexpected response format from Crafter service: {init_data}")
+                raise KeyError(
+                    f"Could not find environment ID in response. Available keys: {list(init_data.keys())}"
+                )
+
+            # Get initial observation (from initialize response)
+            obs = init_data["observation"]
+
+            prev_obs = obs
+            done = False
+            invalid_actions = 0
+            total_actions = 0
+            episode_start_time = time.time()
+
+            for turn in range(config.max_turns):
+                if done:
+                    break
+
+                # Check episode timeout
+                if time.time() - episode_start_time > config.episode_timeout:
+                    print(f"    ⏰ Episode {episode_num} timed out after {config.episode_timeout}s")
+                    done = True
+                    break
+
+                # Progress bar will be updated at end of turn
+
+                set_turn_number(turn)
+
+                # Start timestep for this turn
+                await session_tracer.start_timestep(f"turn_{turn}")
+
+                # Prepare context for the agent
+                inventory_str = ", ".join(
+                    [f"{k}: {v}" for k, v in obs.get("inventory", {}).items() if v > 0]
+                )
+                if not inventory_str:
+                    inventory_str = "empty"
+
+                nearby_str = ", ".join(obs.get("nearby", []))
+                if not nearby_str:
+                    nearby_str = "nothing"
+
+                status = obs.get("status", {})
+                health = status.get("health", 0)
+                hunger = status.get("food", 0)
+
+                # Get more detailed game state
+                position = obs.get("position", [0, 0])
+                achievements = obs.get("achievements_status", {})
+                unlocked = [name for name, status in achievements.items() if status]
+                achievements_str = ", ".join(unlocked) if unlocked else "none"
+
+                # Get semantic map if available
+                semantic_map = obs.get("semantic_map", None)
+                map_str = ""
+                if semantic_map is not None:
+                    # Simple 5x5 view around player
+                    try:
+                        px, py = position
+                        view_size = 5
+                        half = view_size // 2
+                        map_lines = []
+                        for dy in range(-half, half + 1):
+                            row = []
+                            for dx in range(-half, half + 1):
+                                x, y = px + dx, py + dy
+                                if dx == 0 and dy == 0:
+                                    row.append("@")  # Player
+                                elif 0 <= x < len(semantic_map) and 0 <= y < len(semantic_map[0]):
+                                    cell = semantic_map[x][y]
+                                    # Map common items
+                                    if cell == 0:
+                                        row.append(".")  # Empty/grass
+                                    elif cell == 1:
+                                        row.append("T")  # Tree
+                                    elif cell == 2:
+                                        row.append("S")  # Stone
+                                    elif cell == 3:
+                                        row.append("C")  # Cow
+                                    elif cell == 4:
+                                        row.append("W")  # Water
+                                    else:
+                                        row.append("?")
+                                else:
+                                    row.append("#")  # Out of bounds
+                            map_lines.append(" ".join(row))
+                        map_str = "\nMap (5x5 view, @ = you):\n" + "\n".join(map_lines)
+                    except Exception:
+                        map_str = "\nMap view unavailable"
+
+                # Create agent prompt
+                prompt = f"""Game State (Turn {turn}):
+- Position: {position}
+- Health: {health}/9
+- Hunger: {hunger}/9
+- Inventory: {inventory_str}
+- Nearby objects: {nearby_str}
+- Achievements unlocked: {achievements_str}
+{map_str}
+
+Choose your next actions based on what you see. Use the 'interact' tool with a list of action IDs.
+
+Tips:
+- Look at the map! T=tree (wood), S=stone, C=cow (food), W=water
+- To collect resources: move to them (actions 1-4) then use action 5 (do)
+- To craft: place table (8) first, then craft tools (11-16)
+- If hungry and see cow (C), move to it and eat (17)
+
+What actions do you want to take?"""
+
+                # Send observation as message
+                obs_msg = create_message(
+                    f"Observation: {compress_observation_for_trace(obs)}",
+                    "system",
+                    f"crafter_env_{instance_id}",
+                    turn,
+                )
+                await session_tracer.record_message(
+                    content=obs_msg.content,
+                    message_type=obs_msg.message_type,
+                    event_time=obs_msg.time_record.event_time,
+                    message_time=obs_msg.time_record.message_time,
+                    metadata=obs_msg.metadata,
+                )
+
+                # Get action from LM with tools (with timeout)
+                try:
+                    # Define the interact tool for Crafter
+                    from pydantic import BaseModel, Field
+                    from synth_ai.lm.tools.base import BaseTool
+
+                    class InteractArgs(BaseModel):
+                        actions: list[int] = Field(..., description="List of action IDs to execute")
+
+                    interact_tool = BaseTool(
+                        name="interact",
+                        arguments=InteractArgs,
+                        description="Execute actions in the Crafter game",
+                    )
+
+                    # Create system message that explains available actions
+                    action_list = "\n".join(
+                        [f"{action_id}: {action}" for action, action_id in CRAFTER_ACTIONS.items()]
+                    )
+                    system_message = f"""You are an agent playing Crafter, a 2D survival game. Your goal is to survive and unlock achievements.
+
+You MUST use the 'interact' tool to execute actions. The tool takes a list of action IDs.
+
+Action ID mapping:
+{action_list}
+
+Strategy tips:
+- Start by collecting wood (move to trees and use action 5)
+- Place a crafting table (action 8) to unlock crafting recipes
+- Craft tools to collect resources more efficiently
+- Eat when hungry, sleep when tired
+- Explore to find different resources
+
+IMPORTANT: Always use the 'interact' tool with a list of action IDs. For example: interact(actions=[2, 2, 5]) to move right twice and collect."""
+
+                    # Get actions from LM using tools with timeout
+                    try:
+                        action_response = await asyncio.wait_for(
+                            lm.respond_async(
+                                system_message=system_message,
+                                user_message=prompt,
+                                tools=[interact_tool],
+                                turn_number=turn,
+                            ),
+                            timeout=config.turn_timeout,
+                        )
+                    except TimeoutError:
+                        print(
+                            f"    ⏰ Turn {turn} timed out for episode {episode_num} after {config.turn_timeout}s"
+                        )
+                        action_response = None
+                        done = True
+                        break
+
+                    # Debug: print response (removed for clean output)
+
+                    # Extract tool calls from response
+                    if hasattr(action_response, "tool_calls") and action_response.tool_calls:
+                        tool_calls = action_response.tool_calls
+
+                        # Process each tool call
+                        for tool_call in tool_calls:
+                            if tool_call.get("function", {}).get("name") == "interact":
+                                # Extract actions from the tool call
+                                import json
+
+                                args = json.loads(
+                                    tool_call.get("function", {}).get("arguments", "{}")
+                                )
+                                actions = args.get("actions", [])
+
+                                if not actions:
+                                    # If no actions provided, use noop
+                                    actions = [0]
+
+                                # Execute each action separately
+                                for i, action_id in enumerate(actions):
+                                    # Capture BEFORE frame
+                                    frame_before_b64 = None
+                                    try:
+                                        fr = await retry_http_request(
+                                            client,
+                                            "GET",
+                                            f"{config.crafter_service_url}/env/CrafterClassic/frame",
+                                            params={"env_id": instance_id},
+                                        )
+                                        if fr.status_code == 200:
+                                            frame_before_b64 = fr.json().get("image_base64")
+                                    except Exception:
+                                        frame_before_b64 = None
+                                    total_actions += 1
+
+                                    # Validate action ID
+                                    if action_id not in INT_TO_ACTION_STRING:
+                                        # Invalid action logging removed for clean output
+                                        action_id = 0
+                                        invalid_actions += 1
+
+                                    # Send action to Crafter service with timeout
+                                    try:
+                                        step_response = await asyncio.wait_for(
+                                            retry_http_request(
+                                                client,
+                                                "POST",
+                                                f"{config.crafter_service_url}/env/CrafterClassic/step",
+                                                json={
+                                                    "env_id": instance_id,
+                                                    "action": {
+                                                        "tool_calls": [
+                                                            {
+                                                                "tool": "interact",
+                                                                "args": {"action": action_id},
+                                                            }
+                                                        ]
+                                                    },
+                                                },
+                                            ),
+                                            timeout=5.0,  # 5 second timeout for individual action
+                                        )
+                                    except TimeoutError:
+                                        print(
+                                            f"    ⏰ Action execution timed out in episode {episode_num}"
+                                        )
+                                        done = True
+                                        break
+
+                                    if step_response.status_code != 200:
+                                        print(
+                                            f"    ❌ Step failed: {step_response.status_code} - {step_response.text}"
+                                        )
+                                        done = True
+                                        break
+
+                                    step_data = step_response.json()
+
+                                    # Extract data from response
+                                    new_obs = step_data["observation"]
+                                    reward = step_data["reward"]
+                                    done = step_data["done"]
+
+                                    # Record runtime event for action
+                                    action_name = INT_TO_ACTION_STRING.get(action_id, "unknown")
+                                    runtime_event = RuntimeEvent(
+                                        system_instance_id=f"crafter_env_{instance_id}",
+                                        time_record=TimeRecord(
+                                            event_time=time.time(), message_time=turn
+                                        ),
+                                        actions=[action_id],
+                                        metadata={
+                                            "action_name": action_name,
+                                            "valid": action_name != "noop" or invalid_actions == 0,
+                                        },
+                                    )
+                                    await session_tracer.record_event(runtime_event)
+
+                                    # Capture AFTER frame
+                                    frame_after_b64 = None
+                                    try:
+                                        fr = await retry_http_request(
+                                            client,
+                                            "GET",
+                                            f"{config.crafter_service_url}/env/CrafterClassic/frame",
+                                            params={"env_id": instance_id},
+                                        )
+                                        if fr.status_code == 200:
+                                            frame_after_b64 = fr.json().get("image_base64")
+                                    except Exception:
+                                        frame_after_b64 = None
+
+                                    # Save frames to assets and compute URIs
+                                    before_uri = None
+                                    after_uri = None
+                                    try:
+                                        if frame_before_b64:
+                                            import base64
+                                            from pathlib import Path
+
+                                            assets_dir = Path("traces/v3/assets") / session_id
+                                            assets_dir.mkdir(parents=True, exist_ok=True)
+                                            before_path = assets_dir / f"{turn}_{i}_before.png"
+                                            with open(before_path, "wb") as f:
+                                                f.write(base64.b64decode(frame_before_b64))
+                                            before_uri = str(before_path)
+                                        if frame_after_b64:
+                                            import base64
+                                            from pathlib import Path
+
+                                            assets_dir = Path("traces/v3/assets") / session_id
+                                            assets_dir.mkdir(parents=True, exist_ok=True)
+                                            after_path = assets_dir / f"{turn}_{i}_after.png"
+                                            with open(after_path, "wb") as f:
+                                                f.write(base64.b64decode(frame_after_b64))
+                                            after_uri = str(after_path)
+                                    except Exception:
+                                        before_uri = None
+                                        after_uri = None
+
+                                    # Record environment event with visuals
+                                    env_event = EnvironmentEvent(
+                                        system_instance_id=f"crafter_env_{instance_id}",
+                                        time_record=TimeRecord(
+                                            event_time=time.time(), message_time=turn
+                                        ),
+                                        reward=reward,
+                                        terminated=done,
+                                        system_state_before={
+                                            "observation": prev_obs,
+                                            **(
+                                                {"visuals": {"frame_uri": before_uri}}
+                                                if before_uri
+                                                else {}
+                                            ),
+                                        },
+                                        system_state_after={
+                                            "observation": new_obs,
+                                            "public_state": {
+                                                "achievements_status": new_obs.get(
+                                                    "achievements_status", {}
+                                                )
+                                            },
+                                            **(
+                                                {"visuals": {"frame_uri": after_uri}}
+                                                if after_uri
+                                                else {}
+                                            ),
+                                        },
+                                    )
+                                    await session_tracer.record_event(env_event)
+
+                                    # Update for next turn
+                                    prev_obs = obs
+                                    obs = new_obs
+
+                                    if done:
+                                        break
+
+                                # Per-episode progress updated once per turn (not per action)
+                    else:
+                        # No tool calls provided, use noop
+                        action_id = 0
+                        total_actions += 1
+                        invalid_actions += 1
+
+                        # Send noop action with timeout
+                        try:
+                            step_response = await asyncio.wait_for(
+                                retry_http_request(
+                                    client,
+                                    "POST",
+                                    f"{config.crafter_service_url}/env/CrafterClassic/step",
+                                    json={
+                                        "env_id": instance_id,
+                                        "action": {
+                                            "tool_calls": [
+                                                {"tool": "interact", "args": {"action": action_id}}
+                                            ]
+                                        },
+                                    },
+                                ),
+                                timeout=5.0,  # 5 second timeout
+                            )
+                        except TimeoutError:
+                            print(f"    ⏰ Noop action timed out in episode {episode_num}")
+                            done = True
+                            break
+
+                        if step_response.status_code != 200:
+                            print(
+                                f"    ❌ Step failed: {step_response.status_code} - {step_response.text}"
+                            )
+                            done = True
+                        else:
+                            step_data = step_response.json()
+                            new_obs = step_data["observation"]
+                            reward = step_data["reward"]
+                            done = step_data["done"]
+
+                            # Update observation
+                            prev_obs = obs
+                            obs = new_obs
+
+                    # End timestep
+                    await session_tracer.end_timestep(f"turn_{turn}")
+                    # Update per-episode progress bar once per turn
+                    if pbar is not None:
+                        current_achievements = sum(
+                            1 for v in obs.get("achievements_status", {}).values() if v
+                        )
+                        pbar.set_postfix({"ach": current_achievements})
+                        pbar.update(1)
+
+                except Exception as e:
+                    print(f"    ❌ Environment step error: {e}")
+                    done = True
+
+            # Progress bar updated per turn above
+
+            # Calculate invalid action rate
+            invalid_rate = invalid_actions / total_actions if total_actions > 0 else 0
+
+            # Calculate achievements
+            final_achievements = obs.get("achievements_status", {})
+            total_achievements = sum(1 for v in final_achievements.values() if v)
+
+            # Terminate environment
+            try:
+                await retry_http_request(
+                    client,
+                    "POST",
+                    f"{config.crafter_service_url}/env/CrafterClassic/terminate",
+                    json={"env_id": instance_id},
+                )
+            except Exception as e:
+                print(f"    ⚠️  Failed to terminate environment: {e}")
+
+            # End session
+            await session_tracer.end_session(save=config.save_traces)
+            # Close the tracer for this episode
+            await session_tracer.close()
+
+            return {
+                "model": model_name,
+                "episode": episode_num,
+                "total_achievements": total_achievements,
+                "achievements": final_achievements,
+                "invalid_action_rate": invalid_rate,
+                "total_actions": total_actions,
+                "invalid_actions": invalid_actions,
+                "session_id": session_id,
+            }
+
+        except Exception as e:
+            print(f"    ❌ Episode failed: {e}")
+            import traceback
+
+            traceback.print_exc()
+
+            # End session even if failed
+            await session_tracer.end_session(save=config.save_traces)
+            # Close the tracer for this episode
+            await session_tracer.close()
+
+            return {
+                "model": model_name,
+                "episode": episode_num,
+                "total_achievements": 0,
+                "achievements": {},
+                "invalid_action_rate": 1.0,
+                "total_actions": 0,
+                "invalid_actions": 0,
+                "session_id": session_id,
+                "error": str(e),
+            }
+
+
+async def run_model_experiment(
+    config: ExperimentConfig, model_name: str, experiment_id: str, position_base: int = 0
+) -> list[dict[str, Any]]:
+    """Run multiple episodes for a single model in parallel with per-episode stacked progress bars."""
+    # print(f"\nRunning {config.num_episodes} episodes for {model_name} in parallel...\n")
+
+    # One progress bar per episode, stacked
+    episode_bars = [
+        tqdm(
+            total=config.max_turns,
+            desc=f"{model_name} | ep{i + 1}",
+            unit="turn",
+            leave=True,
+            position=position_base + i,
+        )
+        for i in range(config.num_episodes)
+    ]
+
+    try:
+        # Create tasks for all episodes (each will create its own tracer) with concurrency limit
+        sem = asyncio.Semaphore(max(1, int(config.concurrency)))
+
+        async def _limited_run(ep_idx: int):
+            async with sem:
+                pbar = episode_bars[ep_idx]
+                try:
+                    return await run_episode(config, model_name, ep_idx, experiment_id, pbar)
+                finally:
+                    pbar.close()
+
+        tasks = [_limited_run(i) for i in range(config.num_episodes)]
+
+        # Run all episodes in parallel
+        results = await asyncio.gather(*tasks)
+
+        # Optional summary on the last bar
+        successful_results = [r for r in results if "error" not in r]
+        if successful_results and episode_bars:
+            avg_ach = sum(r["total_achievements"] for r in successful_results) / len(
+                successful_results
+            )
+            avg_inv = sum(r["invalid_action_rate"] for r in successful_results) / len(
+                successful_results
+            )
+            with contextlib.suppress(Exception):
+                episode_bars[-1].set_postfix(
+                    {"avg_ach": f"{avg_ach:.1f}", "inv_rate": f"{avg_inv:.1%}"}
+                )
+
+        return results
+    finally:
+        for b in episode_bars:
+            with contextlib.suppress(Exception):
+                b.close()
+
+
+async def analyze_results(config: ExperimentConfig, all_results: dict[str, list[dict[str, Any]]]):
+    """Analyze results across all models using v3 database."""
+    print("\n📊 Analysis Results:")
+    print("=" * 80)
+
+    # Initialize database manager
+    db_manager = AsyncSQLTraceManager(config.database_url)
+    await db_manager.initialize()
+
+    try:
+        # Basic statistics by model
+        model_stats = {}
+        for model, results in all_results.items():
+            valid_results = [r for r in results if "error" not in r]
+            if valid_results:
+                achievements = [r["total_achievements"] for r in valid_results]
+                invalid_rates = [r["invalid_action_rate"] for r in valid_results]
+
+                model_stats[model] = {
+                    "avg_achievements": np.mean(achievements),
+                    "std_achievements": np.std(achievements),
+                    "max_achievements": max(achievements),
+                    "avg_invalid_rate": np.mean(invalid_rates),
+                    "success_rate": len(valid_results) / len(results),
+                }
+
+        # Print model comparison
+        print("\n📈 Model Performance Summary:")
+        print(
+            f"{'Model':<20} {'Avg Achievements':<18} {'Max Achievements':<18} {'Invalid Rate':<15} {'Success Rate':<15}"
+        )
+        print("-" * 86)
+
+        for model, stats in sorted(
+            model_stats.items(), key=lambda x: x[1]["avg_achievements"], reverse=True
+        ):
+            print(
+                f"{model:<20} {stats['avg_achievements']:>6.2f} ± {stats['std_achievements']:>4.2f}     "
+                f"{stats['max_achievements']:>16}     {stats['avg_invalid_rate']:>12.2%}     {stats['success_rate']:>12.2%}"
+            )
+
+        # Achievement frequency analysis
+        print("\n🏆 Achievement Frequencies:")
+        achievement_counts = defaultdict(lambda: defaultdict(int))
+
+        for model, results in all_results.items():
+            for result in results:
+                if "error" not in result:
+                    for achievement, unlocked in result["achievements"].items():
+                        if unlocked:
+                            achievement_counts[model][achievement] += 1
+
+        # Get all unique achievements
+        all_achievements = set()
+        for model_achievements in achievement_counts.values():
+            all_achievements.update(model_achievements.keys())
+
+        # Print achievement table
+        if all_achievements:
+            print(
+                f"\n{'Achievement':<25} "
+                + " ".join(f"{model[:8]:>10}" for model in sorted(all_results.keys()))
+            )
+            print("-" * (25 + 11 * len(all_results)))
+
+            for achievement in sorted(all_achievements):
+                row = f"{achievement:<25}"
+                for model in sorted(all_results.keys()):
+                    count = achievement_counts[model].get(achievement, 0)
+                    total = len([r for r in all_results[model] if "error" not in r])
+                    pct = (count / total * 100) if total > 0 else 0
+                    row += f" {count:>3}/{total:<3} ({pct:>3.0f}%)"
+                print(row)
+
+        # Query model usage from database - filter to only show models used in this experiment
+        print("\n💰 Model Usage Statistics from Current Experiment:")
+        model_usage_df = await db_manager.get_model_usage()
+
+        if model_usage_df is not None and not model_usage_df.empty:
+            # Filter to only show models from this experiment
+            experiment_models = set(all_results.keys())
+            filtered_df = model_usage_df[model_usage_df["model_name"].isin(experiment_models)]
+
+            if not filtered_df.empty:
+                # Format model usage statistics as table
+                print(
+                    f"{'Model':<20} {'Provider':<10} {'Usage Count':<12} {'Avg Latency (ms)':<18} {'Total Cost':<12}"
+                )
+                print("-" * 72)
+                for _, row in filtered_df.iterrows():
+                    avg_latency = row["avg_latency_ms"]
+                    if pd.notna(avg_latency):
+                        print(
+                            f"{row['model_name']:<20} {row['provider'] or 'N/A':<10} {row['usage_count']:<12} "
+                            f"{avg_latency:<18.2f} ${row['total_cost_usd']:<11.4f}"
+                        )
+                    else:
+                        print(
+                            f"{row['model_name']:<20} {row['provider'] or 'N/A':<10} {row['usage_count']:<12} "
+                            f"{'N/A':<18} ${row['total_cost_usd']:<11.4f}"
+                        )
+
+        # Export detailed results under a temp/ directory (git-ignored)
+        import os
+        from pathlib import Path
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        out_dir = Path(os.getenv("SYNTH_OUTPUT_DIR", "temp")).resolve()
+        out_dir.mkdir(parents=True, exist_ok=True)
+        results_path = out_dir / f"crafter_experiment_results_{timestamp}.json"
+
+        with open(results_path, "w") as f:
+            json.dump(
+                {
+                    "config": {
+                        "num_episodes": config.num_episodes,
+                        "max_turns": config.max_turns,
+                        "difficulty": config.difficulty,
+                        "models": list(all_results.keys()),
+                    },
+                    "results": all_results,
+                    "statistics": model_stats,
+                    "timestamp": timestamp,
+                },
+                f,
+                indent=2,
+            )
+
+        print(f"\n💾 Detailed results saved to: {results_path}")
+
+    finally:
+        await db_manager.close()
+
+
+async def main():
+    """Main entry point for the experiment."""
+    parser = argparse.ArgumentParser(description="Run Crafter experiments with multiple models")
+    parser.add_argument("--episodes", type=int, default=5, help="Number of episodes per model")
+    parser.add_argument("--max-turns", type=int, default=100, help="Maximum turns per episode")
+    parser.add_argument(
+        "--difficulty", choices=["easy", "medium", "hard"], default="easy", help="Game difficulty"
+    )
+    parser.add_argument("--models", nargs="+", default=MODELS_TO_TEST, help="Models to test")
+    parser.add_argument("--no-save", action="store_true", help="Don't save traces to database")
+    parser.add_argument("--quiet", action="store_true", help="Reduce output verbosity")
+    parser.add_argument("--db-url", default=DATABASE_URL, help="Database URL for tracing")
+    parser.add_argument(
+        "--concurrency", type=int, default=5, help="Max concurrent rollouts per model"
+    )
+    parser.add_argument(
+        "--base-seed",
+        type=int,
+        default=1000,
+        help="Base seed for episodes (episodes use base_seed+episode_num)",
+    )
+    parser.add_argument(
+        "--turn-timeout", type=float, default=30.0, help="Timeout per turn in seconds"
+    )
+    parser.add_argument(
+        "--episode-timeout", type=float, default=300.0, help="Total timeout per episode in seconds"
+    )
+
+    args = parser.parse_args()
+
+    # Create configuration
+    config = ExperimentConfig()
+    config.num_episodes = args.episodes
+    config.max_turns = args.max_turns
+    config.difficulty = args.difficulty
+    config.save_traces = not args.no_save
+    config.verbose = not args.quiet
+    config.quiet = args.quiet
+    config.database_url = args.db_url
+    config.base_seed = args.base_seed
+    config.turn_timeout = args.turn_timeout
+    config.episode_timeout = args.episode_timeout
+    config.concurrency = max(1, int(args.concurrency))
+
+    # Generate experiment ID
+    experiment_id = f"crafter_multi_model_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+
+    print("🎮 Crafter Multi-Model Experiment")
+    print("=" * 50)
+    print(f"Experiment ID: {experiment_id}")
+    print(f"Models: {', '.join(args.models)}")
+    print(f"Episodes per model: {config.num_episodes}")
+    print(f"Max turns per episode: {config.max_turns}")
+    print(f"Difficulty: {config.difficulty}")
+    print(f"Seeds: {config.base_seed} to {config.base_seed + config.num_episodes - 1}")
+    print(f"Turn timeout: {config.turn_timeout}s")
+    print(f"Episode timeout: {config.episode_timeout}s")
+    print(f"Save traces: {config.save_traces}")
+    print(f"Database URL: {config.database_url}")
+    print("=" * 50)
+
+    # Check Crafter service
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(f"{config.crafter_service_url}/health", timeout=5.0)
+            if response.status_code != 200:
+                print(f"❌ Crafter service not healthy at {config.crafter_service_url}")
+                return
+    except Exception as e:
+        print(f"❌ Cannot connect to Crafter service at {config.crafter_service_url}: {e}")
+        print("Please ensure the Crafter service is running.")
+        return
+
+    print("✅ Crafter service is running")
+
+    # Run experiments for each model in parallel with stacked per-episode progress bars
+    all_results = {}
+    model_tasks = []
+    for idx, model in enumerate(args.models):
+        base = idx * (config.num_episodes + 1)
+        model_tasks.append(run_model_experiment(config, model, experiment_id, position_base=base))
+    results_list = await asyncio.gather(*model_tasks)
+    for model, results in zip(args.models, results_list, strict=False):
+        all_results[model] = results
+
+    # Analyze and compare results
+    await analyze_results(config, all_results)
+
+    print("\n✅ Experiment complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())