synth-ai 0.1.9__py3-none-any.whl → 0.2.1.dev0__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_trace_evaluation.py

@@ -0,0 +1,1412 @@
+"""
+Full Crafter Evaluation with Traces, Rewards, and Viewer
+Extends eval_framework.py with SystemTrace capture, viewer, and comprehensive logging.
+"""
+
+import asyncio
+import json
+import base64
+import io
+import os
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, List, Optional, Set, Tuple, Any, Union
+from dataclasses import dataclass, asdict
+import uuid
+
+import numpy as np
+from PIL import Image
+import pandas as pd
+from tqdm import tqdm
+import time
+from fastapi import FastAPI, HTTPException
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse, FileResponse
+import uvicorn
+
+# Import synth-sdk trace structures
+from synth_sdk.tracing.abstractions import (
+    SystemTrace,
+    EventPartitionElement,
+    Event,
+    AgentComputeStep,
+    EnvironmentComputeStep,
+    MessageInputs,
+    MessageOutputs,
+    ArbitraryInputs,
+    ArbitraryOutputs,
+    TrainingQuestion,
+    RewardSignal,
+    Dataset,
+)
+
+# Import base evaluation framework
+from src.synth_env.examples.crafter_classic.agent_demos.crafter_evaluation_framework import (
+    CrafterEvalFramework,
+    TrajectoryResult,
+    AggregateResults,
+    ACHIEVEMENT_CATEGORIES,
+    ALL_ACHIEVEMENTS,
+    TERMINATION_REASONS,
+    crafter_score,
+    balrog_score,
+)
+
+# Action names mapping for Crafter
+ACTION_NAMES = {
+    -1: "initial_state",
+    0: "noop",
+    1: "move_left",
+    2: "move_right",
+    3: "move_up",
+    4: "move_down",
+    5: "do",
+    6: "sleep",
+    7: "place_stone",
+    8: "place_table",
+    9: "place_furnace",
+    10: "place_plant",
+    11: "make_wood_pickaxe",
+    12: "make_stone_pickaxe",
+    13: "make_iron_pickaxe",
+    14: "make_wood_sword",
+    15: "make_stone_sword",
+    16: "make_iron_sword",
+}
+
+
+class FullCrafterEvalFramework(CrafterEvalFramework):
+    """Extended evaluation framework with trace capture and visualization."""
+
+    def __init__(self, capture_images: bool = True, output_dir: Optional[str] = None):
+        super().__init__()
+        self.capture_images = capture_images
+
+        # Use standardized eval directory structure
+        if output_dir is None:
+            # Create timestamp-based directory under src/evals/
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            self.output_dir = Path("src/evals") / "crafter" / f"run_{timestamp}"
+        else:
+            self.output_dir = Path(output_dir)
+
+        self.traces_dir = self.output_dir / "traces"
+        self.viewer_dir = self.output_dir / "viewer"
+
+        # Create directories
+        self.traces_dir.mkdir(parents=True, exist_ok=True)
+        self.viewer_dir.mkdir(parents=True, exist_ok=True)
+
+        # Store traces and datasets
+        self.system_traces: Dict[str, SystemTrace] = {}
+        self.datasets: Dict[str, Dataset] = {}
+
+    def _encode_image_to_base64(self, rgb_array: np.ndarray) -> str:
+        """Convert RGB numpy array to base64 PNG string."""
+        image = Image.fromarray(rgb_array.astype("uint8"), "RGB")
+        buffer = io.BytesIO()
+        image.save(buffer, format="PNG")
+        buffer.seek(0)
+        return base64.b64encode(buffer.read()).decode("utf-8")
+
+    async def run_single_trajectory_with_trace(
+        self,
+        model_name: str,
+        difficulty: str,
+        seed: int,
+        max_turns: int = 30,
+        collect_detailed_data: bool = True,
+    ) -> TrajectoryResult:
+        """Run a single trajectory with comprehensive trace capture."""
+        from src.synth_env.examples.crafter_classic.agent_demos.crafter_react_agent import (
+            ReActAgent,
+            CrafterHistoryObservationCallable,
+            CrafterMove,
+        )
+        from src.synth_env.examples.crafter_classic.environment import (
+            CrafterClassicEnvironment,
+        )
+        from src.synth_env.examples.crafter_classic.taskset import (
+            CrafterTaskInstance,
+            CrafterTaskInstanceMetadata,
+        )
+        from src.synth_env.tasks.core import Impetus, Intent
+        from synth_ai.zyk import LM
+
+        # Create task instance
+        metadata = CrafterTaskInstanceMetadata(
+            difficulty=difficulty,
+            seed=seed,
+            num_trees_radius=0,
+            num_cows_radius=0,
+            num_hostiles_radius=0,
+        )
+        instance = CrafterTaskInstance(
+            id=uuid.uuid4(),
+            impetus=Impetus(
+                instructions=f"Survive and unlock achievements in a {difficulty} environment."
+            ),
+            intent=Intent(rubric={}, gold_trajectories=None, gold_state_diff={}),
+            metadata=metadata,
+            is_reproducible=True,
+            initial_engine_snapshot=None,
+        )
+
+        # Setup environment and agent
+        start_time = time.perf_counter()
+        hist_cb = CrafterHistoryObservationCallable(max_history=1)
+        env = CrafterClassicEnvironment(instance, custom_step_obs=hist_cb)
+
+        llm = LM(model_name=model_name, formatting_model_name=model_name, temperature=0.0)
+        agent = ReActAgent(llm, max_turns=max_turns)
+
+        # Initialize tracking
+        trajectory_id = str(uuid.uuid4())
+        achievements_unlocked = set()
+        achievement_turn_unlocked = {}
+        actions_per_turn = []
+        turn_by_turn_data = [] if collect_detailed_data else None
+
+        # Initialize SystemTrace
+        system_trace = SystemTrace(
+            system_name="crafter_evaluation",
+            system_id=f"crafter_{model_name}_{difficulty}",
+            system_instance_id=trajectory_id,
+            partition=[],
+            metadata={
+                "model_name": model_name,
+                "difficulty": difficulty,
+                "seed": seed,
+                "max_turns": max_turns,
+            },
+            instance_metadata={
+                "start_time": datetime.now().isoformat(),
+                "capture_images": self.capture_images,
+            },
+        )
+
+        # Create TrainingQuestion for this trajectory
+        training_question = TrainingQuestion(
+            id=trajectory_id,
+            intent=f"Survive and unlock achievements in a {difficulty} Crafter environment (seed={seed})",
+            criteria="Maximize the number of achievements unlocked and survive as long as possible",
+        )
+
+        # Run episode
+        obs_payload = await env.initialize()
+        turn_count = 0
+        termination_reason = "unknown"
+        partition_index = 0
+
+        # Create progress bar for this trajectory
+        pbar = tqdm(
+            total=max_turns,
+            desc=f"{model_name} ({difficulty}) Seed {seed}",
+            unit="turn",
+            leave=False,
+            ncols=100,
+        )
+
+        try:
+            while turn_count < max_turns:
+                turn_count += 1
+                pbar.update(1)
+
+                # Track achievements
+                easy_count = len(
+                    [a for a in achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["easy"]]
+                )
+                medium_count = len(
+                    [a for a in achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["medium"]]
+                )
+                hard_count = len(
+                    [a for a in achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["hard"]]
+                )
+                total_count = len(achievements_unlocked)
+
+                achievement_display = f"{total_count}({easy_count}/{medium_count}/{hard_count})"
+
+                pbar.set_postfix(
+                    {
+                        "achievements": achievement_display,
+                        "steps": obs_payload.get("public", {}).num_steps_taken
+                        if hasattr(obs_payload.get("public", {}), "num_steps_taken")
+                        else 0,
+                    }
+                )
+
+                current_formatted_obs = obs_payload["formatted_obs"]
+
+                # Track achievements at start of turn
+                current_achievements = set()
+                if "public" in obs_payload and hasattr(
+                    obs_payload["public"], "achievements_status"
+                ):
+                    current_achievements = {
+                        ach
+                        for ach, status in obs_payload["public"].achievements_status.items()
+                        if status
+                    }
+
+                # Check for new achievements
+                new_achievements = current_achievements - achievements_unlocked
+                for ach in new_achievements:
+                    achievements_unlocked.add(ach)
+                    achievement_turn_unlocked[ach] = turn_count
+                    agent.current_achievements.add(ach)
+
+                # Create EventPartitionElement for this turn
+                event_partition = EventPartitionElement(partition_index=partition_index, events=[])
+
+                # Capture initial state image before actions
+                initial_image = None
+                if self.capture_images and turn_count == 1:
+                    try:
+                        if hasattr(env, "engine") and hasattr(env.engine, "env"):
+                            crafter_env = env.engine.env
+                            if hasattr(crafter_env, "_render_mode"):
+                                crafter_env._render_mode = "rgb_array"
+                            initial_rgb = crafter_env.render()
+                            if initial_rgb is not None:
+                                initial_image = self._encode_image_to_base64(initial_rgb)
+                                print(f"✓ Initial state captured before actions")
+                    except Exception as e:
+                        print(f"Warning: Failed to capture initial image: {e}")
+
+                # Agent decision phase
+                agent_compute_began = datetime.now()
+
+                # Create proper system and user prompts
+                system_prompt = "You are playing Crafter. Your goal is to survive and unlock as many achievements as possible."
+
+                # Get agent decision with proper message structure
+                agent_decision = await agent.decide(current_formatted_obs, obs_payload)
+                agent_compute_ended = datetime.now()
+
+                if agent_decision == [-1]:  # Agent terminated
+                    termination_reason = "agent_quit"
+                    break
+
+                action_sequence = agent_decision
+                actions_per_turn.append(len(action_sequence))
+
+                # Create proper tool calls for the actions
+                tool_calls = [
+                    {
+                        "id": f"crafter_action_{turn_count}",
+                        "type": "function",
+                        "function": {
+                            "name": "crafter_interact",
+                            "arguments": json.dumps(
+                                {
+                                    "actions": action_sequence,
+                                    "reasoning": f"Executing {len(action_sequence)} actions: {[ACTION_NAMES.get(act, f'action_{act}') for act in action_sequence]}",
+                                }
+                            ),
+                        },
+                    }
+                ]
+
+                tool_results = [
+                    {
+                        "tool_call_id": f"crafter_action_{turn_count}",
+                        "content": f"Planned actions: {[ACTION_NAMES.get(act, f'action_{act}') for act in action_sequence]}",
+                    }
+                ]
+
+                # Create input messages
+                input_messages = [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": current_formatted_obs},
+                ]
+
+                # Create output messages with tool calls
+                output_messages = [
+                    {
+                        "role": "assistant",
+                        "content": f"I need to execute {len(action_sequence)} actions to progress in the game: {[ACTION_NAMES.get(act, f'action_{act}') for act in action_sequence]}",
+                        "tool_calls": tool_calls,
+                    }
+                ]
+
+                # Add tool results
+                for tool_result in tool_results:
+                    output_messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": tool_result["tool_call_id"],
+                            "content": tool_result["content"],
+                        }
+                    )
+
+                # Create AgentComputeStep with proper message structure
+                agent_compute_step = AgentComputeStep(
+                    event_order=0,
+                    compute_began=agent_compute_began,
+                    compute_ended=agent_compute_ended,
+                    compute_input=[MessageInputs(messages=input_messages)],
+                    compute_output=[MessageOutputs(messages=output_messages)],
+                    model_name=model_name,
+                    model_params={"temperature": 0.0},
+                    should_learn=True,
+                )
+
+                # Collect turn data
+                if collect_detailed_data:
+                    turn_data = {
+                        "turn": turn_count,
+                        "actions_planned": len(action_sequence),
+                        "achievements_at_start": list(current_achievements),
+                        "new_achievements_this_turn": list(new_achievements),
+                        "steps_before_turn": obs_payload.get("public", {}).num_steps_taken
+                        if hasattr(obs_payload.get("public", {}), "num_steps_taken")
+                        else 0,
+                    }
+                    turn_by_turn_data.append(turn_data)
+
+                # Execute actions and collect environment steps
+                environment_compute_steps = []
+
+                # Add initial state as first "step" if we have it
+                if initial_image and turn_count == 1:
+                    initial_step = EnvironmentComputeStep(
+                        event_order=0,
+                        compute_began=agent_compute_began,
+                        compute_ended=agent_compute_began,
+                        compute_input=[ArbitraryInputs(inputs={"action": "initial_state"})],
+                        compute_output=[
+                            ArbitraryOutputs(
+                                outputs={
+                                    "action_index": -1,  # Special index for initial state
+                                    "action_order": -1,
+                                    "image_base64": initial_image,
+                                    "error": None,
+                                    "terminated": False,
+                                    "truncated": False,
+                                    "reward": 0.0,
+                                    "total_reward": 0.0,
+                                    "num_steps": 0,
+                                }
+                            )
+                        ],
+                    )
+                    environment_compute_steps.append(initial_step)
+
+                for i, act_idx in enumerate(action_sequence):
+                    env_compute_began = datetime.now()
+
+                    # Execute action
+                    obs_payload = await env.step([[CrafterMove(act_idx)]])
+                    env_compute_ended = datetime.now()
+
+                    # Capture image after step if enabled
+                    post_step_image = None
+                    if self.capture_images:
+                        try:
+                            # Access the underlying crafter environment through the engine
+                            if hasattr(env, "engine") and hasattr(env.engine, "env"):
+                                # Force render mode to 'rgb_array' if needed
+                                crafter_env = env.engine.env
+                                if hasattr(crafter_env, "_render_mode"):
+                                    crafter_env._render_mode = "rgb_array"
+
+                                rgb_array = crafter_env.render()
+                                if rgb_array is not None:
+                                    post_step_image = self._encode_image_to_base64(rgb_array)
+                                    # Debug: check if images are different
+                                    # if (
+                                    #     turn_count == 1 and i < 3
+                                    # ):  # Debug first turn, first 3 actions
+                                    #     print(
+                                    #         f"✓ Action {i} ({act_idx}): Image captured, first pixel: {rgb_array[0, 0]}, shape: {rgb_array.shape}"
+                                    #     )
+                                else:
+                                    print(f"Warning: render() returned None")
+                            else:
+                                print(f"Warning: Cannot access env.engine.env for rendering")
+                        except Exception as e:
+                            print(f"Warning: Failed to capture image: {e}")
+
+                    # Create EnvironmentComputeStep
+                    env_outputs = {
+                        "action_index": act_idx,
+                        "action_order": i,
+                        "error": obs_payload.get("error", None),
+                        "terminated": obs_payload.get("private", {}).terminated
+                        if hasattr(obs_payload.get("private", {}), "terminated")
+                        else False,
+                        "truncated": obs_payload.get("private", {}).truncated
+                        if hasattr(obs_payload.get("private", {}), "truncated")
+                        else False,
+                        "reward": obs_payload.get("private", {}).reward
+                        if hasattr(obs_payload.get("private", {}), "reward")
+                        else 0.0,
+                        "total_reward": obs_payload.get("private", {}).total_reward_episode
+                        if hasattr(obs_payload.get("private", {}), "total_reward_episode")
+                        else 0.0,
+                        "num_steps": obs_payload.get("public", {}).num_steps_taken
+                        if hasattr(obs_payload.get("public", {}), "num_steps_taken")
+                        else 0,
+                    }
+
+                    # Add image if captured
+                    if post_step_image:
+                        env_outputs["image_base64"] = post_step_image
+
+                    # Add player stats
+                    if hasattr(obs_payload.get("private", {}), "player_internal_stats"):
+                        stats = obs_payload["private"].player_internal_stats
+                        env_outputs["player_stats"] = {
+                            "health": stats.get("health"),
+                            "food": stats.get("food"),
+                            "drink": stats.get("drink"),
+                        }
+
+                    # Adjust event order if we have initial state
+                    event_order = i + 2 if (initial_image and turn_count == 1) else i + 1
+                    env_compute_step = EnvironmentComputeStep(
+                        event_order=event_order,
+                        compute_began=env_compute_began,
+                        compute_ended=env_compute_ended,
+                        compute_input=[ArbitraryInputs(inputs={"action": act_idx})],
+                        compute_output=[ArbitraryOutputs(outputs=env_outputs)],
+                    )
+                    environment_compute_steps.append(env_compute_step)
+
+                    if "error" in obs_payload:
+                        termination_reason = "environment_error"
+                        break
+
+                    if obs_payload["private"].terminated or obs_payload["private"].truncated:
+                        termination_reason = (
+                            "timeout" if obs_payload["private"].truncated else "death"
+                        )
+                        break
+
+                # Create Event for this turn
+                event = Event(
+                    system_instance_id=trajectory_id,
+                    event_type="turn",
+                    opened=agent_compute_began,
+                    closed=environment_compute_steps[-1].compute_ended
+                    if environment_compute_steps
+                    else agent_compute_ended,
+                    partition_index=partition_index,
+                    agent_compute_step=agent_compute_step,
+                    environment_compute_steps=environment_compute_steps,
+                    event_metadata={
+                        "turn_number": turn_count,
+                        "new_achievements": list(new_achievements),
+                        "total_achievements": len(achievements_unlocked),
+                    },
+                )
+
+                event_partition.events.append(event)
+                system_trace.partition.append(event_partition)
+                partition_index += 1
+
+                if termination_reason in ["environment_error", "timeout", "death"]:
+                    break
+
+            # Final metrics
+            if termination_reason == "unknown":
+                termination_reason = "timeout"
+
+            final_private = obs_payload.get("private")
+            final_public = obs_payload.get("public")
+
+            total_steps = (
+                final_public.num_steps_taken if hasattr(final_public, "num_steps_taken") else 0
+            )
+            total_reward = (
+                final_private.total_reward_episode
+                if hasattr(final_private, "total_reward_episode")
+                else 0.0
+            )
+
+            # Health/survival stats
+            final_health = None
+            final_food = None
+            final_drink = None
+            if hasattr(final_private, "player_internal_stats"):
+                stats = final_private.player_internal_stats
+                final_health = stats.get("health")
+                final_food = stats.get("food")
+                final_drink = stats.get("drink")
+
+            # Success determination
+            success = len(achievements_unlocked) > 0 or (
+                hasattr(final_private, "terminated") and final_private.terminated
+            )
+
+            avg_actions_per_turn = (
+                sum(actions_per_turn) / len(actions_per_turn) if actions_per_turn else 0.0
+            )
+
+            # Create RewardSignal
+            hafner_score_value = crafter_score(
+                [(achievement_turn_unlocked.get(ach, 0) > 0) * 100.0 for ach in ALL_ACHIEVEMENTS]
+            )
+
+            reward_signal = RewardSignal(
+                question_id=trajectory_id,
+                system_instance_id=trajectory_id,
+                reward=hafner_score_value,
+                annotation=f"Termination: {termination_reason}, Achievements: {len(achievements_unlocked)}/22",
+            )
+
+            # Create Dataset
+            dataset = Dataset(questions=[training_question], reward_signals=[reward_signal])
+
+            # Store trace and dataset
+            self.system_traces[trajectory_id] = system_trace
+            self.datasets[trajectory_id] = dataset
+
+            # Save to disk
+            self._save_trace_to_disk(trajectory_id, system_trace, dataset)
+
+            # Total duration
+            total_duration_sec = time.perf_counter() - start_time
+
+            # Create trajectory result
+            result = TrajectoryResult(
+                trajectory_id=trajectory_id,
+                model_name=model_name,
+                difficulty=difficulty,
+                seed=seed,
+                success=success,
+                total_steps=total_steps,
+                total_turns=turn_count,
+                total_reward=total_reward,
+                total_duration_sec=total_duration_sec,
+                achievements_unlocked=achievements_unlocked,
+                achievement_turn_unlocked=achievement_turn_unlocked,
+                actions_per_turn=actions_per_turn,
+                avg_actions_per_turn=avg_actions_per_turn,
+                termination_reason=termination_reason,
+                final_health=final_health,
+                final_food=final_food,
+                final_drink=final_drink,
+                turn_by_turn_data=turn_by_turn_data,
+            )
+
+            return result
+
+        finally:
+            pbar.close()
+
+    def _save_trace_to_disk(self, trajectory_id: str, trace: SystemTrace, dataset: Dataset):
+        """Save trace and dataset to JSON file."""
+        trace_file = self.traces_dir / f"{trajectory_id}.json"
+
+        trace_data = {
+            "trace": trace.to_dict(),
+            "dataset": dataset.to_dict(),
+            "metadata": {
+                "saved_at": datetime.now().isoformat(),
+                "trajectory_id": trajectory_id,
+            },
+        }
+
+        with open(trace_file, "w") as f:
+            json.dump(trace_data, f, indent=2)
+
+    def _save_evaluation_summary(self, report: Dict[str, Any]):
+        """Save evaluation summary and metadata."""
+        summary_file = self.output_dir / "evaluation_summary.json"
+
+        # Extract key metrics for summary
+        summary_data = {
+            "evaluation_metadata": {
+                "timestamp": datetime.now().isoformat(),
+                "output_directory": str(self.output_dir),
+                "traces_directory": str(self.traces_dir),
+                "viewer_directory": str(self.viewer_dir),
+                "num_trajectories": len(self.trajectory_results),
+            },
+            "evaluation_summary": report.get("evaluation_summary").to_dict()
+            if hasattr(report.get("evaluation_summary"), "to_dict")
+            else None,
+            "traces_info": report.get("traces"),
+            "models_evaluated": list(set(t.model_name for t in self.trajectory_results)),
+            "difficulties_evaluated": list(set(t.difficulty for t in self.trajectory_results)),
+        }
+
+        with open(summary_file, "w") as f:
+            json.dump(summary_data, f, indent=2)
+
+        # Also save the full report as CSV tables
+        if "evaluation_summary" in report and hasattr(report["evaluation_summary"], "to_csv"):
+            report["evaluation_summary"].to_csv(self.output_dir / "summary_table.csv", index=False)
+
+        if "trajectory_by_trajectory_breakdown" in report and hasattr(
+            report["trajectory_by_trajectory_breakdown"], "to_csv"
+        ):
+            report["trajectory_by_trajectory_breakdown"].to_csv(
+                self.output_dir / "trajectories.csv", index=False
+            )
+
+    async def run_evaluation(
+        self,
+        model_names: List[str],
+        difficulties: List[str] = ["easy", "hard"],
+        num_trajectories_per_condition: int = 3,
+        max_turns: int = 30,
+        collect_detailed_data: bool = True,
+    ) -> Dict[str, Any]:
+        """Run comprehensive evaluation with trace capture."""
+
+        print(f"🎯 Starting Full Enchilada Crafter Evaluation")
+        print(f"   Models: {model_names}")
+        print(f"   Difficulties: {difficulties}")
+        print(f"   Trajectories per condition: {num_trajectories_per_condition}")
+        print(f"   Max turns per trajectory: {max_turns}")
+        print(f"   Output directory: {self.output_dir}")
+
+        all_results = []
+
+        for model_name in model_names:
+            for difficulty in difficulties:
+                print(f"\n🔄 Running {model_name} on {difficulty} difficulty...")
+
+                # Run trajectories for this condition
+                trajectory_tasks = []
+                for i in range(num_trajectories_per_condition):
+                    seed = 1000 + i if difficulty == "easy" else 2000 + i
+                    trajectory_tasks.append(
+                        self.run_single_trajectory_with_trace(
+                            model_name=model_name,
+                            difficulty=difficulty,
+                            seed=seed,
+                            max_turns=max_turns,
+                            collect_detailed_data=collect_detailed_data,
+                        )
+                    )
+
+                condition_results = await asyncio.gather(*trajectory_tasks)
+                all_results.extend(condition_results)
+
+        self.trajectory_results = all_results
+
+        # Generate report
+        report = self._generate_comprehensive_report()
+
+        # Create viewer files
+        self._create_viewer_files()
+
+        # Add trace info to report
+        report["traces"] = {
+            "count": len(self.system_traces),
+            "directory": str(self.traces_dir),
+            "viewer_url": f"http://localhost:8999",
+        }
+
+        # Save evaluation summary
+        self._save_evaluation_summary(report)
+
+        return report
+
+    def _create_viewer_files(self):
+        """Create the viewer HTML/JS/CSS files."""
+
+        # Create index.html
+        html_content = """<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Crafter Evaluation Viewer</title>
+    <link rel="stylesheet" href="style.css">
+</head>
+<body>
+    <div id="app">
+        <div class="header">
+            <h1>🎮 Crafter Evaluation Viewer</h1>
+            <div class="trace-selector">
+                <label for="trace-select">Select Trace:</label>
+                <select id="trace-select"></select>
+                <button id="refresh-btn">🔄 Refresh</button>
+            </div>
+        </div>
+        
+        <div class="main-container">
+            <div class="sidebar">
+                <h2>Timeline</h2>
+                <div id="timeline" class="timeline"></div>
+                
+                <div class="trace-info">
+                    <h3>Trace Info</h3>
+                    <div id="trace-metadata"></div>
+                </div>
+            </div>
+            
+            <div class="content">
+                <div class="question-reward">
+                    <h2>Training Question</h2>
+                    <div id="question-display" class="info-box"></div>
+                    
+                    <h2>Reward Signal</h2>
+                    <div id="reward-display" class="info-box"></div>
+                </div>
+                
+                <div class="turn-details">
+                    <h2>Turn Details</h2>
+                    <div id="turn-content">
+                        <p class="placeholder">Select a turn from the timeline</p>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+    
+    <script src="viewer.js"></script>
+</body>
+</html>"""
+
+        # Create style.css
+        css_content = """* {
+    margin: 0;
+    padding: 0;
+    box-sizing: border-box;
+}
+
+body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+    background-color: #f5f5f5;
+    color: #333;
+}
+
+.header {
+    background-color: #2c3e50;
+    color: white;
+    padding: 1rem 2rem;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+}
+
+.header h1 {
+    font-size: 1.5rem;
+}
+
+.trace-selector {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.trace-selector select {
+    padding: 0.5rem;
+    border-radius: 4px;
+    border: 1px solid #34495e;
+    background-color: white;
+    min-width: 200px;
+}
+
+.trace-selector button {
+    padding: 0.5rem 1rem;
+    background-color: #3498db;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    transition: background-color 0.2s;
+}
+
+.trace-selector button:hover {
+    background-color: #2980b9;
+}
+
+.main-container {
+    display: flex;
+    height: calc(100vh - 60px);
+}
+
+.sidebar {
+    width: 300px;
+    background-color: white;
+    border-right: 1px solid #ddd;
+    overflow-y: auto;
+    padding: 1rem;
+}
+
+.timeline {
+    margin-bottom: 2rem;
+}
+
+.timeline-item {
+    padding: 0.75rem;
+    margin-bottom: 0.5rem;
+    background-color: #f8f9fa;
+    border-radius: 4px;
+    cursor: pointer;
+    transition: all 0.2s;
+    border: 2px solid transparent;
+}
+
+.timeline-item:hover {
+    background-color: #e9ecef;
+}
+
+.timeline-item.active {
+    background-color: #3498db;
+    color: white;
+    border-color: #2980b9;
+}
+
+.timeline-item .turn-number {
+    font-weight: bold;
+    margin-bottom: 0.25rem;
+}
+
+.timeline-item .turn-stats {
+    font-size: 0.85rem;
+    opacity: 0.8;
+}
+
+.trace-info {
+    padding: 1rem;
+    background-color: #f8f9fa;
+    border-radius: 4px;
+}
+
+.trace-info h3 {
+    margin-bottom: 0.5rem;
+    color: #2c3e50;
+}
+
+.content {
+    flex: 1;
+    padding: 2rem;
+    overflow-y: auto;
+}
+
+.question-reward {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 2rem;
+    margin-bottom: 2rem;
+}
+
+.info-box {
+    padding: 1rem;
+    background-color: white;
+    border-radius: 4px;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+}
+
+.turn-details {
+    background-color: white;
+    border-radius: 4px;
+    padding: 2rem;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+}
+
+.turn-details h2 {
+    margin-bottom: 1rem;
+    color: #2c3e50;
+}
+
+.placeholder {
+    color: #999;
+    text-align: center;
+    padding: 3rem;
+}
+
+.agent-section, .environment-section {
+    margin-bottom: 2rem;
+}
+
+.agent-section h3, .environment-section h3 {
+    color: #2c3e50;
+    margin-bottom: 1rem;
+}
+
+.message-box {
+    background-color: #f8f9fa;
+    padding: 1rem;
+    border-radius: 4px;
+    margin-bottom: 1rem;
+}
+
+.message-box .role {
+    font-weight: bold;
+    color: #2c3e50;
+    margin-bottom: 0.5rem;
+}
+
+.actions-box {
+    background-color: #e3f2fd;
+    padding: 1rem;
+    border-radius: 4px;
+    margin-bottom: 1rem;
+    font-family: monospace;
+}
+
+.env-step {
+    background-color: #f8f9fa;
+    padding: 1rem;
+    border-radius: 4px;
+    margin-bottom: 1rem;
+    border-left: 4px solid #3498db;
+}
+
+.env-step h4 {
+    color: #2c3e50;
+    margin-bottom: 0.5rem;
+}
+
+.env-image {
+    width: 256px;
+    height: 256px;
+    border-radius: 4px;
+    margin-top: 1rem;
+    image-rendering: pixelated;
+    border: 2px solid #ddd;
+}
+
+.stats-grid {
+    display: inline-flex;
+    gap: 1rem;
+    margin-top: 0.5rem;
+    margin-bottom: 0.5rem;
+}
+
+.stat-item {
+    background-color: white;
+    padding: 0.25rem 0.5rem;
+    border-radius: 3px;
+    text-align: center;
+    border: 1px solid #e0e0e0;
+    font-size: 0.75rem;
+}
+
+.stat-label {
+    font-size: 0.7rem;
+    color: #666;
+    margin-right: 0.25rem;
+}
+
+.stat-value {
+    font-size: 0.8rem;
+    font-weight: bold;
+    color: #2c3e50;
+    display: inline;
+}
+
+.metadata-item {
+    margin-bottom: 0.5rem;
+}
+
+.metadata-label {
+    font-weight: bold;
+    color: #666;
+}
+
+.achievement-badge {
+    display: inline-block;
+    background-color: #27ae60;
+    color: white;
+    padding: 0.25rem 0.5rem;
+    border-radius: 3px;
+    font-size: 0.85rem;
+    margin-right: 0.5rem;
+    margin-bottom: 0.5rem;
+}
+
+.images-row {
+    display: flex;
+    gap: 1rem;
+    overflow-x: auto;
+    padding: 1rem 0;
+}
+
+.image-container {
+    text-align: center;
+    flex-shrink: 0;
+}
+
+.image-caption {
+    margin-top: 0.5rem;
+    font-size: 0.85rem;
+    color: #666;
+    font-family: monospace;
+}
+
+@media (max-width: 768px) {
+    .main-container {
+        flex-direction: column;
+    }
+    
+    .sidebar {
+        width: 100%;
+        height: 200px;
+        border-right: none;
+        border-bottom: 1px solid #ddd;
+    }
+    
+    .question-reward {
+        grid-template-columns: 1fr;
+    }
+}"""
+
+        # Create viewer.js
+        js_content = """let currentTrace = null;
+let currentTurnIndex = null;
+
+// Action mapping
+const ACTION_NAMES = {
+    -1: 'initial_state',
+    0: 'noop',
+    1: 'move_left',
+    2: 'move_right',
+    3: 'move_up',
+    4: 'move_down',
+    5: 'do',
+    6: 'sleep',
+    7: 'place_stone',
+    8: 'place_table',
+    9: 'place_furnace',
+    10: 'place_plant',
+    11: 'make_wood_pickaxe',
+    12: 'make_stone_pickaxe',
+    13: 'make_iron_pickaxe',
+    14: 'make_wood_sword',
+    15: 'make_stone_sword',
+    16: 'make_iron_sword'
+};
+
+// Load available traces
+async function loadTraceList() {
+    try {
+        const response = await fetch('/api/traces');
+        const traces = await response.json();
+        
+        const select = document.getElementById('trace-select');
+        select.innerHTML = '';
+        
+        traces.forEach(trace => {
+            const option = document.createElement('option');
+            option.value = trace.id;
+            option.textContent = `${trace.model_name} - ${trace.difficulty} - ${trace.id.substring(0, 8)}`;
+            select.appendChild(option);
+        });
+        
+        if (traces.length > 0) {
+            loadTrace(traces[0].id);
+        }
+    } catch (error) {
+        console.error('Failed to load traces:', error);
+    }
+}
+
+// Load specific trace
+async function loadTrace(traceId) {
+    try {
+        const response = await fetch(`/api/trace/${traceId}`);
+        const data = await response.json();
+        
+        currentTrace = data;
+        currentTurnIndex = null;
+        
+        displayTraceInfo();
+        displayTimeline();
+        displayQuestionAndReward();
+        clearTurnDetails();
+    } catch (error) {
+        console.error('Failed to load trace:', error);
+    }
+}
+
+// Display trace metadata
+function displayTraceInfo() {
+    const metadataDiv = document.getElementById('trace-metadata');
+    const metadata = currentTrace.trace.metadata;
+    
+    metadataDiv.innerHTML = `
+        <div class="metadata-item">
+            <span class="metadata-label">Model:</span> ${metadata.model_name}
+        </div>
+        <div class="metadata-item">
+            <span class="metadata-label">Difficulty:</span> ${metadata.difficulty}
+        </div>
+        <div class="metadata-item">
+            <span class="metadata-label">Seed:</span> ${metadata.seed}
+        </div>
+        <div class="metadata-item">
+            <span class="metadata-label">Max Turns:</span> ${metadata.max_turns}
+        </div>
+    `;
+}
+
+// Display timeline
+function displayTimeline() {
+    const timeline = document.getElementById('timeline');
+    timeline.innerHTML = '';
+    
+    currentTrace.trace.partition.forEach((partition, index) => {
+        const event = partition.events[0];
+        const metadata = event.event_metadata;
+        
+        const item = document.createElement('div');
+        item.className = 'timeline-item';
+        item.innerHTML = `
+            <div class="turn-number">Turn ${metadata.turn_number}</div>
+            <div class="turn-stats">
+                Actions: ${event.environment_compute_steps.length} | 
+                Achievements: ${metadata.total_achievements}
+                ${metadata.new_achievements.length > 0 ? ' (+' + metadata.new_achievements.length + ')' : ''}
+            </div>
+        `;
+        
+        item.addEventListener('click', () => selectTurn(index));
+        timeline.appendChild(item);
+    });
+}
+
+// Display question and reward
+function displayQuestionAndReward() {
+    const question = currentTrace.dataset.questions[0];
+    const reward = currentTrace.dataset.reward_signals[0];
+    
+    document.getElementById('question-display').innerHTML = `
+        <p><strong>Intent:</strong> ${question.intent}</p>
+        <p><strong>Criteria:</strong> ${question.criteria}</p>
+    `;
+    
+    document.getElementById('reward-display').innerHTML = `
+        <p><strong>Hafner Score:</strong> ${reward.reward.toFixed(2)}%</p>
+        <p><strong>Annotation:</strong> ${reward.annotation}</p>
+    `;
+}
+
+// Select turn
+function selectTurn(index) {
+    currentTurnIndex = index;
+    
+    // Update timeline selection
+    document.querySelectorAll('.timeline-item').forEach((item, i) => {
+        item.classList.toggle('active', i === index);
+    });
+    
+    // Display turn details
+    displayTurnDetails();
+}
+
+// Display turn details - SIMPLIFIED VERSION
+function displayTurnDetails() {
+    if (currentTurnIndex === null) return;
+    
+    const partition = currentTrace.trace.partition[currentTurnIndex];
+    const event = partition.events[0];
+    const agentStep = event.agent_compute_step;
+    const envSteps = event.environment_compute_steps;
+    
+    let html = '';
+    
+    // Display actions planned
+    if (agentStep.compute_output[0] && agentStep.compute_output[0].outputs) {
+        const outputs = agentStep.compute_output[0].outputs;
+        const actionNames = outputs.actions.map(idx => `${ACTION_NAMES[idx] || 'unknown'}`);
+        html += `
+            <div class="actions-box" style="margin-bottom: 1.5rem;">
+                <strong>Turn ${event.event_metadata.turn_number} Actions:</strong> ${actionNames.join(' → ')}
+            </div>
+        `;
+    }
+    
+    // Display all images in a row
+    html += '<div class="images-row">';
+    envSteps.forEach((step, i) => {
+        const outputs = step.compute_output[0].outputs;
+        const actionName = ACTION_NAMES[outputs.action_index] || 'unknown';
+        
+        if (outputs.image_base64) {
+            // For initial state, show "0. initial state", otherwise show action number
+            const stepNumber = outputs.action_index === -1 ? 0 : i;
+            html += `
+                <div class="image-container">
+                    <img src="data:image/png;base64,${outputs.image_base64}" class="env-image" alt="Game state">
+                    <div class="image-caption">${stepNumber}. ${actionName}</div>
+                </div>
+            `;
+        }
+    });
+    html += '</div>';
+    
+    // New achievements
+    if (event.event_metadata.new_achievements.length > 0) {
+        html += '<div class="achievements-section" style="margin-top: 1rem;">';
+        html += '<strong>New achievements: </strong>';
+        event.event_metadata.new_achievements.forEach(ach => {
+            html += `<span class="achievement-badge">${ach}</span>`;
+        });
+        html += '</div>';
+    }
+    
+    document.getElementById('turn-content').innerHTML = html;
+}
+
+// Clear turn details
+function clearTurnDetails() {
+    document.getElementById('turn-content').innerHTML = '<p class="placeholder">Select a turn from the timeline</p>';
+}
+
+// Event listeners
+document.getElementById('trace-select').addEventListener('change', (e) => {
+    if (e.target.value) {
+        loadTrace(e.target.value);
+    }
+});
+
+document.getElementById('refresh-btn').addEventListener('click', () => {
+    loadTraceList();
+});
+
+// Initial load
+loadTraceList();"""
+
+        # Save files
+        with open(self.viewer_dir / "index.html", "w") as f:
+            f.write(html_content)
+
+        with open(self.viewer_dir / "style.css", "w") as f:
+            f.write(css_content)
+
+        with open(self.viewer_dir / "viewer.js", "w") as f:
+            f.write(js_content)
+
+
+# Global variable to store current eval directory
+_current_eval_dir = None
+
+
+def set_current_eval_dir(eval_dir: Path):
+    """Set the current evaluation directory for the viewer."""
+    global _current_eval_dir
+    _current_eval_dir = eval_dir
+
+
+# FastAPI app for viewer
+app = FastAPI()
+
+
+@app.get("/api/traces")
+async def get_traces():
+    """Get list of available traces."""
+    global _current_eval_dir
+    if _current_eval_dir is None:
+        return []
+
+    traces_dir = _current_eval_dir / "traces"
+    if not traces_dir.exists():
+        return []
+
+    traces = []
+    for trace_file in traces_dir.glob("*.json"):
+        try:
+            with open(trace_file, "r") as f:
+                data = json.load(f)
+                trace_meta = data["trace"]["metadata"]
+                traces.append(
+                    {
+                        "id": trace_file.stem,
+                        "model_name": trace_meta["model_name"],
+                        "difficulty": trace_meta["difficulty"],
+                        "seed": trace_meta["seed"],
+                    }
+                )
+        except Exception as e:
+            print(f"Error loading trace {trace_file}: {e}")
+
+    return sorted(traces, key=lambda x: x["id"])
+
+
+@app.get("/api/trace/{trace_id}")
+async def get_trace(trace_id: str):
+    """Get specific trace data."""
+    global _current_eval_dir
+    if _current_eval_dir is None:
+        raise HTTPException(status_code=404, detail="No evaluation directory set")
+
+    trace_file = _current_eval_dir / "traces" / f"{trace_id}.json"
+    if not trace_file.exists():
+        raise HTTPException(status_code=404, detail="Trace not found")
+
+    with open(trace_file, "r") as f:
+        return json.load(f)
+
+
+@app.get("/api/eval_info")
+async def get_eval_info():
+    """Get evaluation metadata."""
+    global _current_eval_dir
+    if _current_eval_dir is None:
+        return {"error": "No evaluation directory set"}
+
+    summary_file = _current_eval_dir / "evaluation_summary.json"
+    if summary_file.exists():
+        with open(summary_file, "r") as f:
+            return json.load(f)
+    return {"error": "No evaluation summary found"}
+
+
+# Convenience function for running evaluation
+async def run_full_crafter_eval(
+    model_names: List[str],
+    difficulties: List[str] = ["easy", "hard"],
+    num_trajectories: int = 3,
+    max_turns: int = 30,
+    capture_images: bool = True,
+    launch_viewer: bool = True,
+    output_dir: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Run full Crafter evaluation with traces and viewer."""
+
+    framework = FullCrafterEvalFramework(capture_images=capture_images, output_dir=output_dir)
+    report = await framework.run_evaluation(
+        model_names=model_names,
+        difficulties=difficulties,
+        num_trajectories_per_condition=num_trajectories,
+        max_turns=max_turns,
+    )
+
+    framework.print_report(report)
+
+    if launch_viewer:
+        print(f"\n📁 Evaluation saved to: {framework.output_dir}")
+        print("🌐 Launching viewer at http://localhost:8999")
+        print("   Press Ctrl+C to stop the viewer")
+
+        # Set the current eval directory for the viewer
+        set_current_eval_dir(framework.output_dir)
+
+        # Mount static files from the viewer directory
+        app.mount(
+            "/",
+            StaticFiles(directory=str(framework.viewer_dir), html=True),
+            name="viewer",
+        )
+
+        # Run viewer
+        config = uvicorn.Config(app, host="0.0.0.0", port=8999, log_level="error")
+        server = uvicorn.Server(config)
+        await server.serve()
+
+    return report
+
+
+if __name__ == "__main__":
+    # Example usage
+    async def main():
+        await run_full_crafter_eval(
+            model_names=["gpt-4.1-mini"],
+            difficulties=["easy"],
+            num_trajectories=10,
+            max_turns=50,
+            capture_images=True,
+            launch_viewer=True,
+        )
+
+    asyncio.run(main())