synth-ai 0.2.0__py3-none-any.whl → 0.2.1.dev0__py3-none-any.whl

synth_ai/environments/examples/nethack/agent_demos/nethack_evaluation_framework.py

@@ -0,0 +1,981 @@
+"""
+NetHack Evaluation Framework
+============================
+Provides detailed metrics, trajectory analysis, and achievement statistics for NetHack.
+Mirrors the Crafter evaluation structure but adapted for NetHack specifics.
+"""
+
+import asyncio
+import json
+import time
+import math
+import uuid
+import os
+from dataclasses import dataclass, asdict
+from typing import Dict, List, Optional, Set, Tuple, Any
+from collections import defaultdict
+
+import pandas as pd
+from tqdm import tqdm
+
+# NetHack specific imports
+from src.synth_env.examples.nethack.environment import NetHackEnvironment
+from src.synth_env.examples.nethack.taskset import (
+    NetHackTaskInstance,
+    NetHackTaskInstanceMetadata,
+)
+from src.synth_env.examples.nethack.agent_demos.test_synth_react import (
+    NetHackReActAgent,
+)
+from src.synth_env.examples.nethack.achievements import NetHackAchievements
+from src.synth_env.examples.nethack.engine import NetHackObservationCallable
+from src.synth_env.tasks.core import Impetus, Intent
+from synth_ai.zyk import LM
+
+# Load achievements mapping for BALROG scoring
+_achievements_path = os.path.join(os.path.dirname(__file__), "..", "helpers", "achievements.json")
+with open(_achievements_path, "r") as f:
+    BALROG_ACHIEVEMENTS = json.load(f)["3.4.3"]
+
+# Achievement categories based on difficulty/complexity
+ACHIEVEMENT_CATEGORIES = {
+    "basic": [
+        "first_kill",
+        "first_spell_cast",
+        "first_prayer",
+        "survived_100_turns",
+        "reached_dlvl_2",
+        "reached_dlvl_5",
+        "killed_10_monsters",
+    ],
+    "intermediate": [
+        "reached_dlvl_10",
+        "reached_dlvl_20",
+        "killed_50_monsters",
+        "killed_100_monsters",
+        "collected_1000_gold",
+        "reached_level_5",
+        "reached_level_10",
+        "reached_minetown",
+    ],
+    "advanced": [
+        "reached_dlvl_30",
+        "reached_castle",
+        "got_quest",
+        "completed_quest",
+        "reached_level_20",
+        "collected_10000_gold",
+        "found_artifact",
+        "reached_mines_end",
+    ],
+}
+
+# Get all achievements from NetHackAchievements
+_sample_achievements = NetHackAchievements()
+ALL_ACHIEVEMENTS = list(_sample_achievements.get_unlocked_achievements().keys())
+
+TERMINATION_REASONS = ["timeout", "death", "agent_quit", "environment_error"]
+
+# SOTA scores (NetHack doesn't have published Hafner scores, only BALROG)
+BALROG_SOTA_SCORES = {
+    "balrog_leaderboard": {
+        # TODO: Add real BALROG leaderboard scores when available
+        "Claude 3.5 Sonnet": 25.0,  # Placeholder
+        "GPT-4o": 20.0,  # Placeholder
+        "GPT-4o-mini": 15.0,  # Placeholder
+        "Gemini 1.5 Flash": 12.0,  # Placeholder
+    }
+}
+
+# Model name mapping for SOTA percentage calculations
+MODEL_NAME_TO_SOTA = {
+    "claude-3-5-sonnet-latest": "Claude 3.5 Sonnet",
+    "gpt-4o": "GPT-4o",
+    "gpt-4o-mini": "GPT-4o-mini",
+    "gemini-1.5-flash": "Gemini 1.5 Flash",
+    "gemini-1.5-flash-latest": "Gemini 1.5 Flash",
+}
+
+
+def hafner_score(success_rates_percent: List[float]) -> float:
+    """Compute the Hafner adjusted score (log-mean) for NetHack."""
+    if not success_rates_percent:
+        return 0.0
+    N = len(success_rates_percent)
+    g = sum(math.log(1 + s) for s in success_rates_percent) / N
+    return math.exp(g) - 1
+
+
+def balrog_score_simple(percent: float) -> float:
+    """BALROG score is already a percentage (0-100)."""
+    return percent
+
+
+@dataclass
+class TrajectoryResult:
+    """Results from a single NetHack trajectory/episode."""
+
+    trajectory_id: str
+    model_name: str
+    difficulty: str
+    seed: int
+
+    # Core metrics
+    success: bool
+    total_steps: int
+    total_turns: int
+    total_reward: float
+
+    # Achievement tracking
+    achievements_unlocked: Set[str]
+    achievement_turn_unlocked: Dict[str, int]
+
+    # Multi-action metrics (if applicable)
+    actions_per_turn: List[int]
+    avg_actions_per_turn: float
+
+    # Termination analysis
+    termination_reason: str
+    final_depth: Optional[int]
+    final_level: Optional[int]
+    final_gold: Optional[int]
+
+    # BALROG scoring
+    balrog_percent: float
+
+    # Trajectory data for detailed analysis
+    turn_by_turn_data: Optional[List[Dict[str, Any]]] = None
+
+
+@dataclass
+class AggregateResults:
+    """Aggregate results across multiple NetHack trajectories."""
+
+    model_name: str
+    difficulty: str
+    num_trajectories: int
+
+    # Success metrics
+    success_rate: float
+    avg_total_steps: float
+    avg_total_turns: float
+    avg_total_reward: float
+
+    # Achievement metrics
+    unique_achievements_unlocked: Set[str]
+    total_achievement_count: int
+    avg_achievements_per_trajectory: float
+    achievement_unlock_rates: Dict[str, float]
+    hafner_score: float
+    balrog_score_avg: float
+    balrog_score_best: float
+
+    # Multi-action metrics
+    avg_actions_per_turn_overall: float
+    actions_per_turn_distribution: Dict[int, int]
+
+    # Termination analysis
+    termination_breakdown: Dict[str, float]
+    avg_final_depth: Optional[float]
+    avg_final_level: Optional[float]
+    avg_final_gold: Optional[float]
+
+
+class NetHackEvalFramework:
+    """Standardized evaluation framework for NetHack environments."""
+
+    def __init__(self):
+        self.trajectory_results: List[TrajectoryResult] = []
+
+    async def run_single_trajectory(
+        self,
+        model_name: str,
+        difficulty: str,
+        seed: int,
+        max_turns: int = 200,
+        collect_detailed_data: bool = True,
+    ) -> TrajectoryResult:
+        """Run a single NetHack trajectory and collect detailed metrics."""
+
+        # Create task instance
+        metadata = NetHackTaskInstanceMetadata(
+            character_role="knight",  # Default role
+            starting_level=1,
+            target_depth=5 if difficulty == "easy" else 10,
+            time_limit=max_turns * 10,  # Generous time limit
+            difficulty=difficulty,
+            special_objectives=[
+                "Survive for as long as possible",
+                "Collect gold",
+                "Kill monsters",
+            ],
+            seed=seed,
+        )
+        instance = NetHackTaskInstance(
+            id=uuid.uuid4(),
+            impetus=Impetus(
+                instructions=f"Explore the NetHack dungeon on {difficulty} difficulty. Survive as long as possible, kill monsters, collect items, and descend to deeper levels."
+            ),
+            intent=Intent(rubric={}, gold_trajectories=None, gold_state_diff={}),
+            metadata=metadata,
+            is_reproducible=True,
+            initial_engine_snapshot=None,
+        )
+
+        # Setup environment and agent
+        obs_callback = NetHackObservationCallable()
+        env = NetHackEnvironment(instance, custom_step_obs=obs_callback)
+
+        llm = LM(model_name=model_name, formatting_model_name=model_name, temperature=0.0)
+        agent = NetHackReActAgent(llm, max_turns=max_turns)
+
+        # Set system prompt for agent
+        task_instructions = instance.impetus.instructions
+        agent.system_prompt = agent._create_system_prompt(task_instructions)
+
+        # Initialize tracking
+        trajectory_id = str(uuid.uuid4())
+        achievements = NetHackAchievements()
+        achievements_unlocked = set()
+        achievement_turn_unlocked = {}
+        actions_per_turn = []
+        turn_by_turn_data = [] if collect_detailed_data else None
+
+        # Progress tracking for BALROG score
+        class BalrogProgress:
+            def __init__(self):
+                self.percent = 0.0
+                self.end_reason = None
+
+            def update(self, depth: int, level: int, done: bool = False, end_reason: str = ""):
+                # Simple progress based on depth and level
+                depth_score = min(depth * 2, 50)  # Max 50 from depth
+                level_score = min(level * 3, 50)  # Max 50 from level
+                self.percent = max(depth_score, level_score)
+                if done:
+                    self.end_reason = end_reason
+
+        balrog_progress = BalrogProgress()
+
+        # Run episode
+        obs_payload = await env.initialize()
+        turn_count = 0
+        termination_reason = "unknown"
+
+        # Create progress bar
+        pbar = tqdm(
+            total=max_turns,
+            desc=f"{model_name} ({difficulty}) Seed {seed}",
+            unit="turn",
+            leave=False,
+            ncols=100,
+        )
+
+        try:
+            while turn_count < max_turns:
+                turn_count += 1
+                pbar.update(1)
+
+                # Extract stats from observation for progress tracking
+                if "formatted_obs" in obs_payload:
+                    current_formatted_obs = obs_payload["formatted_obs"]
+                elif "message" in obs_payload:
+                    # Format the observation for the agent
+                    current_formatted_obs = f"""
+=== NetHack Observation ===
+Message: {obs_payload.get("message", "")}
+Map:
+{obs_payload.get("ascii_map", "")}
+
+Stats: {obs_payload.get("player_stats", {})}
+Inventory: {obs_payload.get("inventory", [])}
+In Menu: {obs_payload.get("in_menu", False)}
+"""
+                else:
+                    # Fallback to string representation
+                    current_formatted_obs = str(obs_payload)
+
+                # Update achievements (simplified - would need real obs parsing)
+                prev_achievements = achievements_unlocked.copy()
+
+                # Extract game state for BALROG scoring
+                try:
+                    # Parse the actual game state from obs
+                    player_stats = obs_payload.get("player_stats", {})
+                    current_depth = player_stats.get("depth", 1)
+                    current_level = player_stats.get("experience_level", 1)
+                    balrog_progress.update(current_depth, current_level)
+                except:
+                    current_depth = 1
+                    current_level = 1
+                    balrog_progress.update(current_depth, current_level)
+
+                # Update progress bar
+                easy_count = len(
+                    [a for a in achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["basic"]]
+                )
+                inter_count = len(
+                    [
+                        a
+                        for a in achievements_unlocked
+                        if a in ACHIEVEMENT_CATEGORIES["intermediate"]
+                    ]
+                )
+                adv_count = len(
+                    [a for a in achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["advanced"]]
+                )
+                total_count = len(achievements_unlocked)
+                achievement_display = f"{total_count}({easy_count}/{inter_count}/{adv_count})"
+
+                pbar.set_postfix(
+                    {
+                        "achievements": achievement_display,
+                        "balrog": f"{balrog_progress.percent:.1f}%",
+                    }
+                )
+
+                # Agent decision
+                decision = await agent.decide(current_formatted_obs)
+
+                # Check for termination - NetHack agent uses different format
+                if isinstance(decision, dict):
+                    # Handle tool call format: {'name': 'tool_name', 'parameters': {...}}
+                    if decision.get("name") == "terminate":
+                        termination_reason = "agent_quit"
+                        break
+
+                    # Extract actions from NetHack agent response
+                    if "parameters" in decision and isinstance(decision["parameters"], dict):
+                        params = decision["parameters"]
+                        if "actions" in params:
+                            actions = params["actions"]
+                        elif "action" in params:
+                            actions = [params["action"]]
+                        else:
+                            actions = ["wait"]  # Default action
+                    elif "actions" in decision:
+                        actions = decision["actions"]
+                    elif "action" in decision:
+                        actions = [decision["action"]]
+                    else:
+                        actions = ["wait"]  # Default action
+                else:
+                    # If decision is not a dict, assume it's a single action or termination
+                    if decision == -1 or decision == [-1]:
+                        termination_reason = "agent_quit"
+                        break
+                    elif isinstance(decision, list):
+                        actions = decision
+                    else:
+                        actions = [str(decision)]
+
+                if not isinstance(actions, list):
+                    actions = [str(actions)]
+
+                actions_per_turn.append(len(actions))
+
+                # Collect turn data
+                if collect_detailed_data:
+                    turn_data = {
+                        "turn": turn_count,
+                        "actions_planned": len(actions),
+                        "achievements_at_start": list(achievements_unlocked),
+                        "balrog_percent": balrog_progress.percent,
+                    }
+                    turn_by_turn_data.append(turn_data)
+
+                # Execute actions
+                for action in actions:
+                    obs_payload = await env.step(action)
+
+                    # Check for REAL environment errors (not NetHack game messages)
+                    if "error" in obs_payload:
+                        error_msg = obs_payload["error"]
+                        # NetHack game messages like "No stairs here" are normal, not environment errors
+                        if error_msg and not any(
+                            phrase in error_msg.lower()
+                            for phrase in [
+                                "no stairs",
+                                "can't go",
+                                "there is nothing",
+                                "you can't",
+                                "you don't",
+                                "you aren't",
+                                "you have no",
+                                "invalid action",
+                                "stairs here to",
+                                "can't",
+                                "there's nothing",
+                                "no door",
+                            ]
+                        ):
+                            print(f"   ⚠️ Real environment error: {error_msg}")
+                            termination_reason = "environment_error"
+                            break
+                        # This is just a NetHack game message, continue playing
+
+                    # Check termination status
+                    private_state = obs_payload.get("private")
+                    if private_state:
+                        if getattr(private_state, "terminated", False) or getattr(
+                            private_state, "truncated", False
+                        ):
+                            termination_reason = (
+                                "timeout" if getattr(private_state, "truncated", False) else "death"
+                            )
+                            balrog_progress.update(
+                                current_depth,
+                                current_level,
+                                done=True,
+                                end_reason=termination_reason,
+                            )
+                            break
+
+                if termination_reason in ["environment_error", "timeout", "death"]:
+                    break
+
+            # Final metrics
+            if termination_reason == "unknown":
+                termination_reason = "timeout"
+
+            final_private = obs_payload.get("private")
+            final_public = obs_payload.get("public")
+
+            total_steps = getattr(final_public, "step_count", turn_count)
+            total_reward = getattr(final_private, "total_reward", 0.0)
+
+            # Final stats from player_stats
+            player_stats = obs_payload.get("player_stats", {})
+            final_depth = player_stats.get("depth", current_depth)
+            final_level = player_stats.get("experience_level", current_level)
+            final_gold = player_stats.get("gold", 0)
+
+            # Success determination
+            success = len(achievements_unlocked) > 0 or balrog_progress.percent > 5.0
+
+            avg_actions_per_turn = (
+                sum(actions_per_turn) / len(actions_per_turn) if actions_per_turn else 0.0
+            )
+
+            return TrajectoryResult(
+                trajectory_id=trajectory_id,
+                model_name=model_name,
+                difficulty=difficulty,
+                seed=seed,
+                success=success,
+                total_steps=total_steps,
+                total_turns=turn_count,
+                total_reward=total_reward,
+                achievements_unlocked=achievements_unlocked,
+                achievement_turn_unlocked=achievement_turn_unlocked,
+                actions_per_turn=actions_per_turn,
+                avg_actions_per_turn=avg_actions_per_turn,
+                termination_reason=termination_reason,
+                final_depth=final_depth,
+                final_level=final_level,
+                final_gold=final_gold,
+                balrog_percent=balrog_progress.percent,
+                turn_by_turn_data=turn_by_turn_data,
+            )
+        finally:
+            pbar.close()
+
+    async def run_evaluation(
+        self,
+        model_names: List[str],
+        difficulties: List[str] = ["easy", "hard"],
+        num_trajectories_per_condition: int = 3,
+        max_turns: int = 200,
+        collect_detailed_data: bool = True,
+    ) -> Dict[str, Any]:
+        """Run comprehensive evaluation across models and difficulties."""
+
+        print(f"🎯 Starting NetHack Evaluation")
+        print(f"   Models: {model_names}")
+        print(f"   Difficulties: {difficulties}")
+        print(f"   Trajectories per condition: {num_trajectories_per_condition}")
+        print(f"   Max turns per trajectory: {max_turns}")
+
+        all_results = []
+
+        for model_name in model_names:
+            for difficulty in difficulties:
+                print(f"\n🔄 Running {model_name} on {difficulty} difficulty...")
+
+                # Run trajectories for this condition
+                trajectory_tasks = []
+                for i in range(num_trajectories_per_condition):
+                    seed = 1000 + i if difficulty == "easy" else 2000 + i
+                    trajectory_tasks.append(
+                        self.run_single_trajectory(
+                            model_name=model_name,
+                            difficulty=difficulty,
+                            seed=seed,
+                            max_turns=max_turns,
+                            collect_detailed_data=collect_detailed_data,
+                        )
+                    )
+
+                condition_results = await asyncio.gather(*trajectory_tasks)
+                all_results.extend(condition_results)
+
+        self.trajectory_results = all_results
+        return self._generate_comprehensive_report()
+
+    def _compute_aggregate_metrics(
+        self, model_name: str, difficulty: str, trajectories: List[TrajectoryResult]
+    ) -> AggregateResults:
+        """Compute aggregate metrics for a model-difficulty condition."""
+
+        num_trajectories = len(trajectories)
+        if num_trajectories == 0:
+            return AggregateResults(
+                model_name=model_name,
+                difficulty=difficulty,
+                num_trajectories=0,
+                success_rate=0.0,
+                avg_total_steps=0.0,
+                avg_total_turns=0.0,
+                avg_total_reward=0.0,
+                unique_achievements_unlocked=set(),
+                total_achievement_count=0,
+                avg_achievements_per_trajectory=0.0,
+                achievement_unlock_rates={},
+                hafner_score=0.0,
+                balrog_score_avg=0.0,
+                balrog_score_best=0.0,
+                avg_actions_per_turn_overall=0.0,
+                actions_per_turn_distribution={},
+                termination_breakdown={},
+                avg_final_depth=None,
+                avg_final_level=None,
+                avg_final_gold=None,
+            )
+
+        # Success metrics
+        success_rate = sum(1 for t in trajectories if t.success) / num_trajectories
+        avg_total_steps = sum(t.total_steps for t in trajectories) / num_trajectories
+        avg_total_turns = sum(t.total_turns for t in trajectories) / num_trajectories
+        avg_total_reward = sum(t.total_reward for t in trajectories) / num_trajectories
+
+        # Achievement analysis
+        all_achievements = set()
+        total_achievement_count = 0
+        achievement_counts = defaultdict(int)
+
+        for traj in trajectories:
+            all_achievements.update(traj.achievements_unlocked)
+            total_achievement_count += len(traj.achievements_unlocked)
+            for ach in traj.achievements_unlocked:
+                achievement_counts[ach] += 1
+
+        achievement_unlock_rates = {
+            ach: count / num_trajectories for ach, count in achievement_counts.items()
+        }
+        avg_achievements_per_trajectory = total_achievement_count / num_trajectories
+
+        # Compute Hafner score
+        all_achievement_rates = []
+        for achievement in ALL_ACHIEVEMENTS:
+            unlock_rate = achievement_counts.get(achievement, 0) / num_trajectories
+            all_achievement_rates.append(unlock_rate * 100.0)
+
+        hafner_adjusted_score = hafner_score(all_achievement_rates)
+
+        # Compute BALROG scores
+        balrog_scores = [t.balrog_percent for t in trajectories]
+        balrog_score_avg = sum(balrog_scores) / len(balrog_scores) if balrog_scores else 0.0
+        balrog_score_best = max(balrog_scores) if balrog_scores else 0.0
+
+        # Multi-action analysis
+        all_actions_per_turn = []
+        actions_per_turn_dist = defaultdict(int)
+        for traj in trajectories:
+            all_actions_per_turn.extend(traj.actions_per_turn)
+            for count in traj.actions_per_turn:
+                actions_per_turn_dist[count] += 1
+
+        avg_actions_per_turn_overall = (
+            sum(all_actions_per_turn) / len(all_actions_per_turn) if all_actions_per_turn else 0.0
+        )
+
+        # Termination analysis
+        termination_counts = defaultdict(int)
+        for traj in trajectories:
+            termination_counts[traj.termination_reason] += 1
+        termination_breakdown = {
+            reason: count / num_trajectories for reason, count in termination_counts.items()
+        }
+
+        # Final stats
+        depth_values = [t.final_depth for t in trajectories if t.final_depth is not None]
+        level_values = [t.final_level for t in trajectories if t.final_level is not None]
+        gold_values = [t.final_gold for t in trajectories if t.final_gold is not None]
+
+        avg_final_depth = sum(depth_values) / len(depth_values) if depth_values else None
+        avg_final_level = sum(level_values) / len(level_values) if level_values else None
+        avg_final_gold = sum(gold_values) / len(gold_values) if gold_values else None
+
+        return AggregateResults(
+            model_name=model_name,
+            difficulty=difficulty,
+            num_trajectories=num_trajectories,
+            success_rate=success_rate,
+            avg_total_steps=avg_total_steps,
+            avg_total_turns=avg_total_turns,
+            avg_total_reward=avg_total_reward,
+            unique_achievements_unlocked=all_achievements,
+            total_achievement_count=total_achievement_count,
+            avg_achievements_per_trajectory=avg_achievements_per_trajectory,
+            achievement_unlock_rates=achievement_unlock_rates,
+            hafner_score=hafner_adjusted_score,
+            balrog_score_avg=balrog_score_avg,
+            balrog_score_best=balrog_score_best,
+            avg_actions_per_turn_overall=avg_actions_per_turn_overall,
+            actions_per_turn_distribution=dict(actions_per_turn_dist),
+            termination_breakdown=termination_breakdown,
+            avg_final_depth=avg_final_depth,
+            avg_final_level=avg_final_level,
+            avg_final_gold=avg_final_gold,
+        )
+
+    def _generate_comprehensive_report(self) -> Dict[str, Any]:
+        """Generate comprehensive evaluation report with all metrics and tables."""
+
+        # Group results by model and difficulty
+        grouped_results = defaultdict(lambda: defaultdict(list))
+        for result in self.trajectory_results:
+            grouped_results[result.model_name][result.difficulty].append(result)
+
+        # Generate aggregate results
+        aggregate_results = []
+        for model_name, difficulties in grouped_results.items():
+            for difficulty, trajectories in difficulties.items():
+                agg = self._compute_aggregate_metrics(model_name, difficulty, trajectories)
+                aggregate_results.append(agg)
+
+        # Generate all tables and analyses
+        report = {
+            "evaluation_summary": self._generate_summary_table(aggregate_results),
+            "achievement_percentage_table": self._generate_achievement_percentage_table(
+                grouped_results
+            ),
+            "termination_breakdown_table": self._generate_termination_breakdown_table(
+                aggregate_results
+            ),
+            "trajectory_by_trajectory_breakdown": self._generate_trajectory_breakdown(),
+            "sota_comparison": self._generate_sota_comparison(aggregate_results),
+            "raw_aggregate_results": [asdict(agg) for agg in aggregate_results],
+            "raw_trajectory_results": [asdict(traj) for traj in self.trajectory_results],
+        }
+
+        return report
+
+    def _generate_summary_table(self, aggregate_results: List[AggregateResults]) -> pd.DataFrame:
+        """Generate main summary table with key metrics."""
+
+        data = []
+        for agg in aggregate_results:
+            data.append(
+                {
+                    "Model": agg.model_name,
+                    "Difficulty": agg.difficulty,
+                    "Success Rate": f"{agg.success_rate:.1%}",
+                    "Hafner Score": f"{agg.hafner_score:.1f}%",
+                    "BALROG Avg": f"{agg.balrog_score_avg:.1f}%",
+                    "BALROG Best": f"{agg.balrog_score_best:.1f}%",
+                    "Avg Steps": f"{agg.avg_total_steps:.1f}",
+                    "Avg Turns": f"{agg.avg_total_turns:.1f}",
+                    "Avg Reward": f"{agg.avg_total_reward:.3f}",
+                    "Unique Achievements": len(agg.unique_achievements_unlocked),
+                    "Avg Achievements/Traj": f"{agg.avg_achievements_per_trajectory:.2f}",
+                    "Avg Actions/Turn": f"{agg.avg_actions_per_turn_overall:.1f}",
+                }
+            )
+
+        return pd.DataFrame(data)
+
+    def _generate_achievement_percentage_table(
+        self, grouped_results: Dict[str, Dict[str, List[TrajectoryResult]]]
+    ) -> pd.DataFrame:
+        """Generate table showing percentage of trajectories achieving each achievement."""
+
+        data = []
+
+        for model_name, difficulties in grouped_results.items():
+            for difficulty, trajectories in difficulties.items():
+                if not trajectories:
+                    continue
+
+                num_trajectories = len(trajectories)
+                row = {"Model": model_name, "Difficulty": difficulty}
+
+                # Count achievements
+                achievement_counts = defaultdict(int)
+                for traj in trajectories:
+                    for ach in traj.achievements_unlocked:
+                        achievement_counts[ach] += 1
+
+                # Add percentage for each achievement
+                for achievement in ALL_ACHIEVEMENTS:
+                    count = achievement_counts[achievement]
+                    percentage = count / num_trajectories if num_trajectories > 0 else 0.0
+                    row[achievement] = f"{percentage:.1%}"
+
+                data.append(row)
+
+        df = pd.DataFrame(data)
+
+        # Reorder columns: Model, Difficulty, then achievements by category
+        base_cols = ["Model", "Difficulty"]
+        achievement_cols = []
+        for category in ["basic", "intermediate", "advanced"]:
+            for ach in ACHIEVEMENT_CATEGORIES[category]:
+                if ach in df.columns:
+                    achievement_cols.append(ach)
+
+        return df[base_cols + achievement_cols]
+
+    def _generate_termination_breakdown_table(
+        self, aggregate_results: List[AggregateResults]
+    ) -> pd.DataFrame:
+        """Generate table showing termination reason percentages."""
+
+        data = []
+        for agg in aggregate_results:
+            row = {
+                "Model": agg.model_name,
+                "Difficulty": agg.difficulty,
+            }
+
+            for reason in TERMINATION_REASONS:
+                percentage = agg.termination_breakdown.get(reason, 0.0)
+                row[f"{reason.title()} %"] = f"{percentage:.1%}"
+
+            data.append(row)
+
+        return pd.DataFrame(data)
+
+    def _generate_trajectory_breakdown(self) -> pd.DataFrame:
+        """Generate detailed trajectory-by-trajectory breakdown."""
+
+        data = []
+        for traj in self.trajectory_results:
+            # Achievement category breakdown
+            basic_achievements = len(
+                [a for a in traj.achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["basic"]]
+            )
+            inter_achievements = len(
+                [
+                    a
+                    for a in traj.achievements_unlocked
+                    if a in ACHIEVEMENT_CATEGORIES["intermediate"]
+                ]
+            )
+            adv_achievements = len(
+                [a for a in traj.achievements_unlocked if a in ACHIEVEMENT_CATEGORIES["advanced"]]
+            )
+
+            data.append(
+                {
+                    "Trajectory ID": traj.trajectory_id[:8],
+                    "Model": traj.model_name,
+                    "Difficulty": traj.difficulty,
+                    "Seed": traj.seed,
+                    "Success": "✓" if traj.success else "✗",
+                    "Steps": traj.total_steps,
+                    "Turns": traj.total_turns,
+                    "Reward": f"{traj.total_reward:.3f}",
+                    "Total Achievements": len(traj.achievements_unlocked),
+                    "Basic": basic_achievements,
+                    "Intermediate": inter_achievements,
+                    "Advanced": adv_achievements,
+                    "BALROG Score": f"{traj.balrog_percent:.1f}%",
+                    "Termination": traj.termination_reason,
+                    "Final Depth": traj.final_depth,
+                    "Achievements": ", ".join(sorted(traj.achievements_unlocked))
+                    if traj.achievements_unlocked
+                    else "None",
+                }
+            )
+
+        return pd.DataFrame(data)
+
+    def _generate_sota_comparison(
+        self, aggregate_results: List[AggregateResults]
+    ) -> Dict[str, pd.DataFrame]:
+        """Generate comparison tables with SOTA benchmarks, separating Hafner and BALROG methodologies."""
+
+        # Create our results table for both methodologies
+        our_hafner_data = []
+        our_balrog_data = []
+
+        for agg in aggregate_results:
+            # Hafner results
+            hafner_row = {
+                "System": f"{agg.model_name} (multi-action)",
+                "Hafner Score": f"{agg.hafner_score:.1f}%",
+                "Category": "Current Evaluation (Hafner)",
+            }
+            our_hafner_data.append(hafner_row)
+
+            # BALROG results
+            balrog_row = {
+                "System": f"{agg.model_name} (multi-action)",
+                "BALROG Score (Avg)": f"{agg.balrog_score_avg:.1f}%",
+                "BALROG Score (Best)": f"{agg.balrog_score_best:.1f}%",
+                "Category": "Current Evaluation (BALROG)",
+            }
+
+            # Add percentage comparison to BALROG SOTA if we can map the model name
+            if agg.model_name in MODEL_NAME_TO_SOTA:
+                sota_name = MODEL_NAME_TO_SOTA[agg.model_name]
+                if sota_name in BALROG_SOTA_SCORES["balrog_leaderboard"]:
+                    balrog_sota_score = BALROG_SOTA_SCORES["balrog_leaderboard"][sota_name]
+                    percentage_of_balrog_sota_avg = (agg.balrog_score_avg / balrog_sota_score) * 100
+                    percentage_of_balrog_sota_best = (
+                        agg.balrog_score_best / balrog_sota_score
+                    ) * 100
+                    balrog_row["% of BALROG SOTA (Avg)"] = f"{percentage_of_balrog_sota_avg:.1f}%"
+                    balrog_row["% of BALROG SOTA (Best)"] = f"{percentage_of_balrog_sota_best:.1f}%"
+                    balrog_row["BALROG SOTA Reference"] = f"{sota_name} ({balrog_sota_score:.1f}%)"
+
+            our_balrog_data.append(balrog_row)
+
+        our_hafner_df = pd.DataFrame(our_hafner_data)
+        our_balrog_df = pd.DataFrame(our_balrog_data)
+
+        return {
+            "our_hafner_results": our_hafner_df,
+            "our_balrog_results": our_balrog_df,
+            "methodology_note": "⚠️  CRITICAL: Hafner scores (log-adjusted multi-episode) and BALROG scores (simple single-episode percentage) use different methodologies and are NOT directly comparable!",
+        }
+
+    def print_report(self, report: Dict[str, Any]):
+        """Print a formatted evaluation report."""
+
+        print("\n" + "=" * 80)
+        print("🎯 NETHACK EVALUATION REPORT")
+        print("=" * 80)
+
+        # Summary table
+        print("\n📊 EVALUATION SUMMARY")
+        summary_df = report["evaluation_summary"]
+        # Clean formatting for summary table
+        for col in summary_df.columns:
+            if len(col) > 12:  # Truncate long column names
+                summary_df = summary_df.rename(columns={col: col[:12]})
+        print(summary_df.to_string(index=False, max_colwidth=12))
+
+        # Create and show vertical achievement table
+        print("\n🏆 ACHIEVEMENT UNLOCK RATES")
+        print("Format: unlocked/total (percentage)")
+
+        # Group results for achievement summary
+        grouped_results = defaultdict(lambda: defaultdict(list))
+        for traj in self.trajectory_results:
+            grouped_results[traj.model_name][traj.difficulty].append(traj)
+
+        achievement_summary = self._generate_achievement_summary_table(grouped_results)
+
+        # Print by category for better readability
+        for category in ["Basic", "Intermediate", "Advanced"]:
+            category_data = achievement_summary[achievement_summary["Category"] == category]
+            if not category_data.empty:
+                print(f"\n{category.upper()} ACHIEVEMENTS:")
+                category_display = category_data.drop("Category", axis=1)
+                print(category_display.to_string(index=False))
+
+        # Trajectory breakdown (summary stats only for space)
+        traj_df = report["trajectory_by_trajectory_breakdown"]
+        print(f"\n📋 TRAJECTORY BREAKDOWN ({len(traj_df)} total trajectories)")
+        print("Sample trajectories:")
+        sample_cols = [
+            "Model",
+            "Difficulty",
+            "Success",
+            "Steps",
+            "Total Achievements",
+            "BALROG Score",
+            "Termination",
+        ]
+        sample_df = traj_df[sample_cols].head(5)
+        print(sample_df.to_string(index=False, max_colwidth=12))
+        if len(traj_df) > 5:
+            print(f"... and {len(traj_df) - 5} more trajectories")
+
+        # SOTA comparison
+        sota_comparison = report["sota_comparison"]
+        print("\n🏆 SOTA COMPARISON")
+        print(sota_comparison["methodology_note"])
+
+        print("\n📊 HAFNER METHODOLOGY RESULTS (Multi-episode log-adjusted)")
+        hafner_df = sota_comparison["our_hafner_results"]
+        print(hafner_df.to_string(index=False, max_colwidth=20))
+
+        print("\n📊 BALROG METHODOLOGY RESULTS (Single-episode percentage)")
+        balrog_df = sota_comparison["our_balrog_results"]
+        # Clean up column names for better display
+        balrog_clean = balrog_df.copy()
+        if "% of BALROG SOTA (Avg)" in balrog_clean.columns:
+            balrog_clean = balrog_clean.rename(columns={"% of BALROG SOTA (Avg)": "% SOTA Avg"})
+        if "% of BALROG SOTA (Best)" in balrog_clean.columns:
+            balrog_clean = balrog_clean.rename(columns={"% of BALROG SOTA (Best)": "% SOTA Best"})
+        print(balrog_clean.to_string(index=False, max_colwidth=20))
+
+        print("\n" + "=" * 80)
+
+    def _generate_achievement_summary_table(
+        self, grouped_results: Dict[str, Dict[str, List[TrajectoryResult]]]
+    ) -> pd.DataFrame:
+        """Generate a vertical achievement summary table that's easier to read."""
+
+        data = []
+
+        # For each achievement, show rates across all model/difficulty combinations
+        for category_name, achievements in ACHIEVEMENT_CATEGORIES.items():
+            for achievement in achievements:
+                row = {
+                    "Category": category_name.capitalize(),
+                    "Achievement": achievement.replace("_", " ").title(),
+                }
+
+                # Add columns for each model/difficulty combination
+                for model_name, difficulties in grouped_results.items():
+                    for difficulty, trajectories in difficulties.items():
+                        if not trajectories:
+                            continue
+
+                        num_trajectories = len(trajectories)
+                        count = sum(
+                            1 for traj in trajectories if achievement in traj.achievements_unlocked
+                        )
+                        percentage = count / num_trajectories if num_trajectories > 0 else 0.0
+
+                        col_name = f"{model_name} ({difficulty})"
+                        row[col_name] = f"{count}/{num_trajectories} ({percentage:.1%})"
+
+                data.append(row)
+
+        return pd.DataFrame(data)
+
+
+# Convenience function for quick evaluations
+async def run_nethack_eval(
+    model_names: List[str],
+    difficulties: List[str] = ["easy", "hard"],
+    num_trajectories: int = 3,
+    max_turns: int = 200,
+) -> Dict[str, Any]:
+    """Quick evaluation runner with automatic report generation."""
+
+    framework = NetHackEvalFramework()
+    report = await framework.run_evaluation(
+        model_names=model_names,
+        difficulties=difficulties,
+        num_trajectories_per_condition=num_trajectories,
+        max_turns=max_turns,
+    )
+
+    framework.print_report(report)
+    return report