synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/custom_eval_pipelines.py

@@ -1,400 +0,0 @@
-#!/usr/bin/env python3
-"""
-Custom evaluation pipelines for analyzing Crafter traces using Gemini-1.5-flash.
-"""
-
-import json
-import duckdb
-from typing import List, Dict, Any, Optional
-import asyncio
-import os
-import sys
-import random
-
-# Add the synth_ai path to import base classes
-sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../../'))
-from synth_ai.evals.base import Judgement, BaseEval
-from synth_ai.lm.core.main_v2 import LM
-
-class MisunderstoodCrafterRulesEval(BaseEval):
-    """Evaluate if the agent misunderstood Crafter game rules."""
-    
-    def __init__(self, model_name: str = "gpt-4o-mini"):
-        self.model_name = model_name
-        self.lm = LM(
-            model_name=model_name,
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.3  # Increased temperature for more variety
-        )
-    
-    async def run(self, session_data: Dict[str, Any]) -> List[Judgement]:
-        """Analyze if the agent misunderstood Crafter rules using LLM."""
-        
-        # Extract relevant data from session
-        actions = session_data.get("actions", [])
-        invalid_actions = session_data.get("invalid_actions", [])
-        achievements = session_data.get("achievements", [])
-        inventory_changes = session_data.get("inventory_changes", [])
-        total_steps = session_data.get("total_steps", 0)
-        
-        # Add some randomization to the prompt
-        random_seed = random.randint(1, 1000)
-        
-        # Create analysis prompt with more specific instructions
-        prompt = f"""
-You are an expert evaluator analyzing a Crafter game session to determine if the agent misunderstood the game rules.
-
-CRAFTER GAME RULES:
-- The agent can move in 4 directions: up, down, left, right
-- The agent can perform actions like: collect, craft, place, eat, sleep
-- Valid actions depend on what's nearby and what's in inventory
-- The goal is to achieve various crafting milestones
-
-SESSION DATA (Seed: {random_seed}):
-Total steps: {total_steps}
-Actions taken: {actions[:30]}  # First 30 actions
-Invalid actions: {invalid_actions[:20]}  # First 20 invalid actions
-Achievements unlocked: {achievements}
-Inventory changes: {inventory_changes[:10]}  # First 10 inventory changes
-
-ANALYSIS TASK:
-Analyze if the agent misunderstood Crafter rules. Look for:
-1. Repeated invalid actions that suggest rule confusion
-2. Actions that don't make sense given the game state
-3. Missing obvious valid actions
-4. Inefficient action patterns
-5. Specific rule violations (movement, crafting, collection)
-
-Provide your analysis in this EXACT JSON format (no additional text):
-{{
-    "score": <float 0-1, where 1=severe misunderstanding>,
-    "reasoning": "<detailed explanation>",
-    "evidence": ["<specific example 1>", "<specific example 2>", ...],
-    "rule_violations": ["<specific rule misunderstood 1>", ...]
-}}
-
-Focus on concrete evidence from the session data. Be specific about what rules the agent seems to misunderstand.
-"""
-        
-        try:
-            # Use the existing LM infrastructure
-            response = await self.lm.respond_async(
-                system_message="You are an expert evaluator analyzing AI agent behavior in games. Respond only with valid JSON.",
-                user_message=prompt
-            )
-            
-            print(f"DEBUG - Raw LLM response: {response.raw_response[:200]}...")
-            
-            # Parse JSON response
-            try:
-                result = json.loads(response.raw_response)
-            except json.JSONDecodeError:
-                # Try to extract JSON from response
-                import re
-                json_match = re.search(r'\{.*\}', response.raw_response, re.DOTALL)
-                if json_match:
-                    result = json.loads(json_match.group())
-                else:
-                    result = {
-                        "score": 0.5,
-                        "reasoning": "Could not parse LLM response",
-                        "evidence": ["Response parsing failed"],
-                        "rule_violations": []
-                    }
-            
-            return [Judgement(
-                criteria="misunderstood_crafter_rules",
-                score=result.get("score", 0.5),
-                reasoning=result.get("reasoning", "No reasoning provided"),
-                evidence=result.get("evidence", [])
-            )]
-            
-        except Exception as e:
-            return [Judgement(
-                criteria="misunderstood_crafter_rules",
-                score=0.5,
-                reasoning=f"Evaluation failed: {str(e)}",
-                evidence=[f"Error: {str(e)}"]
-            )]
-
-class WastedTimeEval(BaseEval):
-    """Evaluate if the agent wasted time in inefficient actions."""
-    
-    def __init__(self, model_name: str = "gpt-4o-mini"):
-        self.model_name = model_name
-        self.lm = LM(
-            model_name=model_name,
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.3  # Increased temperature for more variety
-        )
-    
-    async def run(self, session_data: Dict[str, Any]) -> List[Judgement]:
-        """Analyze if the agent wasted time inefficiently using LLM."""
-        
-        # Extract relevant data from session
-        actions = session_data.get("actions", [])
-        invalid_actions = session_data.get("invalid_actions", [])
-        achievements = session_data.get("achievements", [])
-        inventory_changes = session_data.get("inventory_changes", [])
-        total_steps = session_data.get("total_steps", 0)
-        
-        # Add some randomization to the prompt
-        random_seed = random.randint(1, 1000)
-        
-        # Create analysis prompt with more specific instructions
-        prompt = f"""
-You are an expert evaluator analyzing a Crafter game session to determine if the agent wasted time inefficiently.
-
-EFFICIENCY CRITERIA:
-- Repeated failed actions that could be avoided
-- Unnecessary movement patterns
-- Inefficient resource gathering
-- Poor prioritization of goals
-- Actions that don't contribute to achievements
-- Time spent on non-productive activities
-
-SESSION DATA (Seed: {random_seed}):
-Total steps: {total_steps}
-Actions taken: {actions[:40]}  # First 40 actions
-Invalid actions: {invalid_actions[:25]}  # First 25 invalid actions
-Achievements unlocked: {achievements}
-Inventory changes: {inventory_changes[:15]}  # First 15 inventory changes
-
-ANALYSIS TASK:
-Analyze if the agent wasted time inefficiently. Look for:
-1. Repeated invalid actions that waste steps
-2. Inefficient movement patterns
-3. Poor resource gathering strategies
-4. Actions that don't advance toward goals
-5. Missed opportunities for better actions
-6. Time spent on non-productive activities
-
-Provide your analysis in this EXACT JSON format (no additional text):
-{{
-    "score": <float 0-1, where 1=severe time wasting>,
-    "reasoning": "<detailed explanation>",
-    "evidence": ["<specific example 1>", "<specific example 2>", ...],
-    "inefficiencies": ["<specific inefficiency 1>", ...],
-    "efficiency_score": <float 0-1, where 1=very efficient>
-}}
-
-Focus on concrete evidence from the session data. Be specific about how the agent wasted time.
-"""
-        
-        try:
-            # Use the existing LM infrastructure
-            response = await self.lm.respond_async(
-                system_message="You are an expert evaluator analyzing AI agent efficiency in games. Respond only with valid JSON.",
-                user_message=prompt
-            )
-            
-            print(f"DEBUG - Raw LLM response: {response.raw_response[:200]}...")
-            
-            # Parse JSON response
-            try:
-                result = json.loads(response.raw_response)
-            except json.JSONDecodeError:
-                # Try to extract JSON from response
-                import re
-                json_match = re.search(r'\{.*\}', response.raw_response, re.DOTALL)
-                if json_match:
-                    result = json.loads(json_match.group())
-                else:
-                    result = {
-                        "score": 0.5,
-                        "reasoning": "Could not parse LLM response",
-                        "evidence": ["Response parsing failed"],
-                        "inefficiencies": [],
-                        "efficiency_score": 0.5
-                    }
-            
-            return [Judgement(
-                criteria="wasted_time",
-                score=result.get("score", 0.5),
-                reasoning=result.get("reasoning", "No reasoning provided"),
-                evidence=result.get("evidence", [])
-            )]
-            
-        except Exception as e:
-            return [Judgement(
-                criteria="wasted_time",
-                score=0.5,
-                reasoning=f"Evaluation failed: {str(e)}",
-                evidence=[f"Error: {str(e)}"]
-            )]
-
-class CrafterTraceAnalyzer:
-    """Main analyzer for Crafter traces."""
-    
-    def __init__(self, experiment_id: str, db_path: str = "crafter_traces.duckdb", model_name: str = "gpt-4o-mini"):
-        self.experiment_id = experiment_id
-        self.db_path = db_path
-        self.model_name = model_name
-        
-        # Initialize evaluators
-        self.misunderstood_rules_eval = MisunderstoodCrafterRulesEval(model_name)
-        self.wasted_time_eval = WastedTimeEval(model_name)
-    
-    def extract_session_data(self, session_id: str) -> Dict[str, Any]:
-        """Extract session data from DuckDB."""
-        conn = duckdb.connect(self.db_path)
-        
-        # Get session metadata
-        result = conn.execute("""
-            SELECT metadata FROM session_traces 
-            WHERE session_id = ? AND experiment_id = ?
-        """, [session_id, self.experiment_id]).fetchall()
-        
-        session_data = {}
-        if result:
-            metadata = json.loads(result[0][0]) if isinstance(result[0][0], str) else result[0][0]
-            
-            # Extract achievements
-            for item in metadata:
-                if item.get('metadata_type') == 'SessionMetadum':
-                    data = item.get('data', {})
-                    if 'achievements' in data:
-                        achievements = data['achievements']
-                        session_data['achievements'] = [k for k, v in achievements.items() if v]
-                    if 'num_achievements' in data:
-                        session_data['num_achievements'] = data['num_achievements']
-                    if 'total_reward' in data:
-                        session_data['total_reward'] = data['total_reward']
-                    if 'rollout_length' in data:
-                        session_data['total_steps'] = data['rollout_length']
-        
-        # Get events for action analysis
-        result = conn.execute("""
-            SELECT event_type, metadata, event_metadata
-            FROM events 
-            WHERE session_id = ? 
-            ORDER BY event_time
-        """, [session_id]).fetchall()
-        
-        actions = []
-        invalid_actions = []
-        inventory_changes = []
-        
-        for event_type, metadata, event_metadata in result:
-            if event_type == 'runtime':
-                # Parse action from metadata
-                try:
-                    meta_dict = json.loads(metadata) if isinstance(metadata, str) else metadata
-                    if 'action' in meta_dict:
-                        actions.append(meta_dict['action'])
-                except:
-                    pass
-                
-                # Check for invalid actions in event_metadata
-                if event_metadata:
-                    try:
-                        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
-                        for hook_str in hook_list:
-                            if isinstance(hook_str, str):
-                                import re
-                                if "'hook_name': 'invalid_action'" in hook_str:
-                                    # Extract the actual action name from the description
-                                    action_match = re.search(r"'action':\s*'([^']+)'", hook_str)
-                                    if action_match:
-                                        invalid_actions.append(action_match.group(1))
-                                    else:
-                                        invalid_actions.append("unknown")
-                    except:
-                        pass
-            
-            elif event_type == 'environment':
-                # Check for inventory changes
-                if event_metadata:
-                    try:
-                        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
-                        for hook_str in hook_list:
-                            if isinstance(hook_str, str):
-                                if "'hook_name': 'inventory_increase'" in hook_str:
-                                    inventory_changes.append("inventory_increase")
-                    except:
-                        pass
-        
-        session_data.update({
-            'actions': actions,
-            'invalid_actions': invalid_actions,
-            'inventory_changes': inventory_changes
-        })
-        
-        conn.close()
-        return session_data
-    
-    async def evaluate_session(self, session_id: str) -> Dict[str, List[Judgement]]:
-        """Evaluate a single session."""
-        session_data = self.extract_session_data(session_id)
-        
-        # Run evaluations
-        misunderstood_rules = await self.misunderstood_rules_eval.run(session_data)
-        wasted_time = await self.wasted_time_eval.run(session_data)
-        
-        return {
-            'misunderstood_rules': misunderstood_rules,
-            'wasted_time': wasted_time
-        }
-    
-    async def evaluate_experiment(self) -> Dict[str, Any]:
-        """Evaluate all sessions in the experiment."""
-        conn = duckdb.connect(self.db_path)
-        
-        # Get all session IDs for this experiment
-        result = conn.execute("""
-            SELECT session_id FROM session_traces 
-            WHERE experiment_id = ?
-            ORDER BY session_id
-        """, [self.experiment_id]).fetchall()
-        
-        session_ids = [row[0] for row in result]
-        conn.close()
-        
-        print(f"Evaluating {len(session_ids)} sessions in parallel...")
-        
-        # Create all evaluation tasks
-        tasks = [self.evaluate_session(session_id) for session_id in session_ids]
-        
-        # Run all evaluations in parallel
-        all_results_list = await asyncio.gather(*tasks)
-        
-        # Convert to dictionary
-        all_results = {}
-        for session_id, results in zip(session_ids, all_results_list):
-            all_results[session_id] = results
-        
-        return all_results
-
-async def main():
-    """Example usage."""
-    import sys
-    
-    if len(sys.argv) > 1:
-        experiment_id = sys.argv[1]
-    else:
-        print("Usage: python custom_eval_pipelines.py <experiment_id>")
-        print("Example: python custom_eval_pipelines.py 77022cce-4bda-4415-9bce-0095e4ef2237")
-        return
-    
-    # Use Gemini for evaluation
-    analyzer = CrafterTraceAnalyzer(experiment_id, model_name="gemini-1.5-flash")
-    results = await analyzer.evaluate_experiment()
-    
-    # Print results
-    print("\n" + "="*80)
-    print("EVALUATION RESULTS")
-    print("="*80)
-    
-    for session_id, session_results in results.items():
-        print(f"\nSession: {session_id}")
-        print("-" * 60)
-        
-        for eval_type, judgements in session_results.items():
-            print(f"\n{eval_type.upper()}:")
-            for judgement in judgements:
-                print(f"  Score: {judgement.score:.3f}")
-                print(f"  Reasoning: {judgement.reasoning}")
-                print(f"  Evidence: {judgement.evidence}")
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/plot_hook_frequency.py

@@ -1,195 +0,0 @@
-#!/usr/bin/env python3
-"""
-Plot the frequency of achievements and invalid actions over time (by step number).
-Terminal-only version.
-"""
-
-import duckdb
-import json
-import re
-from collections import defaultdict
-from typing import Dict, List, Tuple
-
-def extract_step_from_metadata(metadata: str) -> int:
-    """Extract step number from event metadata."""
-    try:
-        metadata_dict = json.loads(metadata) if isinstance(metadata, str) else metadata
-        return metadata_dict.get('turn', 0)
-    except:
-        return 0
-
-def parse_hook_metadata(event_metadata: str) -> List[Dict]:
-    """Parse hook metadata from event_metadata string with better error handling."""
-    hooks = []
-    try:
-        # The metadata is stored as a list of strings, each containing a hook dict
-        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
-        
-        for hook_str in hook_list:
-            if isinstance(hook_str, str):
-                # Use regex to extract hook_name more reliably
-                hook_name_match = re.search(r"'hook_name':\s*'([^']+)'", hook_str)
-                if hook_name_match:
-                    hook_name = hook_name_match.group(1)
-                    hooks.append({'hook_name': hook_name})
-            else:
-                hooks.append(hook_str)
-    except Exception as e:
-        # Try alternative parsing if JSON fails
-        try:
-            # Look for hook_name patterns in the string
-            hook_names = re.findall(r"'hook_name':\s*'([^']+)'", event_metadata)
-            for hook_name in hook_names:
-                hooks.append({'hook_name': hook_name})
-        except:
-            pass
-    
-    return hooks
-
-def analyze_hook_frequency(experiment_id: str):
-    """Analyze hook frequency over time."""
-    conn = duckdb.connect("crafter_traces.duckdb")
-    
-    print(f"📊 ANALYZING HOOK FREQUENCY OVER TIME")
-    print("=" * 80)
-    print(f"Experiment ID: {experiment_id}")
-    print()
-    
-    # Get events with hook metadata
-    result = conn.execute("""
-        SELECT e.session_id, e.event_type, e.event_metadata, e.metadata
-        FROM events e 
-        JOIN session_traces st ON e.session_id = st.session_id 
-        WHERE st.experiment_id = ? AND e.event_metadata IS NOT NULL
-        ORDER BY e.event_time
-    """, [experiment_id]).fetchall()
-    
-    # Track hook frequency by step
-    step_achievements = defaultdict(int)
-    step_invalid_actions = defaultdict(int)
-    step_inventory_increases = defaultdict(int)
-    
-    # Track by session for more detailed analysis
-    session_data = defaultdict(lambda: {
-        'achievements': defaultdict(int),
-        'invalid_actions': defaultdict(int),
-        'inventory_increases': defaultdict(int)
-    })
-    
-    for row in result:
-        session_id, event_type, event_metadata, metadata = row
-        
-        # Extract step number
-        step = extract_step_from_metadata(metadata)
-        
-        # Parse hook metadata
-        hooks = parse_hook_metadata(event_metadata)
-        
-        for hook in hooks:
-            hook_name = hook.get('hook_name', 'unknown')
-            
-            if hook_name == 'easy_achievement' or hook_name == 'medium_achievement' or hook_name == 'hard_achievement':
-                step_achievements[step] += 1
-                session_data[session_id]['achievements'][step] += 1
-            elif hook_name == 'invalid_action':
-                step_invalid_actions[step] += 1
-                session_data[session_id]['invalid_actions'][step] += 1
-            elif hook_name == 'inventory_increase':
-                step_inventory_increases[step] += 1
-                session_data[session_id]['inventory_increases'][step] += 1
-    
-    # Prepare data for plotting
-    max_step = max(
-        max(step_achievements.keys()) if step_achievements else 0,
-        max(step_invalid_actions.keys()) if step_invalid_actions else 0,
-        max(step_inventory_increases.keys()) if step_inventory_increases else 0
-    )
-    
-    steps = list(range(max_step + 1))
-    achievement_freq = [step_achievements[step] for step in steps]
-    invalid_action_freq = [step_invalid_actions[step] for step in steps]
-    inventory_freq = [step_inventory_increases[step] for step in steps]
-    
-    # Print summary statistics
-    print("📈 SUMMARY STATISTICS")
-    print("-" * 50)
-    print(f"Total steps analyzed: {max_step + 1}")
-    print(f"Total achievements: {sum(achievement_freq)}")
-    print(f"Total invalid actions: {sum(invalid_action_freq)}")
-    print(f"Total inventory increases: {sum(inventory_freq)}")
-    print()
-    
-    print("🏆 ACHIEVEMENT ANALYSIS")
-    print("-" * 50)
-    achievement_steps = [step for step, freq in step_achievements.items() if freq > 0]
-    if achievement_steps:
-        print(f"Achievements occur at steps: {sorted(achievement_steps)}")
-        print(f"Most common achievement step: {max(step_achievements.items(), key=lambda x: x[1])}")
-    else:
-        print("No achievements found")
-    print()
-    
-    print("❌ INVALID ACTION ANALYSIS")
-    print("-" * 50)
-    invalid_steps = [step for step, freq in step_invalid_actions.items() if freq > 0]
-    if invalid_steps:
-        print(f"Invalid actions occur at steps: {sorted(invalid_steps)}")
-        print(f"Most common invalid action step: {max(step_invalid_actions.items(), key=lambda x: x[1])}")
-    else:
-        print("No invalid actions found")
-    print()
-    
-    print("📦 INVENTORY ANALYSIS")
-    print("-" * 50)
-    inventory_steps = [step for step, freq in step_inventory_increases.items() if freq > 0]
-    if inventory_steps:
-        print(f"Inventory increases occur at steps: {sorted(inventory_steps)}")
-        print(f"Most common inventory increase step: {max(step_inventory_increases.items(), key=lambda x: x[1])}")
-    else:
-        print("No inventory increases found")
-    
-    # Create ASCII chart
-    print("\n📊 ASCII FREQUENCY CHART")
-    print("=" * 80)
-    print("Step | Achievements | Invalid Actions | Inventory")
-    print("-" * 80)
-    
-    for step in steps:
-        achievements = step_achievements[step]
-        invalid_actions = step_invalid_actions[step]
-        inventory = step_inventory_increases[step]
-        
-        if achievements > 0 or invalid_actions > 0 or inventory > 0:
-            print(f"{step:4d} | {achievements:11d} | {invalid_actions:14d} | {inventory:9d}")
-    
-    # Session-by-session breakdown
-    print("\n📋 SESSION-BY-SESSION BREAKDOWN")
-    print("-" * 50)
-    for session_id, data in session_data.items():
-        print(f"\nSession: {session_id}")
-        if data['achievements']:
-            print(f"  Achievements: {dict(data['achievements'])}")
-        if data['invalid_actions']:
-            print(f"  Invalid actions: {dict(data['invalid_actions'])}")
-        if data['inventory_increases']:
-            print(f"  Inventory increases: {dict(data['inventory_increases'])}")
-    
-    conn.close()
-    
-    return {
-        'steps': steps,
-        'achievement_freq': achievement_freq,
-        'invalid_action_freq': invalid_action_freq,
-        'inventory_freq': inventory_freq,
-        'session_data': session_data
-    }
-
-if __name__ == "__main__":
-    import sys
-    
-    if len(sys.argv) > 1:
-        experiment_id = sys.argv[1]
-        analyze_hook_frequency(experiment_id)
-    else:
-        print("Usage: python plot_hook_frequency.py <experiment_id>")
-        print("Example: python plot_hook_frequency.py 77022cce-4bda-4415-9bce-0095e4ef2237") 

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/seed_analysis_summary.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-"""
-Summary analysis of seed performance comparison between GPT-4.1-NANO and GPT-4.1-MINI.
-"""
-
-def print_summary():
-    print("🎯 KEY FINDINGS: SEED PERFORMANCE COMPARISON")
-    print("=" * 60)
-    
-    print("\n📊 OVERALL PERFORMANCE:")
-    print("  • GPT-4.1-MINI: 12 total achievements across 9 instances")
-    print("  • GPT-4.1-NANO:  2 total achievements across 2 instances")
-    print("  • MINI wins: 9 out of 10 instances (90% win rate)")
-    print("  • NANO wins: 0 out of 10 instances")
-    print("  • Ties: 1 instance (instance 9)")
-    
-    print("\n🏆 INSTANCE-BY-INSTANCE BREAKDOWN:")
-    print("  • Instance 1 (Seed 43): MINI wins (collect_wood vs 0)")
-    print("  • Instance 2 (Seed 44): MINI wins (collect_wood vs 0)")
-    print("  • Instance 3 (Seed 45): MINI wins (collect_sapling vs 0)")
-    print("  • Instance 4 (Seed 46): MINI wins (collect_wood vs 0)")
-    print("  • Instance 5 (Seed 47): MINI wins (collect_wood vs 0)")
-    print("  • Instance 6 (Seed 48): MINI wins (collect_sapling + eat_cow vs collect_sapling)")
-    print("  • Instance 7 (Seed 49): MINI wins (collect_sapling + collect_wood vs 0)")
-    print("  • Instance 8 (Seed 50): MINI wins (collect_wood vs 0)")
-    print("  • Instance 9 (Seed 51): TIE (0 vs 0)")
-    print("  • Instance 10 (Seed 52): MINI wins (collect_sapling + collect_wood vs collect_wood)")
-    
-    print("\n🎯 ACHIEVEMENT TYPE ANALYSIS:")
-    print("  • collect_wood: MINI 7, NANO 1 (MINI dominates)")
-    print("  • collect_sapling: MINI 4, NANO 1 (MINI dominates)")
-    print("  • eat_cow: MINI 1, NANO 0 (MINI only)")
-    print("  • All other achievements: 0 for both models")
-    
-    print("\n🔍 PATTERNS OBSERVED:")
-    print("  1. MINI consistently outperforms NANO across almost all seeds")
-    print("  2. MINI achieves more complex combinations (e.g., collect_sapling + eat_cow)")
-    print("  3. NANO struggles with basic achievements (only 2 total vs MINI's 12)")
-    print("  4. Both models struggle with advanced achievements (iron, diamond, etc.)")
-    print("  5. MINI shows better exploration and resource gathering capabilities")
-    
-    print("\n📈 IMPLICATIONS:")
-    print("  • MINI demonstrates significantly better reasoning and planning")
-    print("  • MINI's larger context window may enable better multi-step planning")
-    print("  • NANO may be hitting context limits or reasoning limitations")
-    print("  • Both models struggle with complex crafting and combat achievements")
-    print("  • The performance gap is consistent across different environment seeds")
-    
-    print("\n🎲 RANDOMNESS ANALYSIS:")
-    print("  • Seeds 43-52 were tested (10 different environments)")
-    print("  • MINI wins 9/10 = 90% win rate")
-    print("  • This suggests the performance difference is robust, not random")
-    print("  • Only instance 9 was a tie, suggesting MINI's advantage is consistent")
-
-if __name__ == "__main__":
-    print_summary()