synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.5__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/custom_eval_pipelines.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+"""
+Custom evaluation pipelines for analyzing Crafter traces using Gemini-1.5-flash.
+"""
+
+import json
+import duckdb
+from typing import List, Dict, Any, Optional
+import asyncio
+import os
+import sys
+import random
+
+# Add the synth_ai path to import base classes
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../../'))
+from synth_ai.evals.base import Judgement, BaseEval
+from synth_ai.lm.core.main_v2 import LM
+
+class MisunderstoodCrafterRulesEval(BaseEval):
+    """Evaluate if the agent misunderstood Crafter game rules."""
+    
+    def __init__(self, model_name: str = "gpt-4o-mini"):
+        self.model_name = model_name
+        self.lm = LM(
+            model_name=model_name,
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.3  # Increased temperature for more variety
+        )
+    
+    async def run(self, session_data: Dict[str, Any]) -> List[Judgement]:
+        """Analyze if the agent misunderstood Crafter rules using LLM."""
+        
+        # Extract relevant data from session
+        actions = session_data.get("actions", [])
+        invalid_actions = session_data.get("invalid_actions", [])
+        achievements = session_data.get("achievements", [])
+        inventory_changes = session_data.get("inventory_changes", [])
+        total_steps = session_data.get("total_steps", 0)
+        
+        # Add some randomization to the prompt
+        random_seed = random.randint(1, 1000)
+        
+        # Create analysis prompt with more specific instructions
+        prompt = f"""
+You are an expert evaluator analyzing a Crafter game session to determine if the agent misunderstood the game rules.
+
+CRAFTER GAME RULES:
+- The agent can move in 4 directions: up, down, left, right
+- The agent can perform actions like: collect, craft, place, eat, sleep
+- Valid actions depend on what's nearby and what's in inventory
+- The goal is to achieve various crafting milestones
+
+SESSION DATA (Seed: {random_seed}):
+Total steps: {total_steps}
+Actions taken: {actions[:30]}  # First 30 actions
+Invalid actions: {invalid_actions[:20]}  # First 20 invalid actions
+Achievements unlocked: {achievements}
+Inventory changes: {inventory_changes[:10]}  # First 10 inventory changes
+
+ANALYSIS TASK:
+Analyze if the agent misunderstood Crafter rules. Look for:
+1. Repeated invalid actions that suggest rule confusion
+2. Actions that don't make sense given the game state
+3. Missing obvious valid actions
+4. Inefficient action patterns
+5. Specific rule violations (movement, crafting, collection)
+
+Provide your analysis in this EXACT JSON format (no additional text):
+{{
+    "score": <float 0-1, where 1=severe misunderstanding>,
+    "reasoning": "<detailed explanation>",
+    "evidence": ["<specific example 1>", "<specific example 2>", ...],
+    "rule_violations": ["<specific rule misunderstood 1>", ...]
+}}
+
+Focus on concrete evidence from the session data. Be specific about what rules the agent seems to misunderstand.
+"""
+        
+        try:
+            # Use the existing LM infrastructure
+            response = await self.lm.respond_async(
+                system_message="You are an expert evaluator analyzing AI agent behavior in games. Respond only with valid JSON.",
+                user_message=prompt
+            )
+            
+            print(f"DEBUG - Raw LLM response: {response.raw_response[:200]}...")
+            
+            # Parse JSON response
+            try:
+                result = json.loads(response.raw_response)
+            except json.JSONDecodeError:
+                # Try to extract JSON from response
+                import re
+                json_match = re.search(r'\{.*\}', response.raw_response, re.DOTALL)
+                if json_match:
+                    result = json.loads(json_match.group())
+                else:
+                    result = {
+                        "score": 0.5,
+                        "reasoning": "Could not parse LLM response",
+                        "evidence": ["Response parsing failed"],
+                        "rule_violations": []
+                    }
+            
+            return [Judgement(
+                criteria="misunderstood_crafter_rules",
+                score=result.get("score", 0.5),
+                reasoning=result.get("reasoning", "No reasoning provided"),
+                evidence=result.get("evidence", [])
+            )]
+            
+        except Exception as e:
+            return [Judgement(
+                criteria="misunderstood_crafter_rules",
+                score=0.5,
+                reasoning=f"Evaluation failed: {str(e)}",
+                evidence=[f"Error: {str(e)}"]
+            )]
+
+class WastedTimeEval(BaseEval):
+    """Evaluate if the agent wasted time in inefficient actions."""
+    
+    def __init__(self, model_name: str = "gpt-4o-mini"):
+        self.model_name = model_name
+        self.lm = LM(
+            model_name=model_name,
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.3  # Increased temperature for more variety
+        )
+    
+    async def run(self, session_data: Dict[str, Any]) -> List[Judgement]:
+        """Analyze if the agent wasted time inefficiently using LLM."""
+        
+        # Extract relevant data from session
+        actions = session_data.get("actions", [])
+        invalid_actions = session_data.get("invalid_actions", [])
+        achievements = session_data.get("achievements", [])
+        inventory_changes = session_data.get("inventory_changes", [])
+        total_steps = session_data.get("total_steps", 0)
+        
+        # Add some randomization to the prompt
+        random_seed = random.randint(1, 1000)
+        
+        # Create analysis prompt with more specific instructions
+        prompt = f"""
+You are an expert evaluator analyzing a Crafter game session to determine if the agent wasted time inefficiently.
+
+EFFICIENCY CRITERIA:
+- Repeated failed actions that could be avoided
+- Unnecessary movement patterns
+- Inefficient resource gathering
+- Poor prioritization of goals
+- Actions that don't contribute to achievements
+- Time spent on non-productive activities
+
+SESSION DATA (Seed: {random_seed}):
+Total steps: {total_steps}
+Actions taken: {actions[:40]}  # First 40 actions
+Invalid actions: {invalid_actions[:25]}  # First 25 invalid actions
+Achievements unlocked: {achievements}
+Inventory changes: {inventory_changes[:15]}  # First 15 inventory changes
+
+ANALYSIS TASK:
+Analyze if the agent wasted time inefficiently. Look for:
+1. Repeated invalid actions that waste steps
+2. Inefficient movement patterns
+3. Poor resource gathering strategies
+4. Actions that don't advance toward goals
+5. Missed opportunities for better actions
+6. Time spent on non-productive activities
+
+Provide your analysis in this EXACT JSON format (no additional text):
+{{
+    "score": <float 0-1, where 1=severe time wasting>,
+    "reasoning": "<detailed explanation>",
+    "evidence": ["<specific example 1>", "<specific example 2>", ...],
+    "inefficiencies": ["<specific inefficiency 1>", ...],
+    "efficiency_score": <float 0-1, where 1=very efficient>
+}}
+
+Focus on concrete evidence from the session data. Be specific about how the agent wasted time.
+"""
+        
+        try:
+            # Use the existing LM infrastructure
+            response = await self.lm.respond_async(
+                system_message="You are an expert evaluator analyzing AI agent efficiency in games. Respond only with valid JSON.",
+                user_message=prompt
+            )
+            
+            print(f"DEBUG - Raw LLM response: {response.raw_response[:200]}...")
+            
+            # Parse JSON response
+            try:
+                result = json.loads(response.raw_response)
+            except json.JSONDecodeError:
+                # Try to extract JSON from response
+                import re
+                json_match = re.search(r'\{.*\}', response.raw_response, re.DOTALL)
+                if json_match:
+                    result = json.loads(json_match.group())
+                else:
+                    result = {
+                        "score": 0.5,
+                        "reasoning": "Could not parse LLM response",
+                        "evidence": ["Response parsing failed"],
+                        "inefficiencies": [],
+                        "efficiency_score": 0.5
+                    }
+            
+            return [Judgement(
+                criteria="wasted_time",
+                score=result.get("score", 0.5),
+                reasoning=result.get("reasoning", "No reasoning provided"),
+                evidence=result.get("evidence", [])
+            )]
+            
+        except Exception as e:
+            return [Judgement(
+                criteria="wasted_time",
+                score=0.5,
+                reasoning=f"Evaluation failed: {str(e)}",
+                evidence=[f"Error: {str(e)}"]
+            )]
+
+class CrafterTraceAnalyzer:
+    """Main analyzer for Crafter traces."""
+    
+    def __init__(self, experiment_id: str, db_path: str = "crafter_traces.duckdb", model_name: str = "gpt-4o-mini"):
+        self.experiment_id = experiment_id
+        self.db_path = db_path
+        self.model_name = model_name
+        
+        # Initialize evaluators
+        self.misunderstood_rules_eval = MisunderstoodCrafterRulesEval(model_name)
+        self.wasted_time_eval = WastedTimeEval(model_name)
+    
+    def extract_session_data(self, session_id: str) -> Dict[str, Any]:
+        """Extract session data from DuckDB."""
+        conn = duckdb.connect(self.db_path)
+        
+        # Get session metadata
+        result = conn.execute("""
+            SELECT metadata FROM session_traces 
+            WHERE session_id = ? AND experiment_id = ?
+        """, [session_id, self.experiment_id]).fetchall()
+        
+        session_data = {}
+        if result:
+            metadata = json.loads(result[0][0]) if isinstance(result[0][0], str) else result[0][0]
+            
+            # Extract achievements
+            for item in metadata:
+                if item.get('metadata_type') == 'SessionMetadum':
+                    data = item.get('data', {})
+                    if 'achievements' in data:
+                        achievements = data['achievements']
+                        session_data['achievements'] = [k for k, v in achievements.items() if v]
+                    if 'num_achievements' in data:
+                        session_data['num_achievements'] = data['num_achievements']
+                    if 'total_reward' in data:
+                        session_data['total_reward'] = data['total_reward']
+                    if 'rollout_length' in data:
+                        session_data['total_steps'] = data['rollout_length']
+        
+        # Get events for action analysis
+        result = conn.execute("""
+            SELECT event_type, metadata, event_metadata
+            FROM events 
+            WHERE session_id = ? 
+            ORDER BY event_time
+        """, [session_id]).fetchall()
+        
+        actions = []
+        invalid_actions = []
+        inventory_changes = []
+        
+        for event_type, metadata, event_metadata in result:
+            if event_type == 'runtime':
+                # Parse action from metadata
+                try:
+                    meta_dict = json.loads(metadata) if isinstance(metadata, str) else metadata
+                    if 'action' in meta_dict:
+                        actions.append(meta_dict['action'])
+                except:
+                    pass
+                
+                # Check for invalid actions in event_metadata
+                if event_metadata:
+                    try:
+                        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
+                        for hook_str in hook_list:
+                            if isinstance(hook_str, str):
+                                import re
+                                if "'hook_name': 'invalid_action'" in hook_str:
+                                    # Extract the actual action name from the description
+                                    action_match = re.search(r"'action':\s*'([^']+)'", hook_str)
+                                    if action_match:
+                                        invalid_actions.append(action_match.group(1))
+                                    else:
+                                        invalid_actions.append("unknown")
+                    except:
+                        pass
+            
+            elif event_type == 'environment':
+                # Check for inventory changes
+                if event_metadata:
+                    try:
+                        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
+                        for hook_str in hook_list:
+                            if isinstance(hook_str, str):
+                                if "'hook_name': 'inventory_increase'" in hook_str:
+                                    inventory_changes.append("inventory_increase")
+                    except:
+                        pass
+        
+        session_data.update({
+            'actions': actions,
+            'invalid_actions': invalid_actions,
+            'inventory_changes': inventory_changes
+        })
+        
+        conn.close()
+        return session_data
+    
+    async def evaluate_session(self, session_id: str) -> Dict[str, List[Judgement]]:
+        """Evaluate a single session."""
+        session_data = self.extract_session_data(session_id)
+        
+        # Run evaluations
+        misunderstood_rules = await self.misunderstood_rules_eval.run(session_data)
+        wasted_time = await self.wasted_time_eval.run(session_data)
+        
+        return {
+            'misunderstood_rules': misunderstood_rules,
+            'wasted_time': wasted_time
+        }
+    
+    async def evaluate_experiment(self) -> Dict[str, Any]:
+        """Evaluate all sessions in the experiment."""
+        conn = duckdb.connect(self.db_path)
+        
+        # Get all session IDs for this experiment
+        result = conn.execute("""
+            SELECT session_id FROM session_traces 
+            WHERE experiment_id = ?
+            ORDER BY session_id
+        """, [self.experiment_id]).fetchall()
+        
+        session_ids = [row[0] for row in result]
+        conn.close()
+        
+        print(f"Evaluating {len(session_ids)} sessions in parallel...")
+        
+        # Create all evaluation tasks
+        tasks = [self.evaluate_session(session_id) for session_id in session_ids]
+        
+        # Run all evaluations in parallel
+        all_results_list = await asyncio.gather(*tasks)
+        
+        # Convert to dictionary
+        all_results = {}
+        for session_id, results in zip(session_ids, all_results_list):
+            all_results[session_id] = results
+        
+        return all_results
+
+async def main():
+    """Example usage."""
+    import sys
+    
+    if len(sys.argv) > 1:
+        experiment_id = sys.argv[1]
+    else:
+        print("Usage: python custom_eval_pipelines.py <experiment_id>")
+        print("Example: python custom_eval_pipelines.py 77022cce-4bda-4415-9bce-0095e4ef2237")
+        return
+    
+    # Use Gemini for evaluation
+    analyzer = CrafterTraceAnalyzer(experiment_id, model_name="gemini-1.5-flash")
+    results = await analyzer.evaluate_experiment()
+    
+    # Print results
+    print("\n" + "="*80)
+    print("EVALUATION RESULTS")
+    print("="*80)
+    
+    for session_id, session_results in results.items():
+        print(f"\nSession: {session_id}")
+        print("-" * 60)
+        
+        for eval_type, judgements in session_results.items():
+            print(f"\n{eval_type.upper()}:")
+            for judgement in judgements:
+                print(f"  Score: {judgement.score:.3f}")
+                print(f"  Reasoning: {judgement.reasoning}")
+                print(f"  Evidence: {judgement.evidence}")
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/plot_hook_frequency.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+Plot the frequency of achievements and invalid actions over time (by step number).
+Terminal-only version.
+"""
+
+import duckdb
+import json
+import re
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+def extract_step_from_metadata(metadata: str) -> int:
+    """Extract step number from event metadata."""
+    try:
+        metadata_dict = json.loads(metadata) if isinstance(metadata, str) else metadata
+        return metadata_dict.get('turn', 0)
+    except:
+        return 0
+
+def parse_hook_metadata(event_metadata: str) -> List[Dict]:
+    """Parse hook metadata from event_metadata string with better error handling."""
+    hooks = []
+    try:
+        # The metadata is stored as a list of strings, each containing a hook dict
+        hook_list = json.loads(event_metadata) if isinstance(event_metadata, str) else event_metadata
+        
+        for hook_str in hook_list:
+            if isinstance(hook_str, str):
+                # Use regex to extract hook_name more reliably
+                hook_name_match = re.search(r"'hook_name':\s*'([^']+)'", hook_str)
+                if hook_name_match:
+                    hook_name = hook_name_match.group(1)
+                    hooks.append({'hook_name': hook_name})
+            else:
+                hooks.append(hook_str)
+    except Exception as e:
+        # Try alternative parsing if JSON fails
+        try:
+            # Look for hook_name patterns in the string
+            hook_names = re.findall(r"'hook_name':\s*'([^']+)'", event_metadata)
+            for hook_name in hook_names:
+                hooks.append({'hook_name': hook_name})
+        except:
+            pass
+    
+    return hooks
+
+def analyze_hook_frequency(experiment_id: str):
+    """Analyze hook frequency over time."""
+    conn = duckdb.connect("crafter_traces.duckdb")
+    
+    print(f"📊 ANALYZING HOOK FREQUENCY OVER TIME")
+    print("=" * 80)
+    print(f"Experiment ID: {experiment_id}")
+    print()
+    
+    # Get events with hook metadata
+    result = conn.execute("""
+        SELECT e.session_id, e.event_type, e.event_metadata, e.metadata
+        FROM events e 
+        JOIN session_traces st ON e.session_id = st.session_id 
+        WHERE st.experiment_id = ? AND e.event_metadata IS NOT NULL
+        ORDER BY e.event_time
+    """, [experiment_id]).fetchall()
+    
+    # Track hook frequency by step
+    step_achievements = defaultdict(int)
+    step_invalid_actions = defaultdict(int)
+    step_inventory_increases = defaultdict(int)
+    
+    # Track by session for more detailed analysis
+    session_data = defaultdict(lambda: {
+        'achievements': defaultdict(int),
+        'invalid_actions': defaultdict(int),
+        'inventory_increases': defaultdict(int)
+    })
+    
+    for row in result:
+        session_id, event_type, event_metadata, metadata = row
+        
+        # Extract step number
+        step = extract_step_from_metadata(metadata)
+        
+        # Parse hook metadata
+        hooks = parse_hook_metadata(event_metadata)
+        
+        for hook in hooks:
+            hook_name = hook.get('hook_name', 'unknown')
+            
+            if hook_name == 'easy_achievement' or hook_name == 'medium_achievement' or hook_name == 'hard_achievement':
+                step_achievements[step] += 1
+                session_data[session_id]['achievements'][step] += 1
+            elif hook_name == 'invalid_action':
+                step_invalid_actions[step] += 1
+                session_data[session_id]['invalid_actions'][step] += 1
+            elif hook_name == 'inventory_increase':
+                step_inventory_increases[step] += 1
+                session_data[session_id]['inventory_increases'][step] += 1
+    
+    # Prepare data for plotting
+    max_step = max(
+        max(step_achievements.keys()) if step_achievements else 0,
+        max(step_invalid_actions.keys()) if step_invalid_actions else 0,
+        max(step_inventory_increases.keys()) if step_inventory_increases else 0
+    )
+    
+    steps = list(range(max_step + 1))
+    achievement_freq = [step_achievements[step] for step in steps]
+    invalid_action_freq = [step_invalid_actions[step] for step in steps]
+    inventory_freq = [step_inventory_increases[step] for step in steps]
+    
+    # Print summary statistics
+    print("📈 SUMMARY STATISTICS")
+    print("-" * 50)
+    print(f"Total steps analyzed: {max_step + 1}")
+    print(f"Total achievements: {sum(achievement_freq)}")
+    print(f"Total invalid actions: {sum(invalid_action_freq)}")
+    print(f"Total inventory increases: {sum(inventory_freq)}")
+    print()
+    
+    print("🏆 ACHIEVEMENT ANALYSIS")
+    print("-" * 50)
+    achievement_steps = [step for step, freq in step_achievements.items() if freq > 0]
+    if achievement_steps:
+        print(f"Achievements occur at steps: {sorted(achievement_steps)}")
+        print(f"Most common achievement step: {max(step_achievements.items(), key=lambda x: x[1])}")
+    else:
+        print("No achievements found")
+    print()
+    
+    print("❌ INVALID ACTION ANALYSIS")
+    print("-" * 50)
+    invalid_steps = [step for step, freq in step_invalid_actions.items() if freq > 0]
+    if invalid_steps:
+        print(f"Invalid actions occur at steps: {sorted(invalid_steps)}")
+        print(f"Most common invalid action step: {max(step_invalid_actions.items(), key=lambda x: x[1])}")
+    else:
+        print("No invalid actions found")
+    print()
+    
+    print("📦 INVENTORY ANALYSIS")
+    print("-" * 50)
+    inventory_steps = [step for step, freq in step_inventory_increases.items() if freq > 0]
+    if inventory_steps:
+        print(f"Inventory increases occur at steps: {sorted(inventory_steps)}")
+        print(f"Most common inventory increase step: {max(step_inventory_increases.items(), key=lambda x: x[1])}")
+    else:
+        print("No inventory increases found")
+    
+    # Create ASCII chart
+    print("\n📊 ASCII FREQUENCY CHART")
+    print("=" * 80)
+    print("Step | Achievements | Invalid Actions | Inventory")
+    print("-" * 80)
+    
+    for step in steps:
+        achievements = step_achievements[step]
+        invalid_actions = step_invalid_actions[step]
+        inventory = step_inventory_increases[step]
+        
+        if achievements > 0 or invalid_actions > 0 or inventory > 0:
+            print(f"{step:4d} | {achievements:11d} | {invalid_actions:14d} | {inventory:9d}")
+    
+    # Session-by-session breakdown
+    print("\n📋 SESSION-BY-SESSION BREAKDOWN")
+    print("-" * 50)
+    for session_id, data in session_data.items():
+        print(f"\nSession: {session_id}")
+        if data['achievements']:
+            print(f"  Achievements: {dict(data['achievements'])}")
+        if data['invalid_actions']:
+            print(f"  Invalid actions: {dict(data['invalid_actions'])}")
+        if data['inventory_increases']:
+            print(f"  Inventory increases: {dict(data['inventory_increases'])}")
+    
+    conn.close()
+    
+    return {
+        'steps': steps,
+        'achievement_freq': achievement_freq,
+        'invalid_action_freq': invalid_action_freq,
+        'inventory_freq': inventory_freq,
+        'session_data': session_data
+    }
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) > 1:
+        experiment_id = sys.argv[1]
+        analyze_hook_frequency(experiment_id)
+    else:
+        print("Usage: python plot_hook_frequency.py <experiment_id>")
+        print("Example: python plot_hook_frequency.py 77022cce-4bda-4415-9bce-0095e4ef2237") 

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/seed_analysis_summary.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+Summary analysis of seed performance comparison between GPT-4.1-NANO and GPT-4.1-MINI.
+"""
+
+def print_summary():
+    print("🎯 KEY FINDINGS: SEED PERFORMANCE COMPARISON")
+    print("=" * 60)
+    
+    print("\n📊 OVERALL PERFORMANCE:")
+    print("  • GPT-4.1-MINI: 12 total achievements across 9 instances")
+    print("  • GPT-4.1-NANO:  2 total achievements across 2 instances")
+    print("  • MINI wins: 9 out of 10 instances (90% win rate)")
+    print("  • NANO wins: 0 out of 10 instances")
+    print("  • Ties: 1 instance (instance 9)")
+    
+    print("\n🏆 INSTANCE-BY-INSTANCE BREAKDOWN:")
+    print("  • Instance 1 (Seed 43): MINI wins (collect_wood vs 0)")
+    print("  • Instance 2 (Seed 44): MINI wins (collect_wood vs 0)")
+    print("  • Instance 3 (Seed 45): MINI wins (collect_sapling vs 0)")
+    print("  • Instance 4 (Seed 46): MINI wins (collect_wood vs 0)")
+    print("  • Instance 5 (Seed 47): MINI wins (collect_wood vs 0)")
+    print("  • Instance 6 (Seed 48): MINI wins (collect_sapling + eat_cow vs collect_sapling)")
+    print("  • Instance 7 (Seed 49): MINI wins (collect_sapling + collect_wood vs 0)")
+    print("  • Instance 8 (Seed 50): MINI wins (collect_wood vs 0)")
+    print("  • Instance 9 (Seed 51): TIE (0 vs 0)")
+    print("  • Instance 10 (Seed 52): MINI wins (collect_sapling + collect_wood vs collect_wood)")
+    
+    print("\n🎯 ACHIEVEMENT TYPE ANALYSIS:")
+    print("  • collect_wood: MINI 7, NANO 1 (MINI dominates)")
+    print("  • collect_sapling: MINI 4, NANO 1 (MINI dominates)")
+    print("  • eat_cow: MINI 1, NANO 0 (MINI only)")
+    print("  • All other achievements: 0 for both models")
+    
+    print("\n🔍 PATTERNS OBSERVED:")
+    print("  1. MINI consistently outperforms NANO across almost all seeds")
+    print("  2. MINI achieves more complex combinations (e.g., collect_sapling + eat_cow)")
+    print("  3. NANO struggles with basic achievements (only 2 total vs MINI's 12)")
+    print("  4. Both models struggle with advanced achievements (iron, diamond, etc.)")
+    print("  5. MINI shows better exploration and resource gathering capabilities")
+    
+    print("\n📈 IMPLICATIONS:")
+    print("  • MINI demonstrates significantly better reasoning and planning")
+    print("  • MINI's larger context window may enable better multi-step planning")
+    print("  • NANO may be hitting context limits or reasoning limitations")
+    print("  • Both models struggle with complex crafting and combat achievements")
+    print("  • The performance gap is consistent across different environment seeds")
+    
+    print("\n🎲 RANDOMNESS ANALYSIS:")
+    print("  • Seeds 43-52 were tested (10 different environments)")
+    print("  • MINI wins 9/10 = 90% win rate")
+    print("  • This suggests the performance difference is robust, not random")
+    print("  • Only instance 9 was a tie, suggesting MINI's advantage is consistent")
+
+if __name__ == "__main__":
+    print_summary()