synth-ai 0.2.4.dev7__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/check_hook_storage.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+Check where hook results are being stored in the database.
+"""
+
+import duckdb
+import json
+
+def check_hook_storage():
+    conn = duckdb.connect("crafter_traces.duckdb")
+    
+    # Get recent experiment
+    result = conn.execute("SELECT id, name FROM experiments ORDER BY created_at DESC LIMIT 1").fetchall()
+    exp_id = result[0][0]
+    exp_name = result[0][1]
+    
+    print(f"🔍 Checking hook storage for experiment: {exp_name} ({exp_id})")
+    print("=" * 60)
+    
+    # Check session metadata
+    print("\n📋 SESSION METADATA ANALYSIS:")
+    result = conn.execute("SELECT session_id, metadata FROM session_traces WHERE experiment_id = ?", [exp_id]).fetchall()
+    
+    for row in result:
+        session_id, metadata = row
+        metadata_list = json.loads(metadata) if isinstance(metadata, str) else metadata
+        
+        print(f"\nSession: {session_id}")
+        for i, item in enumerate(metadata_list):
+            metadata_type = item.get('metadata_type', 'unknown')
+            data = item.get('data', {})
+            print(f"  Item {i}: {metadata_type}")
+            print(f"    Keys: {list(data.keys())}")
+            
+            # Check for hook-related data
+            if 'achievements' in data:
+                achievements = data['achievements']
+                unlocked = [k for k, v in achievements.items() if v]
+                print(f"    Achievements: {unlocked}")
+            
+            if 'num_achievements' in data:
+                print(f"    Num achievements: {data['num_achievements']}")
+    
+    # Check if there are any hook events
+    print(f"\n🔍 HOOK EVENTS CHECK:")
+    result = conn.execute("""
+        SELECT COUNT(*) 
+        FROM events e 
+        JOIN session_traces st ON e.session_id = st.session_id 
+        WHERE st.experiment_id = ? AND e.event_type = 'hook'
+    """, [exp_id]).fetchall()
+    
+    hook_count = result[0][0]
+    print(f"Hook events found: {hook_count}")
+    
+    if hook_count > 0:
+        print("Sample hook events:")
+        result = conn.execute("""
+            SELECT e.session_id, e.metadata 
+            FROM events e 
+            JOIN session_traces st ON e.session_id = st.session_id 
+            WHERE st.experiment_id = ? AND e.event_type = 'hook'
+            LIMIT 3
+        """, [exp_id]).fetchall()
+        
+        for row in result:
+            session_id, metadata = row
+            print(f"  Session {session_id}: {metadata}")
+    
+    # Check for any other hook-related data
+    print(f"\n🔍 OTHER HOOK DATA:")
+    result = conn.execute("""
+        SELECT e.event_type, COUNT(*) 
+        FROM events e 
+        JOIN session_traces st ON e.session_id = st.session_id 
+        WHERE st.experiment_id = ? 
+        GROUP BY e.event_type
+    """, [exp_id]).fetchall()
+    
+    print("Event types in experiment:")
+    for event_type, count in result:
+        print(f"  {event_type}: {count}")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    check_hook_storage() 

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/check_seeds.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+Check what seeds are being used in the two experiments.
+"""
+
+import duckdb
+import json
+
+# Experiment IDs
+EXPERIMENTS = {
+    "gpt-4.1-nano": "194a3cd2-ecd3-4081-b46d-a7883e4a86f9",
+    "gpt-4.1-mini": "da74a769-b33d-4b60-ae2a-52a4b67b3f35"
+}
+
+def check_seeds():
+    """Check seeds for both experiments."""
+    conn = duckdb.connect("crafter_traces.duckdb")
+    
+    for model_name, exp_id in EXPERIMENTS.items():
+        print(f"\n🔍 {model_name.upper()} EXPERIMENT SEEDS")
+        print("-" * 50)
+        
+        # Get all sessions for this experiment
+        query = """
+        SELECT session_id, metadata 
+        FROM session_traces 
+        WHERE experiment_id = ?
+        ORDER BY session_id
+        """
+        
+        results = conn.execute(query, [exp_id]).fetchall()
+        
+        seeds = []
+        for session_id, metadata in results:
+            if metadata:
+                try:
+                    metadata_list = json.loads(metadata) if isinstance(metadata, str) else metadata
+                    
+                    for meta_item in metadata_list:
+                        if isinstance(meta_item, dict) and meta_item.get('metadata_type') == 'SessionMetadum':
+                            data = meta_item.get('data', {})
+                            
+                            # Look for seed information
+                            if 'seed' in data:
+                                seeds.append({
+                                    'session_id': session_id,
+                                    'seed': data['seed']
+                                })
+                            elif 'instance_num' in data:
+                                # Sometimes seed is derived from instance_num
+                                seeds.append({
+                                    'session_id': session_id,
+                                    'instance_num': data['instance_num']
+                                })
+                except Exception as e:
+                    print(f"Error parsing metadata for {session_id}: {e}")
+        
+        if seeds:
+            print(f"Found {len(seeds)} sessions with seed info:")
+            for seed_info in seeds:
+                print(f"  {seed_info}")
+        else:
+            print("No explicit seed information found in metadata")
+            print("Checking for instance numbers...")
+            
+            # Check for instance numbers as a proxy for seeds
+            instance_nums = []
+            for session_id, metadata in results:
+                if metadata:
+                    try:
+                        metadata_list = json.loads(metadata) if isinstance(metadata, str) else metadata
+                        for meta_item in metadata_list:
+                            if isinstance(meta_item, dict) and meta_item.get('metadata_type') == 'SessionMetadum':
+                                data = meta_item.get('data', {})
+                                if 'instance_num' in data:
+                                    instance_nums.append(data['instance_num'])
+                    except:
+                        pass
+            
+            if instance_nums:
+                print(f"Instance numbers found: {sorted(instance_nums)}")
+            else:
+                print("No instance numbers found either")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    check_seeds() 

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/old/compare_seed_performance.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+Compare which seeds/instances performed better in nano vs mini.
+Shows detailed performance breakdown by instance.
+"""
+
+import duckdb
+import json
+from typing import Dict, List, Any
+
+# Experiment IDs
+EXPERIMENTS = {
+    "gpt-4.1-nano": "194a3cd2-ecd3-4081-b46d-a7883e4a86f9",
+    "gpt-4.1-mini": "da74a769-b33d-4b60-ae2a-52a4b67b3f35"
+}
+
+def get_instance_performance(conn, experiment_id: str) -> Dict[int, Dict[str, Any]]:
+    """Get performance data for each instance in an experiment."""
+    query = """
+    SELECT session_id, metadata 
+    FROM session_traces 
+    WHERE experiment_id = ?
+    ORDER BY session_id
+    """
+    
+    results = conn.execute(query, [experiment_id]).fetchall()
+    
+    instance_data = {}
+    
+    for session_id, metadata in results:
+        if metadata:
+            try:
+                metadata_list = json.loads(metadata) if isinstance(metadata, str) else metadata
+                
+                # Look for instance_num in first metadata item
+                instance_num = None
+                for meta_item in metadata_list:
+                    if isinstance(meta_item, dict) and meta_item.get('metadata_type') == 'SessionMetadum':
+                        data = meta_item.get('data', {})
+                        if 'instance_num' in data:
+                            instance_num = data['instance_num']
+                            break
+                
+                # Look for achievement data in any metadata item
+                achievements = {}
+                num_achievements = 0
+                total_reward = 0.0
+                rollout_length = 0
+                terminated = False
+                
+                for meta_item in metadata_list:
+                    if isinstance(meta_item, dict) and meta_item.get('metadata_type') == 'SessionMetadum':
+                        data = meta_item.get('data', {})
+                        
+                        if 'achievements' in data:
+                            achievements = data.get('achievements', {})
+                            num_achievements = data.get('num_achievements', 0)
+                            total_reward = data.get('total_reward', 0.0)
+                            rollout_length = data.get('rollout_length', 0)
+                            terminated = data.get('terminated', False)
+                            break
+                
+                if instance_num is not None:
+                    unlocked_achievements = [ach for ach, unlocked in achievements.items() if unlocked]
+                    
+                    instance_data[instance_num] = {
+                        'session_id': session_id,
+                        'num_achievements': num_achievements,
+                        'unlocked_achievements': unlocked_achievements,
+                        'total_reward': total_reward,
+                        'rollout_length': rollout_length,
+                        'terminated': terminated,
+                        'all_achievements': achievements
+                    }
+            except Exception as e:
+                print(f"Error parsing metadata for {session_id}: {e}")
+    
+    return instance_data
+
+def compare_instance_performance():
+    """Compare performance between nano and mini for each instance."""
+    conn = duckdb.connect("crafter_traces.duckdb")
+    
+    # Get performance data for both experiments
+    nano_data = get_instance_performance(conn, EXPERIMENTS["gpt-4.1-nano"])
+    mini_data = get_instance_performance(conn, EXPERIMENTS["gpt-4.1-mini"])
+    
+    print("🔍 INSTANCE-BY-INSTANCE PERFORMANCE COMPARISON")
+    print("=" * 80)
+    
+    # Compare each instance
+    nano_wins = []
+    mini_wins = []
+    ties = []
+    
+    for instance_num in range(1, 11):  # Instances 1-10
+        nano_perf = nano_data.get(instance_num, {})
+        mini_perf = mini_data.get(instance_num, {})
+        
+        nano_achievements = nano_perf.get('num_achievements', 0)
+        mini_achievements = mini_perf.get('num_achievements', 0)
+        
+        nano_unlocked = nano_perf.get('unlocked_achievements', [])
+        mini_unlocked = mini_perf.get('unlocked_achievements', [])
+        
+        print(f"\n📊 Instance {instance_num} (Seed {42 + instance_num}):")
+        print(f"  GPT-4.1-NANO:  {nano_achievements} achievements - {nano_unlocked}")
+        print(f"  GPT-4.1-MINI:  {mini_achievements} achievements - {mini_unlocked}")
+        
+        if nano_achievements > mini_achievements:
+            nano_wins.append(instance_num)
+            print(f"  🏆 NANO WINS")
+        elif mini_achievements > nano_achievements:
+            mini_wins.append(instance_num)
+            print(f"  🏆 MINI WINS")
+        else:
+            ties.append(instance_num)
+            print(f"  🤝 TIE")
+    
+    # Summary statistics
+    print(f"\n📈 PERFORMANCE SUMMARY")
+    print("=" * 50)
+    print(f"NANO wins: {len(nano_wins)} instances - {nano_wins}")
+    print(f"MINI wins: {len(mini_wins)} instances - {mini_wins}")
+    print(f"Ties: {len(ties)} instances - {ties}")
+    
+    # Calculate total achievements by model
+    total_nano_achievements = sum(nano_data[i].get('num_achievements', 0) for i in range(1, 11))
+    total_mini_achievements = sum(mini_data[i].get('num_achievements', 0) for i in range(1, 11))
+    
+    print(f"\n🏆 TOTAL ACHIEVEMENTS:")
+    print(f"  GPT-4.1-NANO:  {total_nano_achievements}")
+    print(f"  GPT-4.1-MINI:  {total_mini_achievements}")
+    
+    # Show which instances each model dominated
+    print(f"\n🎯 INSTANCE DOMINANCE:")
+    if nano_wins:
+        print(f"  NANO dominated: {nano_wins}")
+    if mini_wins:
+        print(f"  MINI dominated: {mini_wins}")
+    if ties:
+        print(f"  Tied instances: {ties}")
+    
+    # Detailed breakdown by achievement type
+    print(f"\n📊 ACHIEVEMENT BREAKDOWN BY INSTANCE:")
+    print("-" * 60)
+    
+    all_achievements = set()
+    for data in [nano_data, mini_data]:
+        for instance_data in data.values():
+            all_achievements.update(instance_data.get('all_achievements', {}).keys())
+    
+    achievement_types = sorted(all_achievements)
+    
+    for achievement in achievement_types:
+        nano_count = sum(1 for data in nano_data.values() 
+                        if data.get('all_achievements', {}).get(achievement, False))
+        mini_count = sum(1 for data in mini_data.values() 
+                        if data.get('all_achievements', {}).get(achievement, False))
+        
+        print(f"{achievement:20} | NANO: {nano_count:2d} | MINI: {mini_count:2d} | {'MINI' if mini_count > nano_count else 'NANO' if nano_count > mini_count else 'TIE'}")
+    
+    conn.close()
+
+def show_detailed_instance_analysis():
+    """Show detailed analysis of each instance."""
+    conn = duckdb.connect("crafter_traces.duckdb")
+    
+    nano_data = get_instance_performance(conn, EXPERIMENTS["gpt-4.1-nano"])
+    mini_data = get_instance_performance(conn, EXPERIMENTS["gpt-4.1-mini"])
+    
+    print(f"\n🔍 DETAILED INSTANCE ANALYSIS")
+    print("=" * 80)
+    
+    for instance_num in range(1, 11):
+        nano_perf = nano_data.get(instance_num, {})
+        mini_perf = mini_data.get(instance_num, {})
+        
+        print(f"\n📋 Instance {instance_num} (Seed {42 + instance_num}):")
+        print(f"  NANO: {nano_perf.get('num_achievements', 0)} achievements, reward: {nano_perf.get('total_reward', 0.0):.2f}, length: {nano_perf.get('rollout_length', 0)}")
+        print(f"  MINI: {mini_perf.get('num_achievements', 0)} achievements, reward: {mini_perf.get('total_reward', 0.0):.2f}, length: {mini_perf.get('rollout_length', 0)}")
+        
+        # Show specific achievements
+        nano_achievements = nano_perf.get('unlocked_achievements', [])
+        mini_achievements = mini_perf.get('unlocked_achievements', [])
+        
+        if nano_achievements or mini_achievements:
+            print(f"  NANO unlocked: {nano_achievements}")
+            print(f"  MINI unlocked: {mini_achievements}")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    compare_instance_performance()
+    show_detailed_instance_analysis()