synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.5__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_latest_run.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""Analyze the latest run to understand why no achievements were unlocked."""
+
+import duckdb
+import json
+from pathlib import Path
+from collections import Counter, defaultdict
+from datetime import datetime
+
+def analyze_latest_run(db_path: str):
+    """Analyze the most recent run."""
+    conn = duckdb.connect(db_path, read_only=True)
+    
+    print("🔍 Analyzing latest run with 97 steps and 0 achievements...\n")
+    
+    # Get the latest experiment
+    latest_experiment = conn.execute("""
+        SELECT experiment_id, experiment_name, created_at
+        FROM experiments
+        ORDER BY created_at DESC
+        LIMIT 1
+    """).fetchone()
+    
+    if latest_experiment:
+        exp_id, exp_name, created_at = latest_experiment
+        print(f"📊 Latest Experiment: {exp_name}")
+        print(f"   ID: {exp_id}")
+        print(f"   Created: {created_at}")
+    
+    # Get all sessions from latest experiment
+    sessions = conn.execute("""
+        SELECT DISTINCT s.session_id, s.num_timesteps, s.num_events
+        FROM session_traces s
+        JOIN events e ON s.session_id = e.session_id
+        WHERE e.timestep_id IN (
+            SELECT id FROM session_timesteps 
+            WHERE experiment_id = ?
+        )
+        ORDER BY s.created_at DESC
+    """, [exp_id]).fetchall()
+    
+    print(f"\n📊 Sessions in latest experiment: {len(sessions)}")
+    
+    for session_id, num_timesteps, num_events in sessions:
+        print(f"\n{'='*60}")
+        print(f"SESSION: {session_id}")
+        print(f"Timesteps: {num_timesteps}, Events: {num_events}")
+        print(f"{'='*60}")
+        
+        # Get all events for this session
+        events = conn.execute("""
+            SELECT event_type, metadata, system_state_after, reward
+            FROM events
+            WHERE session_id = ?
+            ORDER BY id
+        """, [session_id]).fetchall()
+        
+        # Analyze actions taken
+        actions_taken = []
+        action_sequences = []
+        current_sequence = []
+        total_reward = 0
+        achievements_over_time = []
+        
+        for event_type, metadata_str, state_after_str, reward in events:
+            if reward:
+                total_reward += reward
+                
+            if metadata_str:
+                metadata = json.loads(metadata_str)
+                
+                # Track runtime actions
+                if event_type == 'runtime' and 'action_name' in metadata:
+                    action = metadata['action_name']
+                    actions_taken.append(action)
+                    current_sequence.append(action)
+                    
+                    # Group actions into sequences (reset on certain actions)
+                    if action in ['do', 'make_wood_pickaxe', 'place_table', 'sleep']:
+                        if len(current_sequence) > 1:
+                            action_sequences.append(current_sequence[:-1])
+                        current_sequence = [action]
+            
+            # Check achievements
+            if state_after_str:
+                state_after = json.loads(state_after_str)
+                if 'public_state' in state_after:
+                    ps = state_after['public_state']
+                    if 'achievements_status' in ps:
+                        unlocked = [k for k, v in ps['achievements_status'].items() if v]
+                        achievements_over_time.append(len(unlocked))
+                    
+                    # Check inventory
+                    if 'inventory' in ps and len(actions_taken) % 20 == 0:  # Sample every 20 actions
+                        inv = ps['inventory']
+                        non_zero = {k: v for k, v in inv.items() if v > 0}
+                        if non_zero:
+                            print(f"\n📦 Inventory at action {len(actions_taken)}: {non_zero}")
+        
+        # Action analysis
+        print(f"\n📊 ACTION ANALYSIS")
+        print(f"Total actions taken: {len(actions_taken)}")
+        
+        if actions_taken:
+            action_counts = Counter(actions_taken)
+            print(f"\nAction distribution:")
+            for action, count in action_counts.most_common():
+                percentage = (count / len(actions_taken)) * 100
+                print(f"  {action:20} {count:4} ({percentage:5.1f}%)")
+            
+            # Check for repetitive patterns
+            print(f"\n🔄 REPETITIVE PATTERNS")
+            # Find consecutive repeated actions
+            consecutive_repeats = []
+            if actions_taken:
+                current_action = actions_taken[0]
+                repeat_count = 1
+                
+                for action in actions_taken[1:]:
+                    if action == current_action:
+                        repeat_count += 1
+                    else:
+                        if repeat_count > 3:
+                            consecutive_repeats.append((current_action, repeat_count))
+                        current_action = action
+                        repeat_count = 1
+                
+                if repeat_count > 3:
+                    consecutive_repeats.append((current_action, repeat_count))
+            
+            if consecutive_repeats:
+                print("Found repetitive sequences:")
+                for action, count in consecutive_repeats[:5]:
+                    print(f"  {action} repeated {count} times consecutively")
+            else:
+                print("No significant repetitive patterns found")
+            
+            # Check action sequences
+            if action_sequences:
+                print(f"\n🎯 ACTION SEQUENCES (movement -> action):")
+                seq_counter = Counter([' → '.join(seq) for seq in action_sequences if len(seq) <= 5])
+                for seq, count in seq_counter.most_common(5):
+                    print(f"  {seq}: {count} times")
+        
+        # Achievement progress
+        print(f"\n🏆 ACHIEVEMENT PROGRESS")
+        print(f"Total reward: {total_reward}")
+        if achievements_over_time:
+            max_achievements = max(achievements_over_time)
+            print(f"Max achievements reached: {max_achievements}")
+            
+            # Find when achievements were unlocked
+            achievement_unlocks = []
+            prev_count = 0
+            for i, count in enumerate(achievements_over_time):
+                if count > prev_count:
+                    achievement_unlocks.append((i, count))
+                    prev_count = count
+            
+            if achievement_unlocks:
+                print("Achievement unlock timeline:")
+                for step, count in achievement_unlocks:
+                    print(f"  Step {step}: {count} achievements")
+        else:
+            print("No achievement data found")
+        
+        # Check final state
+        final_event = events[-1] if events else None
+        if final_event and final_event[2]:  # state_after
+            final_state = json.loads(final_event[2])
+            if 'public_state' in final_state:
+                ps = final_state['public_state']
+                
+                print(f"\n📊 FINAL STATE")
+                
+                # Final inventory
+                if 'inventory' in ps:
+                    inv = ps['inventory']
+                    non_zero = {k: v for k, v in inv.items() if v > 0}
+                    print(f"Final inventory: {non_zero if non_zero else 'Empty'}")
+                
+                # Final achievements
+                if 'achievements_status' in ps:
+                    unlocked = [k for k, v in ps['achievements_status'].items() if v]
+                    print(f"Final achievements: {unlocked if unlocked else 'None'}")
+                
+                # Player stats
+                if 'health' in ps:
+                    print(f"Final stats: Health={ps.get('health', '?')}, Food={ps.get('food', '?')}, Energy={ps.get('energy', '?')}")
+    
+    # Check for any generation/LM events
+    print(f"\n{'='*60}")
+    print("📊 LM/GENERATION EVENT CHECK")
+    print(f"{'='*60}")
+    
+    lm_events = conn.execute("""
+        SELECT COUNT(*) 
+        FROM events 
+        WHERE session_id IN (
+            SELECT DISTINCT s.session_id
+            FROM session_traces s
+            JOIN events e ON s.session_id = e.session_id
+            WHERE e.timestep_id IN (
+                SELECT id FROM session_timesteps 
+                WHERE experiment_id = ?
+            )
+        ) AND (event_type = 'lm' OR event_type = 'generation')
+    """, [exp_id]).fetchone()[0]
+    
+    print(f"LM/Generation events in this experiment: {lm_events}")
+    
+    if lm_events == 0:
+        print("⚠️  No LM events found - the agent may not be generating proper decisions")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    db_path = "./traces_v2_synth/traces.duckdb"
+    if Path(db_path).exists():
+        analyze_latest_run(db_path)
+    else:
+        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_lm_traces.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Analyze LM traces to see what the agent is actually doing."""
+
+import duckdb
+import json
+from pathlib import Path
+from collections import Counter, defaultdict
+
+def analyze_lm_traces(db_path: str):
+    """Analyze LM traces to understand agent behavior."""
+    conn = duckdb.connect(db_path, read_only=True)
+    
+    print("🔍 Analyzing LM traces after fix...\n")
+    
+    # First, check if we have any LM events
+    print("📊 CHECKING FOR LM EVENTS")
+    print("=" * 50)
+    
+    lm_events = conn.execute("""
+        SELECT COUNT(*) 
+        FROM events 
+        WHERE event_type = 'lm' OR event_type = 'generation'
+    """).fetchone()[0]
+    
+    print(f"LM/Generation events found: {lm_events}")
+    
+    # Get all unique event types
+    event_types = conn.execute("""
+        SELECT DISTINCT event_type, COUNT(*) as count
+        FROM events
+        GROUP BY event_type
+        ORDER BY count DESC
+    """).fetchall()
+    
+    print("\nAll event types:")
+    for event_type, count in event_types:
+        print(f"  {event_type}: {count}")
+    
+    # Analyze sessions
+    print("\n📊 SESSION ANALYSIS")
+    print("=" * 50)
+    
+    sessions = conn.execute("""
+        SELECT DISTINCT session_id
+        FROM events
+        ORDER BY session_id
+        LIMIT 5
+    """).fetchall()
+    
+    for i, (session_id,) in enumerate(sessions):
+        print(f"\n--- Session {i+1}: {session_id} ---")
+        
+        # Get all events for this session
+        events = conn.execute("""
+            SELECT event_type, metadata, system_state_after
+            FROM events
+            WHERE session_id = ?
+            ORDER BY id
+        """, [session_id]).fetchall()
+        
+        print(f"Total events: {len(events)}")
+        
+        # Count event types
+        event_type_counts = Counter()
+        actions_taken = []
+        achievements = set()
+        
+        for event_type, metadata_str, state_after_str in events:
+            event_type_counts[event_type] += 1
+            
+            if metadata_str:
+                metadata = json.loads(metadata_str)
+                
+                # Track actions
+                if 'action_name' in metadata:
+                    actions_taken.append(metadata['action_name'])
+                elif 'action' in metadata:
+                    actions_taken.append(metadata['action'])
+            
+            # Check for achievements
+            if state_after_str:
+                state_after = json.loads(state_after_str)
+                if 'public_state' in state_after:
+                    public_state = state_after['public_state']
+                    if 'achievements_status' in public_state:
+                        for ach, unlocked in public_state['achievements_status'].items():
+                            if unlocked:
+                                achievements.add(ach)
+        
+        print(f"Event type distribution: {dict(event_type_counts)}")
+        print(f"Total actions: {len(actions_taken)}")
+        if actions_taken:
+            action_counts = Counter(actions_taken)
+            print(f"Top actions: {action_counts.most_common(5)}")
+        print(f"Achievements unlocked: {achievements if achievements else 'None'}")
+    
+    # Look for any LM-related metadata
+    print("\n📊 LM METADATA ANALYSIS")
+    print("=" * 50)
+    
+    # Check for model information in metadata
+    model_events = conn.execute("""
+        SELECT metadata
+        FROM events
+        WHERE metadata LIKE '%model%'
+        LIMIT 10
+    """).fetchall()
+    
+    if model_events:
+        print(f"Found {len(model_events)} events with model metadata")
+        for i, (metadata_str,) in enumerate(model_events[:3]):
+            metadata = json.loads(metadata_str)
+            print(f"\nEvent {i+1} metadata keys: {list(metadata.keys())}")
+            if 'model' in metadata:
+                print(f"  Model: {metadata['model']}")
+    else:
+        print("No events with model metadata found")
+    
+    # Check messages table
+    print("\n📊 MESSAGES ANALYSIS")
+    print("=" * 50)
+    
+    message_count = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
+    print(f"Total messages: {message_count}")
+    
+    if message_count > 0:
+        messages = conn.execute("""
+            SELECT message_type, content
+            FROM messages
+            LIMIT 10
+        """).fetchall()
+        
+        for i, (msg_type, content_str) in enumerate(messages[:5]):
+            content = json.loads(content_str)
+            print(f"\nMessage {i+1} ({msg_type}):")
+            if isinstance(content, dict):
+                print(f"  Keys: {list(content.keys())}")
+                if 'payload' in content:
+                    payload = content['payload']
+                    if isinstance(payload, dict) and 'inventory' in payload:
+                        # Show non-zero inventory items
+                        inv = payload['inventory']
+                        non_zero = {k: v for k, v in inv.items() if v > 0}
+                        if non_zero:
+                            print(f"  Non-zero inventory: {non_zero}")
+    
+    # Check for tool calls or function calls
+    print("\n📊 TOOL CALL ANALYSIS")
+    print("=" * 50)
+    
+    tool_events = conn.execute("""
+        SELECT metadata
+        FROM events
+        WHERE metadata LIKE '%tool%' OR metadata LIKE '%function%'
+        LIMIT 20
+    """).fetchall()
+    
+    if tool_events:
+        print(f"Found {len(tool_events)} events with tool/function mentions")
+        tool_names = Counter()
+        
+        for (metadata_str,) in tool_events:
+            metadata = json.loads(metadata_str)
+            if 'tool_name' in metadata:
+                tool_names[metadata['tool_name']] += 1
+            elif 'function' in metadata:
+                tool_names[metadata['function']] += 1
+        
+        if tool_names:
+            print("Tool usage:")
+            for tool, count in tool_names.most_common():
+                print(f"  {tool}: {count}")
+    else:
+        print("No tool/function events found")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    db_path = "./traces_v2_synth/traces.duckdb"
+    if Path(db_path).exists():
+        analyze_lm_traces(db_path)
+    else:
+        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_no_rewards.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""Analyze why there are no rewards or achievements."""
+
+import duckdb
+import json
+from pathlib import Path
+from collections import Counter, defaultdict
+
+def analyze_no_rewards(db_path: str):
+    """Analyze why there are no rewards despite actions being taken."""
+    conn = duckdb.connect(db_path, read_only=True)
+    
+    print("🔍 Analyzing why there are no rewards or achievements...\n")
+    
+    # Get the latest experiment
+    latest_exp = conn.execute("""
+        SELECT experiment_id, experiment_name
+        FROM experiments
+        ORDER BY created_at DESC
+        LIMIT 1
+    """).fetchone()
+    
+    if not latest_exp:
+        print("No experiments found")
+        return
+        
+    exp_id, exp_name = latest_exp
+    print(f"📊 Latest Experiment: {exp_name}")
+    print(f"   ID: {exp_id}\n")
+    
+    # Get all sessions from latest experiment
+    sessions = conn.execute("""
+        SELECT DISTINCT e.session_id
+        FROM events e
+        JOIN session_timesteps st ON e.timestep_id = st.id
+        WHERE st.experiment_id = ?
+        ORDER BY e.session_id
+    """, [exp_id]).fetchall()
+    
+    print(f"Found {len(sessions)} sessions\n")
+    
+    # Analyze each session
+    for i, (session_id,) in enumerate(sessions[:3]):  # First 3 sessions
+        print(f"\n{'='*60}")
+        print(f"SESSION {i}: {session_id}")
+        print(f"{'='*60}")
+        
+        # Get all events for this session
+        events = conn.execute("""
+            SELECT event_type, metadata, system_state_after, reward
+            FROM events
+            WHERE session_id = ?
+            ORDER BY id
+        """, [session_id]).fetchall()
+        
+        # Track actions and their results
+        action_results = []
+        total_reward = 0
+        achievements_timeline = []
+        inventory_timeline = []
+        
+        for event_type, metadata_str, state_after_str, reward in events:
+            if event_type == 'runtime' and metadata_str:
+                metadata = json.loads(metadata_str)
+                action_name = metadata.get('action_name', 'unknown')
+                
+                # Look for the corresponding environment event
+                result = {
+                    'action': action_name,
+                    'reward': reward or 0,
+                    'achievements_unlocked': []
+                }
+                
+                if state_after_str:
+                    state_after = json.loads(state_after_str)
+                    if 'public_state' in state_after:
+                        ps = state_after['public_state']
+                        
+                        # Check achievements
+                        if 'achievements_status' in ps:
+                            unlocked = [k for k, v in ps['achievements_status'].items() if v]
+                            achievements_timeline.append((action_name, unlocked))
+                            if len(unlocked) > len(result['achievements_unlocked']):
+                                result['achievements_unlocked'] = unlocked
+                        
+                        # Check inventory
+                        if 'inventory' in ps:
+                            inv = ps['inventory']
+                            non_zero = {k: v for k, v in inv.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
+                            if non_zero:
+                                inventory_timeline.append((action_name, non_zero))
+                
+                action_results.append(result)
+                if reward:
+                    total_reward += reward
+        
+        # Analyze action effectiveness
+        print(f"\n📊 ACTION ANALYSIS")
+        print(f"Total actions: {len(action_results)}")
+        print(f"Total reward: {total_reward}")
+        
+        # Count actions by type
+        action_counts = Counter([r['action'] for r in action_results])
+        print(f"\nAction distribution:")
+        for action, count in action_counts.most_common(10):
+            print(f"  {action:20} {count:3}")
+        
+        # Check for successful resource collection
+        print(f"\n📦 RESOURCE COLLECTION")
+        successful_collections = []
+        for i, (action, inv) in enumerate(inventory_timeline):
+            if i > 0 and inv != inventory_timeline[i-1][1]:
+                # Inventory changed
+                prev_inv = inventory_timeline[i-1][1] if i > 0 else {}
+                new_items = {k: v for k, v in inv.items() if v > prev_inv.get(k, 0)}
+                if new_items:
+                    successful_collections.append((action, new_items))
+        
+        if successful_collections:
+            print("Successful collections:")
+            for action, items in successful_collections[:5]:
+                print(f"  After '{action}': gained {items}")
+        else:
+            print("No successful resource collections detected!")
+        
+        # Check specific action sequences
+        print(f"\n🔍 ACTION SEQUENCE ANALYSIS")
+        # Look for 'do' actions and their context
+        do_actions = []
+        for i, result in enumerate(action_results):
+            if result['action'] == 'do':
+                context = {
+                    'prev_action': action_results[i-1]['action'] if i > 0 else 'start',
+                    'next_action': action_results[i+1]['action'] if i < len(action_results)-1 else 'end',
+                    'reward': result['reward']
+                }
+                do_actions.append(context)
+        
+        if do_actions:
+            print(f"Found {len(do_actions)} 'do' actions")
+            # Check what happened before 'do' actions
+            prev_action_counts = Counter([d['prev_action'] for d in do_actions])
+            print("Actions before 'do':")
+            for action, count in prev_action_counts.most_common(5):
+                print(f"  {action}: {count}")
+        
+        # Check for make_wood_pickaxe attempts
+        pickaxe_attempts = [r for r in action_results if r['action'] == 'make_wood_pickaxe']
+        if pickaxe_attempts:
+            print(f"\n🔨 PICKAXE CRAFTING")
+            print(f"Attempted to make wood pickaxe {len(pickaxe_attempts)} times")
+        
+        # Final inventory check
+        if inventory_timeline:
+            final_inv = inventory_timeline[-1][1]
+            print(f"\n📦 FINAL INVENTORY: {final_inv if final_inv else 'Empty'}")
+        
+        # Achievement check
+        if achievements_timeline:
+            final_achievements = achievements_timeline[-1][1]
+            print(f"🏆 FINAL ACHIEVEMENTS: {final_achievements if final_achievements else 'None'}")
+    
+    # Check for any rewards across all sessions
+    print(f"\n\n{'='*60}")
+    print("📊 OVERALL REWARD ANALYSIS")
+    print(f"{'='*60}")
+    
+    total_rewards = conn.execute("""
+        SELECT SUM(reward) as total, COUNT(*) as count
+        FROM events
+        WHERE reward IS NOT NULL AND reward != 0
+        AND session_id IN (
+            SELECT DISTINCT e.session_id
+            FROM events e
+            JOIN session_timesteps st ON e.timestep_id = st.id
+            WHERE st.experiment_id = ?
+        )
+    """, [exp_id]).fetchone()
+    
+    if total_rewards:
+        total, count = total_rewards
+        print(f"Total non-zero rewards: {total or 0}")
+        print(f"Number of reward events: {count or 0}")
+    
+    # Check if rewards are being recorded at all
+    print("\n🔍 REWARD RECORDING CHECK")
+    sample_rewards = conn.execute("""
+        SELECT reward, metadata
+        FROM events
+        WHERE event_type = 'environment'
+        AND session_id IN (
+            SELECT DISTINCT e.session_id
+            FROM events e
+            JOIN session_timesteps st ON e.timestep_id = st.id
+            WHERE st.experiment_id = ?
+        )
+        LIMIT 20
+    """, [exp_id]).fetchall()
+    
+    reward_values = [r for r, _ in sample_rewards if r is not None]
+    print(f"Sample reward values: {reward_values[:10]}")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    db_path = "./traces_v2_synth/traces.duckdb"
+    if Path(db_path).exists():
+        analyze_no_rewards(db_path)
+    else:
+        print(f"❌ Database not found at {db_path}")