synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_latest_run.py

@@ -1,222 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze the latest run to understand why no achievements were unlocked."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-from datetime import datetime
-
-def analyze_latest_run(db_path: str):
-    """Analyze the most recent run."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing latest run with 97 steps and 0 achievements...\n")
-    
-    # Get the latest experiment
-    latest_experiment = conn.execute("""
-        SELECT experiment_id, experiment_name, created_at
-        FROM experiments
-        ORDER BY created_at DESC
-        LIMIT 1
-    """).fetchone()
-    
-    if latest_experiment:
-        exp_id, exp_name, created_at = latest_experiment
-        print(f"📊 Latest Experiment: {exp_name}")
-        print(f"   ID: {exp_id}")
-        print(f"   Created: {created_at}")
-    
-    # Get all sessions from latest experiment
-    sessions = conn.execute("""
-        SELECT DISTINCT s.session_id, s.num_timesteps, s.num_events
-        FROM session_traces s
-        JOIN events e ON s.session_id = e.session_id
-        WHERE e.timestep_id IN (
-            SELECT id FROM session_timesteps 
-            WHERE experiment_id = ?
-        )
-        ORDER BY s.created_at DESC
-    """, [exp_id]).fetchall()
-    
-    print(f"\n📊 Sessions in latest experiment: {len(sessions)}")
-    
-    for session_id, num_timesteps, num_events in sessions:
-        print(f"\n{'='*60}")
-        print(f"SESSION: {session_id}")
-        print(f"Timesteps: {num_timesteps}, Events: {num_events}")
-        print(f"{'='*60}")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after, reward
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        # Analyze actions taken
-        actions_taken = []
-        action_sequences = []
-        current_sequence = []
-        total_reward = 0
-        achievements_over_time = []
-        
-        for event_type, metadata_str, state_after_str, reward in events:
-            if reward:
-                total_reward += reward
-                
-            if metadata_str:
-                metadata = json.loads(metadata_str)
-                
-                # Track runtime actions
-                if event_type == 'runtime' and 'action_name' in metadata:
-                    action = metadata['action_name']
-                    actions_taken.append(action)
-                    current_sequence.append(action)
-                    
-                    # Group actions into sequences (reset on certain actions)
-                    if action in ['do', 'make_wood_pickaxe', 'place_table', 'sleep']:
-                        if len(current_sequence) > 1:
-                            action_sequences.append(current_sequence[:-1])
-                        current_sequence = [action]
-            
-            # Check achievements
-            if state_after_str:
-                state_after = json.loads(state_after_str)
-                if 'public_state' in state_after:
-                    ps = state_after['public_state']
-                    if 'achievements_status' in ps:
-                        unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                        achievements_over_time.append(len(unlocked))
-                    
-                    # Check inventory
-                    if 'inventory' in ps and len(actions_taken) % 20 == 0:  # Sample every 20 actions
-                        inv = ps['inventory']
-                        non_zero = {k: v for k, v in inv.items() if v > 0}
-                        if non_zero:
-                            print(f"\n📦 Inventory at action {len(actions_taken)}: {non_zero}")
-        
-        # Action analysis
-        print(f"\n📊 ACTION ANALYSIS")
-        print(f"Total actions taken: {len(actions_taken)}")
-        
-        if actions_taken:
-            action_counts = Counter(actions_taken)
-            print(f"\nAction distribution:")
-            for action, count in action_counts.most_common():
-                percentage = (count / len(actions_taken)) * 100
-                print(f"  {action:20} {count:4} ({percentage:5.1f}%)")
-            
-            # Check for repetitive patterns
-            print(f"\n🔄 REPETITIVE PATTERNS")
-            # Find consecutive repeated actions
-            consecutive_repeats = []
-            if actions_taken:
-                current_action = actions_taken[0]
-                repeat_count = 1
-                
-                for action in actions_taken[1:]:
-                    if action == current_action:
-                        repeat_count += 1
-                    else:
-                        if repeat_count > 3:
-                            consecutive_repeats.append((current_action, repeat_count))
-                        current_action = action
-                        repeat_count = 1
-                
-                if repeat_count > 3:
-                    consecutive_repeats.append((current_action, repeat_count))
-            
-            if consecutive_repeats:
-                print("Found repetitive sequences:")
-                for action, count in consecutive_repeats[:5]:
-                    print(f"  {action} repeated {count} times consecutively")
-            else:
-                print("No significant repetitive patterns found")
-            
-            # Check action sequences
-            if action_sequences:
-                print(f"\n🎯 ACTION SEQUENCES (movement -> action):")
-                seq_counter = Counter([' → '.join(seq) for seq in action_sequences if len(seq) <= 5])
-                for seq, count in seq_counter.most_common(5):
-                    print(f"  {seq}: {count} times")
-        
-        # Achievement progress
-        print(f"\n🏆 ACHIEVEMENT PROGRESS")
-        print(f"Total reward: {total_reward}")
-        if achievements_over_time:
-            max_achievements = max(achievements_over_time)
-            print(f"Max achievements reached: {max_achievements}")
-            
-            # Find when achievements were unlocked
-            achievement_unlocks = []
-            prev_count = 0
-            for i, count in enumerate(achievements_over_time):
-                if count > prev_count:
-                    achievement_unlocks.append((i, count))
-                    prev_count = count
-            
-            if achievement_unlocks:
-                print("Achievement unlock timeline:")
-                for step, count in achievement_unlocks:
-                    print(f"  Step {step}: {count} achievements")
-        else:
-            print("No achievement data found")
-        
-        # Check final state
-        final_event = events[-1] if events else None
-        if final_event and final_event[2]:  # state_after
-            final_state = json.loads(final_event[2])
-            if 'public_state' in final_state:
-                ps = final_state['public_state']
-                
-                print(f"\n📊 FINAL STATE")
-                
-                # Final inventory
-                if 'inventory' in ps:
-                    inv = ps['inventory']
-                    non_zero = {k: v for k, v in inv.items() if v > 0}
-                    print(f"Final inventory: {non_zero if non_zero else 'Empty'}")
-                
-                # Final achievements
-                if 'achievements_status' in ps:
-                    unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                    print(f"Final achievements: {unlocked if unlocked else 'None'}")
-                
-                # Player stats
-                if 'health' in ps:
-                    print(f"Final stats: Health={ps.get('health', '?')}, Food={ps.get('food', '?')}, Energy={ps.get('energy', '?')}")
-    
-    # Check for any generation/LM events
-    print(f"\n{'='*60}")
-    print("📊 LM/GENERATION EVENT CHECK")
-    print(f"{'='*60}")
-    
-    lm_events = conn.execute("""
-        SELECT COUNT(*) 
-        FROM events 
-        WHERE session_id IN (
-            SELECT DISTINCT s.session_id
-            FROM session_traces s
-            JOIN events e ON s.session_id = e.session_id
-            WHERE e.timestep_id IN (
-                SELECT id FROM session_timesteps 
-                WHERE experiment_id = ?
-            )
-        ) AND (event_type = 'lm' OR event_type = 'generation')
-    """, [exp_id]).fetchone()[0]
-    
-    print(f"LM/Generation events in this experiment: {lm_events}")
-    
-    if lm_events == 0:
-        print("⚠️  No LM events found - the agent may not be generating proper decisions")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_latest_run(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_lm_traces.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze LM traces to see what the agent is actually doing."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-
-def analyze_lm_traces(db_path: str):
-    """Analyze LM traces to understand agent behavior."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing LM traces after fix...\n")
-    
-    # First, check if we have any LM events
-    print("📊 CHECKING FOR LM EVENTS")
-    print("=" * 50)
-    
-    lm_events = conn.execute("""
-        SELECT COUNT(*) 
-        FROM events 
-        WHERE event_type = 'lm' OR event_type = 'generation'
-    """).fetchone()[0]
-    
-    print(f"LM/Generation events found: {lm_events}")
-    
-    # Get all unique event types
-    event_types = conn.execute("""
-        SELECT DISTINCT event_type, COUNT(*) as count
-        FROM events
-        GROUP BY event_type
-        ORDER BY count DESC
-    """).fetchall()
-    
-    print("\nAll event types:")
-    for event_type, count in event_types:
-        print(f"  {event_type}: {count}")
-    
-    # Analyze sessions
-    print("\n📊 SESSION ANALYSIS")
-    print("=" * 50)
-    
-    sessions = conn.execute("""
-        SELECT DISTINCT session_id
-        FROM events
-        ORDER BY session_id
-        LIMIT 5
-    """).fetchall()
-    
-    for i, (session_id,) in enumerate(sessions):
-        print(f"\n--- Session {i+1}: {session_id} ---")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        print(f"Total events: {len(events)}")
-        
-        # Count event types
-        event_type_counts = Counter()
-        actions_taken = []
-        achievements = set()
-        
-        for event_type, metadata_str, state_after_str in events:
-            event_type_counts[event_type] += 1
-            
-            if metadata_str:
-                metadata = json.loads(metadata_str)
-                
-                # Track actions
-                if 'action_name' in metadata:
-                    actions_taken.append(metadata['action_name'])
-                elif 'action' in metadata:
-                    actions_taken.append(metadata['action'])
-            
-            # Check for achievements
-            if state_after_str:
-                state_after = json.loads(state_after_str)
-                if 'public_state' in state_after:
-                    public_state = state_after['public_state']
-                    if 'achievements_status' in public_state:
-                        for ach, unlocked in public_state['achievements_status'].items():
-                            if unlocked:
-                                achievements.add(ach)
-        
-        print(f"Event type distribution: {dict(event_type_counts)}")
-        print(f"Total actions: {len(actions_taken)}")
-        if actions_taken:
-            action_counts = Counter(actions_taken)
-            print(f"Top actions: {action_counts.most_common(5)}")
-        print(f"Achievements unlocked: {achievements if achievements else 'None'}")
-    
-    # Look for any LM-related metadata
-    print("\n📊 LM METADATA ANALYSIS")
-    print("=" * 50)
-    
-    # Check for model information in metadata
-    model_events = conn.execute("""
-        SELECT metadata
-        FROM events
-        WHERE metadata LIKE '%model%'
-        LIMIT 10
-    """).fetchall()
-    
-    if model_events:
-        print(f"Found {len(model_events)} events with model metadata")
-        for i, (metadata_str,) in enumerate(model_events[:3]):
-            metadata = json.loads(metadata_str)
-            print(f"\nEvent {i+1} metadata keys: {list(metadata.keys())}")
-            if 'model' in metadata:
-                print(f"  Model: {metadata['model']}")
-    else:
-        print("No events with model metadata found")
-    
-    # Check messages table
-    print("\n📊 MESSAGES ANALYSIS")
-    print("=" * 50)
-    
-    message_count = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
-    print(f"Total messages: {message_count}")
-    
-    if message_count > 0:
-        messages = conn.execute("""
-            SELECT message_type, content
-            FROM messages
-            LIMIT 10
-        """).fetchall()
-        
-        for i, (msg_type, content_str) in enumerate(messages[:5]):
-            content = json.loads(content_str)
-            print(f"\nMessage {i+1} ({msg_type}):")
-            if isinstance(content, dict):
-                print(f"  Keys: {list(content.keys())}")
-                if 'payload' in content:
-                    payload = content['payload']
-                    if isinstance(payload, dict) and 'inventory' in payload:
-                        # Show non-zero inventory items
-                        inv = payload['inventory']
-                        non_zero = {k: v for k, v in inv.items() if v > 0}
-                        if non_zero:
-                            print(f"  Non-zero inventory: {non_zero}")
-    
-    # Check for tool calls or function calls
-    print("\n📊 TOOL CALL ANALYSIS")
-    print("=" * 50)
-    
-    tool_events = conn.execute("""
-        SELECT metadata
-        FROM events
-        WHERE metadata LIKE '%tool%' OR metadata LIKE '%function%'
-        LIMIT 20
-    """).fetchall()
-    
-    if tool_events:
-        print(f"Found {len(tool_events)} events with tool/function mentions")
-        tool_names = Counter()
-        
-        for (metadata_str,) in tool_events:
-            metadata = json.loads(metadata_str)
-            if 'tool_name' in metadata:
-                tool_names[metadata['tool_name']] += 1
-            elif 'function' in metadata:
-                tool_names[metadata['function']] += 1
-        
-        if tool_names:
-            print("Tool usage:")
-            for tool, count in tool_names.most_common():
-                print(f"  {tool}: {count}")
-    else:
-        print("No tool/function events found")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_lm_traces(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_no_rewards.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze why there are no rewards or achievements."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-
-def analyze_no_rewards(db_path: str):
-    """Analyze why there are no rewards despite actions being taken."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing why there are no rewards or achievements...\n")
-    
-    # Get the latest experiment
-    latest_exp = conn.execute("""
-        SELECT experiment_id, experiment_name
-        FROM experiments
-        ORDER BY created_at DESC
-        LIMIT 1
-    """).fetchone()
-    
-    if not latest_exp:
-        print("No experiments found")
-        return
-        
-    exp_id, exp_name = latest_exp
-    print(f"📊 Latest Experiment: {exp_name}")
-    print(f"   ID: {exp_id}\n")
-    
-    # Get all sessions from latest experiment
-    sessions = conn.execute("""
-        SELECT DISTINCT e.session_id
-        FROM events e
-        JOIN session_timesteps st ON e.timestep_id = st.id
-        WHERE st.experiment_id = ?
-        ORDER BY e.session_id
-    """, [exp_id]).fetchall()
-    
-    print(f"Found {len(sessions)} sessions\n")
-    
-    # Analyze each session
-    for i, (session_id,) in enumerate(sessions[:3]):  # First 3 sessions
-        print(f"\n{'='*60}")
-        print(f"SESSION {i}: {session_id}")
-        print(f"{'='*60}")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after, reward
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        # Track actions and their results
-        action_results = []
-        total_reward = 0
-        achievements_timeline = []
-        inventory_timeline = []
-        
-        for event_type, metadata_str, state_after_str, reward in events:
-            if event_type == 'runtime' and metadata_str:
-                metadata = json.loads(metadata_str)
-                action_name = metadata.get('action_name', 'unknown')
-                
-                # Look for the corresponding environment event
-                result = {
-                    'action': action_name,
-                    'reward': reward or 0,
-                    'achievements_unlocked': []
-                }
-                
-                if state_after_str:
-                    state_after = json.loads(state_after_str)
-                    if 'public_state' in state_after:
-                        ps = state_after['public_state']
-                        
-                        # Check achievements
-                        if 'achievements_status' in ps:
-                            unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                            achievements_timeline.append((action_name, unlocked))
-                            if len(unlocked) > len(result['achievements_unlocked']):
-                                result['achievements_unlocked'] = unlocked
-                        
-                        # Check inventory
-                        if 'inventory' in ps:
-                            inv = ps['inventory']
-                            non_zero = {k: v for k, v in inv.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
-                            if non_zero:
-                                inventory_timeline.append((action_name, non_zero))
-                
-                action_results.append(result)
-                if reward:
-                    total_reward += reward
-        
-        # Analyze action effectiveness
-        print(f"\n📊 ACTION ANALYSIS")
-        print(f"Total actions: {len(action_results)}")
-        print(f"Total reward: {total_reward}")
-        
-        # Count actions by type
-        action_counts = Counter([r['action'] for r in action_results])
-        print(f"\nAction distribution:")
-        for action, count in action_counts.most_common(10):
-            print(f"  {action:20} {count:3}")
-        
-        # Check for successful resource collection
-        print(f"\n📦 RESOURCE COLLECTION")
-        successful_collections = []
-        for i, (action, inv) in enumerate(inventory_timeline):
-            if i > 0 and inv != inventory_timeline[i-1][1]:
-                # Inventory changed
-                prev_inv = inventory_timeline[i-1][1] if i > 0 else {}
-                new_items = {k: v for k, v in inv.items() if v > prev_inv.get(k, 0)}
-                if new_items:
-                    successful_collections.append((action, new_items))
-        
-        if successful_collections:
-            print("Successful collections:")
-            for action, items in successful_collections[:5]:
-                print(f"  After '{action}': gained {items}")
-        else:
-            print("No successful resource collections detected!")
-        
-        # Check specific action sequences
-        print(f"\n🔍 ACTION SEQUENCE ANALYSIS")
-        # Look for 'do' actions and their context
-        do_actions = []
-        for i, result in enumerate(action_results):
-            if result['action'] == 'do':
-                context = {
-                    'prev_action': action_results[i-1]['action'] if i > 0 else 'start',
-                    'next_action': action_results[i+1]['action'] if i < len(action_results)-1 else 'end',
-                    'reward': result['reward']
-                }
-                do_actions.append(context)
-        
-        if do_actions:
-            print(f"Found {len(do_actions)} 'do' actions")
-            # Check what happened before 'do' actions
-            prev_action_counts = Counter([d['prev_action'] for d in do_actions])
-            print("Actions before 'do':")
-            for action, count in prev_action_counts.most_common(5):
-                print(f"  {action}: {count}")
-        
-        # Check for make_wood_pickaxe attempts
-        pickaxe_attempts = [r for r in action_results if r['action'] == 'make_wood_pickaxe']
-        if pickaxe_attempts:
-            print(f"\n🔨 PICKAXE CRAFTING")
-            print(f"Attempted to make wood pickaxe {len(pickaxe_attempts)} times")
-        
-        # Final inventory check
-        if inventory_timeline:
-            final_inv = inventory_timeline[-1][1]
-            print(f"\n📦 FINAL INVENTORY: {final_inv if final_inv else 'Empty'}")
-        
-        # Achievement check
-        if achievements_timeline:
-            final_achievements = achievements_timeline[-1][1]
-            print(f"🏆 FINAL ACHIEVEMENTS: {final_achievements if final_achievements else 'None'}")
-    
-    # Check for any rewards across all sessions
-    print(f"\n\n{'='*60}")
-    print("📊 OVERALL REWARD ANALYSIS")
-    print(f"{'='*60}")
-    
-    total_rewards = conn.execute("""
-        SELECT SUM(reward) as total, COUNT(*) as count
-        FROM events
-        WHERE reward IS NOT NULL AND reward != 0
-        AND session_id IN (
-            SELECT DISTINCT e.session_id
-            FROM events e
-            JOIN session_timesteps st ON e.timestep_id = st.id
-            WHERE st.experiment_id = ?
-        )
-    """, [exp_id]).fetchone()
-    
-    if total_rewards:
-        total, count = total_rewards
-        print(f"Total non-zero rewards: {total or 0}")
-        print(f"Number of reward events: {count or 0}")
-    
-    # Check if rewards are being recorded at all
-    print("\n🔍 REWARD RECORDING CHECK")
-    sample_rewards = conn.execute("""
-        SELECT reward, metadata
-        FROM events
-        WHERE event_type = 'environment'
-        AND session_id IN (
-            SELECT DISTINCT e.session_id
-            FROM events e
-            JOIN session_timesteps st ON e.timestep_id = st.id
-            WHERE st.experiment_id = ?
-        )
-        LIMIT 20
-    """, [exp_id]).fetchall()
-    
-    reward_values = [r for r, _ in sample_rewards if r is not None]
-    print(f"Sample reward values: {reward_values[:10]}")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_no_rewards(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")