synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_agent_actions.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze agent actions and model responses from trace data."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter
-
-def analyze_agent_actions(db_path: str):
-    """Analyze agent actions and responses."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing agent actions and responses...\n")
-    
-    # Get all events
-    all_events = conn.execute("""
-        SELECT 
-            e.session_id,
-            e.event_type,
-            e.event_time,
-            e.metadata,
-            e.system_state_after,
-            e.system_state_before,
-            e.reward,
-            e.terminated
-        FROM events e
-        ORDER BY e.session_id, e.event_time
-    """).fetchall()
-    
-    print(f"Total events found: {len(all_events)}\n")
-    
-    # Group events by session
-    sessions = {}
-    for event in all_events:
-        session_id = event[0]
-        if session_id not in sessions:
-            sessions[session_id] = []
-        sessions[session_id].append(event)
-    
-    print(f"Total sessions: {len(sessions)}\n")
-    
-    # Analyze each session
-    for i, (session_id, events) in enumerate(sessions.items()):
-        if i >= 3:  # Only analyze first 3 sessions
-            break
-            
-        print(f"\n{'='*60}")
-        print(f"SESSION {i+1}: {session_id}")
-        print(f"{'='*60}")
-        print(f"Total events in session: {len(events)}")
-        
-        # Track actions and achievements
-        actions_taken = []
-        achievements_unlocked = set()
-        total_reward = 0
-        
-        for event in events:
-            event_type = event[1]
-            metadata = json.loads(event[3]) if event[3] else {}
-            state_after = json.loads(event[4]) if event[4] else {}
-            state_before = json.loads(event[5]) if event[5] else {}
-            reward = event[6] or 0
-            terminated = event[7]
-            
-            total_reward += reward
-            
-            # Track runtime events (actions)
-            if event_type == 'runtime' and 'action_name' in metadata:
-                action = metadata['action_name']
-                actions_taken.append(action)
-                
-            # Check for achievements
-            if 'public_state' in state_after:
-                public_state = state_after['public_state']
-                if 'achievements_status' in public_state:
-                    for ach, unlocked in public_state['achievements_status'].items():
-                        if unlocked:
-                            achievements_unlocked.add(ach)
-            
-            # Look for generation responses
-            if event_type == 'generation' and metadata:
-                print(f"\n--- Generation Event ---")
-                if 'model' in metadata:
-                    print(f"Model: {metadata['model']}")
-                if 'response' in metadata:
-                    response = metadata['response']
-                    if isinstance(response, str):
-                        print(f"Response preview: {response[:200]}")
-                    elif isinstance(response, dict):
-                        if 'content' in response:
-                            print(f"Content: {response['content'][:200]}")
-                        if 'tool_calls' in response:
-                            print(f"Tool calls: {response['tool_calls']}")
-        
-        # Summary for this session
-        print(f"\n--- Session Summary ---")
-        print(f"Total reward: {total_reward}")
-        print(f"Achievements unlocked: {achievements_unlocked if achievements_unlocked else 'None'}")
-        print(f"Total actions taken: {len(actions_taken)}")
-        
-        if actions_taken:
-            action_counts = Counter(actions_taken)
-            print(f"\nAction distribution:")
-            for action, count in action_counts.most_common(10):
-                print(f"  {action}: {count}")
-    
-    # Overall analysis
-    print(f"\n\n{'='*60}")
-    print("OVERALL ANALYSIS")
-    print(f"{'='*60}")
-    
-    # Check for any achievements across all sessions
-    all_achievements = conn.execute("""
-        SELECT DISTINCT
-            json_extract_string(system_state_after, '$.public_state.achievements_status') as achievements
-        FROM events
-        WHERE system_state_after IS NOT NULL
-        AND json_extract_string(system_state_after, '$.public_state.achievements_status') IS NOT NULL
-        LIMIT 10
-    """).fetchall()
-    
-    print(f"\nChecking achievement states...")
-    total_unlocked = 0
-    for ach_json, in all_achievements[:3]:
-        if ach_json:
-            achievements = json.loads(ach_json)
-            unlocked = [k for k, v in achievements.items() if v]
-            if unlocked:
-                total_unlocked += len(unlocked)
-                print(f"Found unlocked: {unlocked}")
-    
-    if total_unlocked == 0:
-        print("❌ No achievements were unlocked in any session!")
-    
-    # Check for specific issues
-    print(f"\n\n{'='*60}")
-    print("POTENTIAL ISSUES")
-    print(f"{'='*60}")
-    
-    # Look for model responses
-    model_responses = conn.execute("""
-        SELECT COUNT(*) 
-        FROM events 
-        WHERE event_type = 'generation'
-    """).fetchone()[0]
-    
-    print(f"Generation events found: {model_responses}")
-    
-    if model_responses == 0:
-        print("❌ No generation events found - agent may not be responding")
-    
-    # Look for specific metadata patterns
-    print("\nChecking event metadata patterns...")
-    metadata_samples = conn.execute("""
-        SELECT event_type, metadata
-        FROM events
-        WHERE metadata IS NOT NULL
-        AND metadata != '{}'
-        AND metadata != '[]'
-        LIMIT 20
-    """).fetchall()
-    
-    for event_type, metadata_str in metadata_samples[:5]:
-        metadata = json.loads(metadata_str)
-        print(f"\n{event_type}: {list(metadata.keys())}")
-        if 'action_name' in metadata:
-            print(f"  Action: {metadata['action_name']}")
-        if 'model' in metadata:
-            print(f"  Model: {metadata['model']}")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_agent_actions(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_latest_run.py

@@ -1,222 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze the latest run to understand why no achievements were unlocked."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-from datetime import datetime
-
-def analyze_latest_run(db_path: str):
-    """Analyze the most recent run."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing latest run with 97 steps and 0 achievements...\n")
-    
-    # Get the latest experiment
-    latest_experiment = conn.execute("""
-        SELECT experiment_id, experiment_name, created_at
-        FROM experiments
-        ORDER BY created_at DESC
-        LIMIT 1
-    """).fetchone()
-    
-    if latest_experiment:
-        exp_id, exp_name, created_at = latest_experiment
-        print(f"📊 Latest Experiment: {exp_name}")
-        print(f"   ID: {exp_id}")
-        print(f"   Created: {created_at}")
-    
-    # Get all sessions from latest experiment
-    sessions = conn.execute("""
-        SELECT DISTINCT s.session_id, s.num_timesteps, s.num_events
-        FROM session_traces s
-        JOIN events e ON s.session_id = e.session_id
-        WHERE e.timestep_id IN (
-            SELECT id FROM session_timesteps 
-            WHERE experiment_id = ?
-        )
-        ORDER BY s.created_at DESC
-    """, [exp_id]).fetchall()
-    
-    print(f"\n📊 Sessions in latest experiment: {len(sessions)}")
-    
-    for session_id, num_timesteps, num_events in sessions:
-        print(f"\n{'='*60}")
-        print(f"SESSION: {session_id}")
-        print(f"Timesteps: {num_timesteps}, Events: {num_events}")
-        print(f"{'='*60}")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after, reward
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        # Analyze actions taken
-        actions_taken = []
-        action_sequences = []
-        current_sequence = []
-        total_reward = 0
-        achievements_over_time = []
-        
-        for event_type, metadata_str, state_after_str, reward in events:
-            if reward:
-                total_reward += reward
-                
-            if metadata_str:
-                metadata = json.loads(metadata_str)
-                
-                # Track runtime actions
-                if event_type == 'runtime' and 'action_name' in metadata:
-                    action = metadata['action_name']
-                    actions_taken.append(action)
-                    current_sequence.append(action)
-                    
-                    # Group actions into sequences (reset on certain actions)
-                    if action in ['do', 'make_wood_pickaxe', 'place_table', 'sleep']:
-                        if len(current_sequence) > 1:
-                            action_sequences.append(current_sequence[:-1])
-                        current_sequence = [action]
-            
-            # Check achievements
-            if state_after_str:
-                state_after = json.loads(state_after_str)
-                if 'public_state' in state_after:
-                    ps = state_after['public_state']
-                    if 'achievements_status' in ps:
-                        unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                        achievements_over_time.append(len(unlocked))
-                    
-                    # Check inventory
-                    if 'inventory' in ps and len(actions_taken) % 20 == 0:  # Sample every 20 actions
-                        inv = ps['inventory']
-                        non_zero = {k: v for k, v in inv.items() if v > 0}
-                        if non_zero:
-                            print(f"\n📦 Inventory at action {len(actions_taken)}: {non_zero}")
-        
-        # Action analysis
-        print(f"\n📊 ACTION ANALYSIS")
-        print(f"Total actions taken: {len(actions_taken)}")
-        
-        if actions_taken:
-            action_counts = Counter(actions_taken)
-            print(f"\nAction distribution:")
-            for action, count in action_counts.most_common():
-                percentage = (count / len(actions_taken)) * 100
-                print(f"  {action:20} {count:4} ({percentage:5.1f}%)")
-            
-            # Check for repetitive patterns
-            print(f"\n🔄 REPETITIVE PATTERNS")
-            # Find consecutive repeated actions
-            consecutive_repeats = []
-            if actions_taken:
-                current_action = actions_taken[0]
-                repeat_count = 1
-                
-                for action in actions_taken[1:]:
-                    if action == current_action:
-                        repeat_count += 1
-                    else:
-                        if repeat_count > 3:
-                            consecutive_repeats.append((current_action, repeat_count))
-                        current_action = action
-                        repeat_count = 1
-                
-                if repeat_count > 3:
-                    consecutive_repeats.append((current_action, repeat_count))
-            
-            if consecutive_repeats:
-                print("Found repetitive sequences:")
-                for action, count in consecutive_repeats[:5]:
-                    print(f"  {action} repeated {count} times consecutively")
-            else:
-                print("No significant repetitive patterns found")
-            
-            # Check action sequences
-            if action_sequences:
-                print(f"\n🎯 ACTION SEQUENCES (movement -> action):")
-                seq_counter = Counter([' → '.join(seq) for seq in action_sequences if len(seq) <= 5])
-                for seq, count in seq_counter.most_common(5):
-                    print(f"  {seq}: {count} times")
-        
-        # Achievement progress
-        print(f"\n🏆 ACHIEVEMENT PROGRESS")
-        print(f"Total reward: {total_reward}")
-        if achievements_over_time:
-            max_achievements = max(achievements_over_time)
-            print(f"Max achievements reached: {max_achievements}")
-            
-            # Find when achievements were unlocked
-            achievement_unlocks = []
-            prev_count = 0
-            for i, count in enumerate(achievements_over_time):
-                if count > prev_count:
-                    achievement_unlocks.append((i, count))
-                    prev_count = count
-            
-            if achievement_unlocks:
-                print("Achievement unlock timeline:")
-                for step, count in achievement_unlocks:
-                    print(f"  Step {step}: {count} achievements")
-        else:
-            print("No achievement data found")
-        
-        # Check final state
-        final_event = events[-1] if events else None
-        if final_event and final_event[2]:  # state_after
-            final_state = json.loads(final_event[2])
-            if 'public_state' in final_state:
-                ps = final_state['public_state']
-                
-                print(f"\n📊 FINAL STATE")
-                
-                # Final inventory
-                if 'inventory' in ps:
-                    inv = ps['inventory']
-                    non_zero = {k: v for k, v in inv.items() if v > 0}
-                    print(f"Final inventory: {non_zero if non_zero else 'Empty'}")
-                
-                # Final achievements
-                if 'achievements_status' in ps:
-                    unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                    print(f"Final achievements: {unlocked if unlocked else 'None'}")
-                
-                # Player stats
-                if 'health' in ps:
-                    print(f"Final stats: Health={ps.get('health', '?')}, Food={ps.get('food', '?')}, Energy={ps.get('energy', '?')}")
-    
-    # Check for any generation/LM events
-    print(f"\n{'='*60}")
-    print("📊 LM/GENERATION EVENT CHECK")
-    print(f"{'='*60}")
-    
-    lm_events = conn.execute("""
-        SELECT COUNT(*) 
-        FROM events 
-        WHERE session_id IN (
-            SELECT DISTINCT s.session_id
-            FROM session_traces s
-            JOIN events e ON s.session_id = e.session_id
-            WHERE e.timestep_id IN (
-                SELECT id FROM session_timesteps 
-                WHERE experiment_id = ?
-            )
-        ) AND (event_type = 'lm' OR event_type = 'generation')
-    """, [exp_id]).fetchone()[0]
-    
-    print(f"LM/Generation events in this experiment: {lm_events}")
-    
-    if lm_events == 0:
-        print("⚠️  No LM events found - the agent may not be generating proper decisions")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_latest_run(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_lm_traces.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze LM traces to see what the agent is actually doing."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-
-def analyze_lm_traces(db_path: str):
-    """Analyze LM traces to understand agent behavior."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing LM traces after fix...\n")
-    
-    # First, check if we have any LM events
-    print("📊 CHECKING FOR LM EVENTS")
-    print("=" * 50)
-    
-    lm_events = conn.execute("""
-        SELECT COUNT(*) 
-        FROM events 
-        WHERE event_type = 'lm' OR event_type = 'generation'
-    """).fetchone()[0]
-    
-    print(f"LM/Generation events found: {lm_events}")
-    
-    # Get all unique event types
-    event_types = conn.execute("""
-        SELECT DISTINCT event_type, COUNT(*) as count
-        FROM events
-        GROUP BY event_type
-        ORDER BY count DESC
-    """).fetchall()
-    
-    print("\nAll event types:")
-    for event_type, count in event_types:
-        print(f"  {event_type}: {count}")
-    
-    # Analyze sessions
-    print("\n📊 SESSION ANALYSIS")
-    print("=" * 50)
-    
-    sessions = conn.execute("""
-        SELECT DISTINCT session_id
-        FROM events
-        ORDER BY session_id
-        LIMIT 5
-    """).fetchall()
-    
-    for i, (session_id,) in enumerate(sessions):
-        print(f"\n--- Session {i+1}: {session_id} ---")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        print(f"Total events: {len(events)}")
-        
-        # Count event types
-        event_type_counts = Counter()
-        actions_taken = []
-        achievements = set()
-        
-        for event_type, metadata_str, state_after_str in events:
-            event_type_counts[event_type] += 1
-            
-            if metadata_str:
-                metadata = json.loads(metadata_str)
-                
-                # Track actions
-                if 'action_name' in metadata:
-                    actions_taken.append(metadata['action_name'])
-                elif 'action' in metadata:
-                    actions_taken.append(metadata['action'])
-            
-            # Check for achievements
-            if state_after_str:
-                state_after = json.loads(state_after_str)
-                if 'public_state' in state_after:
-                    public_state = state_after['public_state']
-                    if 'achievements_status' in public_state:
-                        for ach, unlocked in public_state['achievements_status'].items():
-                            if unlocked:
-                                achievements.add(ach)
-        
-        print(f"Event type distribution: {dict(event_type_counts)}")
-        print(f"Total actions: {len(actions_taken)}")
-        if actions_taken:
-            action_counts = Counter(actions_taken)
-            print(f"Top actions: {action_counts.most_common(5)}")
-        print(f"Achievements unlocked: {achievements if achievements else 'None'}")
-    
-    # Look for any LM-related metadata
-    print("\n📊 LM METADATA ANALYSIS")
-    print("=" * 50)
-    
-    # Check for model information in metadata
-    model_events = conn.execute("""
-        SELECT metadata
-        FROM events
-        WHERE metadata LIKE '%model%'
-        LIMIT 10
-    """).fetchall()
-    
-    if model_events:
-        print(f"Found {len(model_events)} events with model metadata")
-        for i, (metadata_str,) in enumerate(model_events[:3]):
-            metadata = json.loads(metadata_str)
-            print(f"\nEvent {i+1} metadata keys: {list(metadata.keys())}")
-            if 'model' in metadata:
-                print(f"  Model: {metadata['model']}")
-    else:
-        print("No events with model metadata found")
-    
-    # Check messages table
-    print("\n📊 MESSAGES ANALYSIS")
-    print("=" * 50)
-    
-    message_count = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
-    print(f"Total messages: {message_count}")
-    
-    if message_count > 0:
-        messages = conn.execute("""
-            SELECT message_type, content
-            FROM messages
-            LIMIT 10
-        """).fetchall()
-        
-        for i, (msg_type, content_str) in enumerate(messages[:5]):
-            content = json.loads(content_str)
-            print(f"\nMessage {i+1} ({msg_type}):")
-            if isinstance(content, dict):
-                print(f"  Keys: {list(content.keys())}")
-                if 'payload' in content:
-                    payload = content['payload']
-                    if isinstance(payload, dict) and 'inventory' in payload:
-                        # Show non-zero inventory items
-                        inv = payload['inventory']
-                        non_zero = {k: v for k, v in inv.items() if v > 0}
-                        if non_zero:
-                            print(f"  Non-zero inventory: {non_zero}")
-    
-    # Check for tool calls or function calls
-    print("\n📊 TOOL CALL ANALYSIS")
-    print("=" * 50)
-    
-    tool_events = conn.execute("""
-        SELECT metadata
-        FROM events
-        WHERE metadata LIKE '%tool%' OR metadata LIKE '%function%'
-        LIMIT 20
-    """).fetchall()
-    
-    if tool_events:
-        print(f"Found {len(tool_events)} events with tool/function mentions")
-        tool_names = Counter()
-        
-        for (metadata_str,) in tool_events:
-            metadata = json.loads(metadata_str)
-            if 'tool_name' in metadata:
-                tool_names[metadata['tool_name']] += 1
-            elif 'function' in metadata:
-                tool_names[metadata['function']] += 1
-        
-        if tool_names:
-            print("Tool usage:")
-            for tool, count in tool_names.most_common():
-                print(f"  {tool}: {count}")
-    else:
-        print("No tool/function events found")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_lm_traces(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")