synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.5__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_trace_issue.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Analyze trace data to understand why no achievements were unlocked."""
+
+import duckdb
+import json
+from pathlib import Path
+from collections import defaultdict, Counter
+
+def analyze_traces(db_path: str):
+    """Analyze trace data to identify issues."""
+    conn = duckdb.connect(db_path, read_only=True)
+    
+    print("🔍 Analyzing trace data...\n")
+    
+    # 1. Check basic statistics
+    print("📊 BASIC STATISTICS")
+    print("=" * 50)
+    
+    # First, show available tables
+    tables = conn.execute("SHOW TABLES").fetchall()
+    print("Available tables:")
+    for table in tables:
+        print(f"  - {table[0]}")
+    
+    # Count sessions
+    session_count = conn.execute("SELECT COUNT(DISTINCT session_id) FROM session_traces").fetchone()[0]
+    print(f"\nTotal sessions: {session_count}")
+    
+    # Count events
+    event_count = conn.execute("SELECT COUNT(*) FROM session_traces").fetchone()[0]
+    print(f"Total events: {event_count}")
+    
+    # Check event types
+    print("\n📋 EVENT TYPE DISTRIBUTION")
+    print("-" * 30)
+    event_types = conn.execute("""
+        SELECT event_type, COUNT(*) as count 
+        FROM session_traces 
+        GROUP BY event_type 
+        ORDER BY count DESC
+    """).fetchall()
+    
+    for event_type, count in event_types:
+        print(f"{event_type}: {count}")
+    
+    # 2. Analyze agent decisions
+    print("\n🤖 AGENT DECISIONS ANALYSIS")
+    print("=" * 50)
+    
+    # Get all generation completion events
+    completions = conn.execute("""
+        SELECT event_data 
+        FROM session_traces 
+        WHERE event_type = 'generation_completion'
+        LIMIT 50
+    """).fetchall()
+    
+    if completions:
+        print(f"Found {len(completions)} generation completions (showing first 50)")
+        
+        # Analyze first few completions
+        for i, (event_data,) in enumerate(completions[:5]):
+            data = json.loads(event_data)
+            print(f"\n--- Completion {i+1} ---")
+            
+            # Extract response
+            if 'response' in data and data['response']:
+                response = data['response']
+                if 'content' in response:
+                    print(f"Content preview: {response['content'][:200]}...")
+                if 'tool_calls' in response:
+                    print(f"Tool calls: {response['tool_calls']}")
+            else:
+                print("No response found in event data")
+    else:
+        print("❌ No generation completion events found!")
+    
+    # 3. Analyze runtime events (actions taken)
+    print("\n🎮 RUNTIME EVENTS (ACTIONS)")
+    print("=" * 50)
+    
+    runtime_events = conn.execute("""
+        SELECT event_data 
+        FROM session_traces 
+        WHERE event_type = 'runtime_event'
+        LIMIT 100
+    """).fetchall()
+    
+    if runtime_events:
+        action_counter = Counter()
+        
+        for (event_data,) in runtime_events:
+            data = json.loads(event_data)
+            if 'metadata' in data and 'action_name' in data['metadata']:
+                action_counter[data['metadata']['action_name']] += 1
+        
+        print(f"Found {len(runtime_events)} runtime events")
+        print("\nAction distribution:")
+        for action, count in action_counter.most_common():
+            print(f"  {action}: {count}")
+    else:
+        print("❌ No runtime events found!")
+    
+    # 4. Analyze environment events (results)
+    print("\n🌍 ENVIRONMENT EVENTS")
+    print("=" * 50)
+    
+    env_events = conn.execute("""
+        SELECT event_data 
+        FROM session_traces 
+        WHERE event_type = 'environment_event'
+        LIMIT 100
+    """).fetchall()
+    
+    if env_events:
+        reward_sum = 0
+        achievements_found = []
+        
+        for (event_data,) in env_events:
+            data = json.loads(event_data)
+            
+            # Check rewards
+            if 'reward' in data:
+                reward_sum += data['reward'] or 0
+            
+            # Check for achievements in state
+            if 'system_state_after' in data:
+                state = data['system_state_after']
+                if 'public_state' in state and 'achievements' in state['public_state']:
+                    achievements = state['public_state']['achievements']
+                    for ach, unlocked in achievements.items():
+                        if unlocked:
+                            achievements_found.append(ach)
+        
+        print(f"Found {len(env_events)} environment events")
+        print(f"Total reward across all events: {reward_sum}")
+        print(f"Achievements found: {set(achievements_found) if achievements_found else 'None'}")
+    else:
+        print("❌ No environment events found!")
+    
+    # 5. Check for errors
+    print("\n⚠️ ERROR CHECK")
+    print("=" * 50)
+    
+    # Look for error messages in events
+    error_events = conn.execute("""
+        SELECT event_type, event_data 
+        FROM session_traces 
+        WHERE event_data LIKE '%error%' OR event_data LIKE '%Error%'
+        LIMIT 10
+    """).fetchall()
+    
+    if error_events:
+        print(f"Found {len(error_events)} events with potential errors:")
+        for event_type, event_data in error_events[:3]:
+            print(f"\n{event_type}:")
+            data = json.loads(event_data)
+            print(json.dumps(data, indent=2)[:500])
+    else:
+        print("No obvious errors found in events")
+    
+    # 6. Sample a full episode flow
+    print("\n📖 SAMPLE EPISODE FLOW")
+    print("=" * 50)
+    
+    # Get events from first session
+    first_session = conn.execute("SELECT DISTINCT session_id FROM session_traces LIMIT 1").fetchone()
+    if first_session:
+        session_id = first_session[0]
+        print(f"Analyzing session: {session_id}")
+        
+        session_events = conn.execute("""
+            SELECT event_type, event_data, created_at
+            FROM session_traces 
+            WHERE session_id = ?
+            ORDER BY created_at
+            LIMIT 20
+        """, [session_id]).fetchall()
+        
+        print(f"\nFirst 20 events in session:")
+        for i, (event_type, event_data, created_at) in enumerate(session_events):
+            data = json.loads(event_data)
+            print(f"\n{i+1}. {event_type} at {created_at}")
+            
+            # Show relevant info based on event type
+            if event_type == 'generation_completion':
+                if 'response' in data and 'tool_calls' in data['response']:
+                    print(f"   Tool calls: {data['response']['tool_calls']}")
+            elif event_type == 'runtime_event':
+                if 'metadata' in data:
+                    print(f"   Action: {data['metadata'].get('action_name', 'Unknown')}")
+            elif event_type == 'environment_event':
+                if 'reward' in data:
+                    print(f"   Reward: {data['reward']}")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    db_path = "./traces_v2_synth/traces.duckdb"
+    if Path(db_path).exists():
+        analyze_traces(db_path)
+    else:
+        print(f"❌ Database not found at {db_path}")
+        print("Available databases:")
+        for db in Path(".").glob("**/traces.duckdb"):
+            print(f"  - {db}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_db_schema.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Check DuckDB schema to understand table structure."""
+
+import duckdb
+from pathlib import Path
+
+def check_schema(db_path: str):
+    """Check database schema."""
+    conn = duckdb.connect(db_path, read_only=True)
+    
+    print("🔍 Checking database schema...\n")
+    
+    # Get all tables
+    tables = conn.execute("SHOW TABLES").fetchall()
+    print("📋 Tables in database:")
+    for table in tables:
+        print(f"  - {table[0]}")
+    
+    # Check schema of key tables
+    key_tables = ['session_traces', 'events', 'messages', 'session_timesteps']
+    
+    for table_name in key_tables:
+        if any(t[0] == table_name for t in tables):
+            print(f"\n📊 Schema for {table_name}:")
+            print("-" * 50)
+            schema = conn.execute(f"DESCRIBE {table_name}").fetchall()
+            for col_name, col_type, _, _, _, _ in schema:
+                print(f"  {col_name}: {col_type}")
+            
+            # Show sample data
+            print(f"\n📄 Sample data from {table_name} (first 2 rows):")
+            sample = conn.execute(f"SELECT * FROM {table_name} LIMIT 2").fetchall()
+            if sample:
+                # Get column names
+                cols = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
+                print(f"  Columns: {cols}")
+                for row in sample:
+                    print(f"  {row}")
+            else:
+                print("  (No data)")
+    
+    conn.close()
+
+if __name__ == "__main__":
+    db_path = "./traces_v2_synth/traces.duckdb"
+    if Path(db_path).exists():
+        check_schema(db_path)
+    else:
+        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_latest_results.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Check the latest results JSON file."""
+
+import json
+import glob
+from pathlib import Path
+
+# Find the latest results file
+result_files = glob.glob("crafter_lm_synth_results_*.json")
+if not result_files:
+    print("No result files found")
+    exit(1)
+
+# Get the most recent file
+latest_file = max(result_files, key=lambda f: Path(f).stat().st_mtime)
+print(f"📊 Checking latest results: {latest_file}\n")
+
+with open(latest_file) as f:
+    data = json.load(f)
+
+# Extract key metrics
+total_episodes = data.get('total_episodes', 0)
+total_steps = data.get('total_steps', 0)
+model = data.get('model', 'unknown')
+
+print(f"Model: {model}")
+print(f"Episodes: {total_episodes}")
+print(f"Total Steps: {total_steps}")
+
+# Check episode results
+episodes = data.get('episodes', [])
+if episodes:
+    print(f"\n📊 Episode Summary:")
+    for i, ep in enumerate(episodes):
+        steps = ep.get('steps', 0)
+        reward = ep.get('total_reward', 0)
+        achievements = ep.get('achievements_unlocked', [])
+        
+        print(f"\nEpisode {i}:")
+        print(f"  Steps: {steps}")
+        print(f"  Reward: {reward}")
+        print(f"  Achievements: {len(achievements)}")
+        if achievements:
+            print(f"    - {', '.join(achievements[:5])}")
+            if len(achievements) > 5:
+                print(f"    ... and {len(achievements) - 5} more")
+        
+        # Check inventory at end
+        inventory = ep.get('final_inventory', {})
+        non_zero = {k: v for k, v in inventory.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
+        if non_zero:
+            print(f"  Final inventory: {non_zero}")
+        
+        # Sample actions
+        if 'action_history' in ep and ep['action_history']:
+            print(f"  Sample actions: {ep['action_history'][:5]}")
+
+# Overall statistics
+avg_reward = sum(ep.get('total_reward', 0) for ep in episodes) / len(episodes) if episodes else 0
+total_achievements = sum(len(ep.get('achievements_unlocked', [])) for ep in episodes)
+
+print(f"\n📊 Overall Statistics:")
+print(f"Average reward per episode: {avg_reward:.2f}")
+print(f"Total achievements unlocked: {total_achievements}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/debug_agent_responses.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Debug why agent is not using multiple actions."""
+
+import json
+import sys
+from pathlib import Path
+
+# Read the latest run output from stdin or file
+if len(sys.argv) > 1:
+    with open(sys.argv[1]) as f:
+        output = f.read()
+else:
+    # Check for latest log file
+    log_files = list(Path(".").glob("crafter_run_*.log"))
+    if not log_files:
+        print("No log files found. Run with: python test_crafter_react_agent_lm_synth.py --model 'Qwen/Qwen2.5-14B-Instruct' --episodes 1 --max-steps 10 --verbose 2>&1 | tee crafter_run.log")
+        exit(1)
+    
+    latest_log = max(log_files, key=lambda f: f.stat().st_mtime)
+    with open(latest_log) as f:
+        output = f.read()
+
+# Parse tool calls
+tool_calls = []
+single_action_count = 0
+multi_action_count = 0
+
+for line in output.split('\n'):
+    if "🔧 Turn" in line and "Tool Call:" in line:
+        turn_info = {"turn": line}
+        tool_calls.append(turn_info)
+    elif "Actions:" in line and tool_calls:
+        actions_str = line.strip().split("Actions:")[1].strip()
+        try:
+            # Parse the action list
+            actions = eval(actions_str) if actions_str else []
+            tool_calls[-1]["actions"] = actions
+            tool_calls[-1]["action_count"] = len(actions)
+            
+            if len(actions) == 1:
+                single_action_count += 1
+            elif len(actions) > 1:
+                multi_action_count += 1
+        except:
+            tool_calls[-1]["actions"] = "parse_error"
+            tool_calls[-1]["action_count"] = 0
+
+print("🔍 AGENT ACTION ANALYSIS\n")
+print(f"Total tool calls: {len(tool_calls)}")
+print(f"Single action calls: {single_action_count}")
+print(f"Multi-action calls: {multi_action_count}")
+print(f"Average actions per call: {sum(tc.get('action_count', 0) for tc in tool_calls) / len(tool_calls) if tool_calls else 0:.2f}")
+
+# Show distribution
+action_counts = {}
+for tc in tool_calls:
+    count = tc.get('action_count', 0)
+    action_counts[count] = action_counts.get(count, 0) + 1
+
+print("\nAction count distribution:")
+for count in sorted(action_counts.keys()):
+    print(f"  {count} actions: {action_counts[count]} times")
+
+# Show examples of multi-action calls
+print("\n📋 Multi-action examples:")
+multi_examples = [tc for tc in tool_calls if tc.get('action_count', 0) > 1]
+for example in multi_examples[:5]:
+    print(f"  {example['turn']}")
+    print(f"    Actions: {example['actions']}")
+
+# Check for response parsing issues
+print("\n🔍 Response preview analysis:")
+response_previews = []
+for line in output.split('\n'):
+    if "📝 Raw response preview:" in line:
+        preview = line.split("preview:")[1].strip()
+        response_previews.append(preview)
+
+if response_previews:
+    print(f"Found {len(response_previews)} response previews")
+    # Check if responses mention multiple actions
+    multi_action_mentions = 0
+    for preview in response_previews[:5]:
+        if any(word in preview.lower() for word in ['multiple', 'sequence', 'then', 'after']):
+            multi_action_mentions += 1
+        print(f"  - {preview[:100]}...")
+    
+    print(f"\nResponses mentioning sequences: {multi_action_mentions}/{len(response_previews[:5])}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/quick_trace_check.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Quick check of recent traces without locking the database."""
+
+import sqlite3
+import json
+from pathlib import Path
+from collections import Counter
+
+# Use SQLite interface which is more permissive with locks
+db_path = "./traces_v2_synth/traces.duckdb"
+
+if Path(db_path).exists():
+    try:
+        # DuckDB files can be read with SQLite in read-only mode
+        conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+        cursor = conn.cursor()
+        
+        print("🔍 Quick trace analysis...\n")
+        
+        # Get recent events
+        cursor.execute("""
+            SELECT event_type, metadata 
+            FROM events 
+            ORDER BY id DESC 
+            LIMIT 100
+        """)
+        
+        events = cursor.fetchall()
+        print(f"Found {len(events)} recent events\n")
+        
+        # Count event types
+        event_types = Counter([e[0] for e in events])
+        print("Event type distribution:")
+        for etype, count in event_types.most_common():
+            print(f"  {etype}: {count}")
+        
+        # Check for actions
+        print("\n🎮 Recent actions:")
+        action_count = 0
+        action_types = Counter()
+        
+        for event_type, metadata_str in events:
+            if metadata_str and event_type == 'runtime':
+                try:
+                    metadata = json.loads(metadata_str)
+                    if 'action_name' in metadata:
+                        action_types[metadata['action_name']] += 1
+                        action_count += 1
+                        if action_count <= 10:
+                            print(f"  - {metadata['action_name']}")
+                except:
+                    pass
+        
+        if action_types:
+            print(f"\nAction summary:")
+            for action, count in action_types.most_common():
+                print(f"  {action}: {count}")
+        
+        conn.close()
+        
+    except Exception as e:
+        print(f"Error: {e}")
+        print("\nTrying alternative analysis...")
+        
+        # If we can't read the DB, check for any JSON trace files
+        trace_files = list(Path("./traces_v2_synth").glob("session_*.json"))
+        if trace_files:
+            print(f"Found {len(trace_files)} JSON trace files")
+            latest = max(trace_files, key=lambda f: f.stat().st_mtime)
+            print(f"Latest: {latest.name}")
+            
+            with open(latest) as f:
+                data = json.load(f)
+                print(f"Session ID: {data.get('session_id', 'Unknown')}")
+                print(f"Events: {len(data.get('events', []))}")
+else:
+    print("No trace database found")