synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_trace_issue.py

@@ -1,206 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze trace data to understand why no achievements were unlocked."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import defaultdict, Counter
-
-def analyze_traces(db_path: str):
-    """Analyze trace data to identify issues."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing trace data...\n")
-    
-    # 1. Check basic statistics
-    print("📊 BASIC STATISTICS")
-    print("=" * 50)
-    
-    # First, show available tables
-    tables = conn.execute("SHOW TABLES").fetchall()
-    print("Available tables:")
-    for table in tables:
-        print(f"  - {table[0]}")
-    
-    # Count sessions
-    session_count = conn.execute("SELECT COUNT(DISTINCT session_id) FROM session_traces").fetchone()[0]
-    print(f"\nTotal sessions: {session_count}")
-    
-    # Count events
-    event_count = conn.execute("SELECT COUNT(*) FROM session_traces").fetchone()[0]
-    print(f"Total events: {event_count}")
-    
-    # Check event types
-    print("\n📋 EVENT TYPE DISTRIBUTION")
-    print("-" * 30)
-    event_types = conn.execute("""
-        SELECT event_type, COUNT(*) as count 
-        FROM session_traces 
-        GROUP BY event_type 
-        ORDER BY count DESC
-    """).fetchall()
-    
-    for event_type, count in event_types:
-        print(f"{event_type}: {count}")
-    
-    # 2. Analyze agent decisions
-    print("\n🤖 AGENT DECISIONS ANALYSIS")
-    print("=" * 50)
-    
-    # Get all generation completion events
-    completions = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'generation_completion'
-        LIMIT 50
-    """).fetchall()
-    
-    if completions:
-        print(f"Found {len(completions)} generation completions (showing first 50)")
-        
-        # Analyze first few completions
-        for i, (event_data,) in enumerate(completions[:5]):
-            data = json.loads(event_data)
-            print(f"\n--- Completion {i+1} ---")
-            
-            # Extract response
-            if 'response' in data and data['response']:
-                response = data['response']
-                if 'content' in response:
-                    print(f"Content preview: {response['content'][:200]}...")
-                if 'tool_calls' in response:
-                    print(f"Tool calls: {response['tool_calls']}")
-            else:
-                print("No response found in event data")
-    else:
-        print("❌ No generation completion events found!")
-    
-    # 3. Analyze runtime events (actions taken)
-    print("\n🎮 RUNTIME EVENTS (ACTIONS)")
-    print("=" * 50)
-    
-    runtime_events = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'runtime_event'
-        LIMIT 100
-    """).fetchall()
-    
-    if runtime_events:
-        action_counter = Counter()
-        
-        for (event_data,) in runtime_events:
-            data = json.loads(event_data)
-            if 'metadata' in data and 'action_name' in data['metadata']:
-                action_counter[data['metadata']['action_name']] += 1
-        
-        print(f"Found {len(runtime_events)} runtime events")
-        print("\nAction distribution:")
-        for action, count in action_counter.most_common():
-            print(f"  {action}: {count}")
-    else:
-        print("❌ No runtime events found!")
-    
-    # 4. Analyze environment events (results)
-    print("\n🌍 ENVIRONMENT EVENTS")
-    print("=" * 50)
-    
-    env_events = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'environment_event'
-        LIMIT 100
-    """).fetchall()
-    
-    if env_events:
-        reward_sum = 0
-        achievements_found = []
-        
-        for (event_data,) in env_events:
-            data = json.loads(event_data)
-            
-            # Check rewards
-            if 'reward' in data:
-                reward_sum += data['reward'] or 0
-            
-            # Check for achievements in state
-            if 'system_state_after' in data:
-                state = data['system_state_after']
-                if 'public_state' in state and 'achievements' in state['public_state']:
-                    achievements = state['public_state']['achievements']
-                    for ach, unlocked in achievements.items():
-                        if unlocked:
-                            achievements_found.append(ach)
-        
-        print(f"Found {len(env_events)} environment events")
-        print(f"Total reward across all events: {reward_sum}")
-        print(f"Achievements found: {set(achievements_found) if achievements_found else 'None'}")
-    else:
-        print("❌ No environment events found!")
-    
-    # 5. Check for errors
-    print("\n⚠️ ERROR CHECK")
-    print("=" * 50)
-    
-    # Look for error messages in events
-    error_events = conn.execute("""
-        SELECT event_type, event_data 
-        FROM session_traces 
-        WHERE event_data LIKE '%error%' OR event_data LIKE '%Error%'
-        LIMIT 10
-    """).fetchall()
-    
-    if error_events:
-        print(f"Found {len(error_events)} events with potential errors:")
-        for event_type, event_data in error_events[:3]:
-            print(f"\n{event_type}:")
-            data = json.loads(event_data)
-            print(json.dumps(data, indent=2)[:500])
-    else:
-        print("No obvious errors found in events")
-    
-    # 6. Sample a full episode flow
-    print("\n📖 SAMPLE EPISODE FLOW")
-    print("=" * 50)
-    
-    # Get events from first session
-    first_session = conn.execute("SELECT DISTINCT session_id FROM session_traces LIMIT 1").fetchone()
-    if first_session:
-        session_id = first_session[0]
-        print(f"Analyzing session: {session_id}")
-        
-        session_events = conn.execute("""
-            SELECT event_type, event_data, created_at
-            FROM session_traces 
-            WHERE session_id = ?
-            ORDER BY created_at
-            LIMIT 20
-        """, [session_id]).fetchall()
-        
-        print(f"\nFirst 20 events in session:")
-        for i, (event_type, event_data, created_at) in enumerate(session_events):
-            data = json.loads(event_data)
-            print(f"\n{i+1}. {event_type} at {created_at}")
-            
-            # Show relevant info based on event type
-            if event_type == 'generation_completion':
-                if 'response' in data and 'tool_calls' in data['response']:
-                    print(f"   Tool calls: {data['response']['tool_calls']}")
-            elif event_type == 'runtime_event':
-                if 'metadata' in data:
-                    print(f"   Action: {data['metadata'].get('action_name', 'Unknown')}")
-            elif event_type == 'environment_event':
-                if 'reward' in data:
-                    print(f"   Reward: {data['reward']}")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_traces(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")
-        print("Available databases:")
-        for db in Path(".").glob("**/traces.duckdb"):
-            print(f"  - {db}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_db_schema.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-"""Check DuckDB schema to understand table structure."""
-
-import duckdb
-from pathlib import Path
-
-def check_schema(db_path: str):
-    """Check database schema."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Checking database schema...\n")
-    
-    # Get all tables
-    tables = conn.execute("SHOW TABLES").fetchall()
-    print("📋 Tables in database:")
-    for table in tables:
-        print(f"  - {table[0]}")
-    
-    # Check schema of key tables
-    key_tables = ['session_traces', 'events', 'messages', 'session_timesteps']
-    
-    for table_name in key_tables:
-        if any(t[0] == table_name for t in tables):
-            print(f"\n📊 Schema for {table_name}:")
-            print("-" * 50)
-            schema = conn.execute(f"DESCRIBE {table_name}").fetchall()
-            for col_name, col_type, _, _, _, _ in schema:
-                print(f"  {col_name}: {col_type}")
-            
-            # Show sample data
-            print(f"\n📄 Sample data from {table_name} (first 2 rows):")
-            sample = conn.execute(f"SELECT * FROM {table_name} LIMIT 2").fetchall()
-            if sample:
-                # Get column names
-                cols = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
-                print(f"  Columns: {cols}")
-                for row in sample:
-                    print(f"  {row}")
-            else:
-                print("  (No data)")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        check_schema(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_latest_results.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-"""Check the latest results JSON file."""
-
-import json
-import glob
-from pathlib import Path
-
-# Find the latest results file
-result_files = glob.glob("crafter_lm_synth_results_*.json")
-if not result_files:
-    print("No result files found")
-    exit(1)
-
-# Get the most recent file
-latest_file = max(result_files, key=lambda f: Path(f).stat().st_mtime)
-print(f"📊 Checking latest results: {latest_file}\n")
-
-with open(latest_file) as f:
-    data = json.load(f)
-
-# Extract key metrics
-total_episodes = data.get('total_episodes', 0)
-total_steps = data.get('total_steps', 0)
-model = data.get('model', 'unknown')
-
-print(f"Model: {model}")
-print(f"Episodes: {total_episodes}")
-print(f"Total Steps: {total_steps}")
-
-# Check episode results
-episodes = data.get('episodes', [])
-if episodes:
-    print(f"\n📊 Episode Summary:")
-    for i, ep in enumerate(episodes):
-        steps = ep.get('steps', 0)
-        reward = ep.get('total_reward', 0)
-        achievements = ep.get('achievements_unlocked', [])
-        
-        print(f"\nEpisode {i}:")
-        print(f"  Steps: {steps}")
-        print(f"  Reward: {reward}")
-        print(f"  Achievements: {len(achievements)}")
-        if achievements:
-            print(f"    - {', '.join(achievements[:5])}")
-            if len(achievements) > 5:
-                print(f"    ... and {len(achievements) - 5} more")
-        
-        # Check inventory at end
-        inventory = ep.get('final_inventory', {})
-        non_zero = {k: v for k, v in inventory.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
-        if non_zero:
-            print(f"  Final inventory: {non_zero}")
-        
-        # Sample actions
-        if 'action_history' in ep and ep['action_history']:
-            print(f"  Sample actions: {ep['action_history'][:5]}")
-
-# Overall statistics
-avg_reward = sum(ep.get('total_reward', 0) for ep in episodes) / len(episodes) if episodes else 0
-total_achievements = sum(len(ep.get('achievements_unlocked', [])) for ep in episodes)
-
-print(f"\n📊 Overall Statistics:")
-print(f"Average reward per episode: {avg_reward:.2f}")
-print(f"Total achievements unlocked: {total_achievements}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/debug_agent_responses.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""Debug why agent is not using multiple actions."""
-
-import json
-import sys
-from pathlib import Path
-
-# Read the latest run output from stdin or file
-if len(sys.argv) > 1:
-    with open(sys.argv[1]) as f:
-        output = f.read()
-else:
-    # Check for latest log file
-    log_files = list(Path(".").glob("crafter_run_*.log"))
-    if not log_files:
-        print("No log files found. Run with: python test_crafter_react_agent_lm_synth.py --model 'Qwen/Qwen2.5-14B-Instruct' --episodes 1 --max-steps 10 --verbose 2>&1 | tee crafter_run.log")
-        exit(1)
-    
-    latest_log = max(log_files, key=lambda f: f.stat().st_mtime)
-    with open(latest_log) as f:
-        output = f.read()
-
-# Parse tool calls
-tool_calls = []
-single_action_count = 0
-multi_action_count = 0
-
-for line in output.split('\n'):
-    if "🔧 Turn" in line and "Tool Call:" in line:
-        turn_info = {"turn": line}
-        tool_calls.append(turn_info)
-    elif "Actions:" in line and tool_calls:
-        actions_str = line.strip().split("Actions:")[1].strip()
-        try:
-            # Parse the action list
-            actions = eval(actions_str) if actions_str else []
-            tool_calls[-1]["actions"] = actions
-            tool_calls[-1]["action_count"] = len(actions)
-            
-            if len(actions) == 1:
-                single_action_count += 1
-            elif len(actions) > 1:
-                multi_action_count += 1
-        except:
-            tool_calls[-1]["actions"] = "parse_error"
-            tool_calls[-1]["action_count"] = 0
-
-print("🔍 AGENT ACTION ANALYSIS\n")
-print(f"Total tool calls: {len(tool_calls)}")
-print(f"Single action calls: {single_action_count}")
-print(f"Multi-action calls: {multi_action_count}")
-print(f"Average actions per call: {sum(tc.get('action_count', 0) for tc in tool_calls) / len(tool_calls) if tool_calls else 0:.2f}")
-
-# Show distribution
-action_counts = {}
-for tc in tool_calls:
-    count = tc.get('action_count', 0)
-    action_counts[count] = action_counts.get(count, 0) + 1
-
-print("\nAction count distribution:")
-for count in sorted(action_counts.keys()):
-    print(f"  {count} actions: {action_counts[count]} times")
-
-# Show examples of multi-action calls
-print("\n📋 Multi-action examples:")
-multi_examples = [tc for tc in tool_calls if tc.get('action_count', 0) > 1]
-for example in multi_examples[:5]:
-    print(f"  {example['turn']}")
-    print(f"    Actions: {example['actions']}")
-
-# Check for response parsing issues
-print("\n🔍 Response preview analysis:")
-response_previews = []
-for line in output.split('\n'):
-    if "📝 Raw response preview:" in line:
-        preview = line.split("preview:")[1].strip()
-        response_previews.append(preview)
-
-if response_previews:
-    print(f"Found {len(response_previews)} response previews")
-    # Check if responses mention multiple actions
-    multi_action_mentions = 0
-    for preview in response_previews[:5]:
-        if any(word in preview.lower() for word in ['multiple', 'sequence', 'then', 'after']):
-            multi_action_mentions += 1
-        print(f"  - {preview[:100]}...")
-    
-    print(f"\nResponses mentioning sequences: {multi_action_mentions}/{len(response_previews[:5])}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/quick_trace_check.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-"""Quick check of recent traces without locking the database."""
-
-import sqlite3
-import json
-from pathlib import Path
-from collections import Counter
-
-# Use SQLite interface which is more permissive with locks
-db_path = "./traces_v2_synth/traces.duckdb"
-
-if Path(db_path).exists():
-    try:
-        # DuckDB files can be read with SQLite in read-only mode
-        conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
-        cursor = conn.cursor()
-        
-        print("🔍 Quick trace analysis...\n")
-        
-        # Get recent events
-        cursor.execute("""
-            SELECT event_type, metadata 
-            FROM events 
-            ORDER BY id DESC 
-            LIMIT 100
-        """)
-        
-        events = cursor.fetchall()
-        print(f"Found {len(events)} recent events\n")
-        
-        # Count event types
-        event_types = Counter([e[0] for e in events])
-        print("Event type distribution:")
-        for etype, count in event_types.most_common():
-            print(f"  {etype}: {count}")
-        
-        # Check for actions
-        print("\n🎮 Recent actions:")
-        action_count = 0
-        action_types = Counter()
-        
-        for event_type, metadata_str in events:
-            if metadata_str and event_type == 'runtime':
-                try:
-                    metadata = json.loads(metadata_str)
-                    if 'action_name' in metadata:
-                        action_types[metadata['action_name']] += 1
-                        action_count += 1
-                        if action_count <= 10:
-                            print(f"  - {metadata['action_name']}")
-                except:
-                    pass
-        
-        if action_types:
-            print(f"\nAction summary:")
-            for action, count in action_types.most_common():
-                print(f"  {action}: {count}")
-        
-        conn.close()
-        
-    except Exception as e:
-        print(f"Error: {e}")
-        print("\nTrying alternative analysis...")
-        
-        # If we can't read the DB, check for any JSON trace files
-        trace_files = list(Path("./traces_v2_synth").glob("session_*.json"))
-        if trace_files:
-            print(f"Found {len(trace_files)} JSON trace files")
-            latest = max(trace_files, key=lambda f: f.stat().st_mtime)
-            print(f"Latest: {latest.name}")
-            
-            with open(latest) as f:
-                data = json.load(f)
-                print(f"Session ID: {data.get('session_id', 'Unknown')}")
-                print(f"Events: {len(data.get('events', []))}")
-else:
-    print("No trace database found")