synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_no_rewards.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze why there are no rewards or achievements."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import Counter, defaultdict
-
-def analyze_no_rewards(db_path: str):
-    """Analyze why there are no rewards despite actions being taken."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing why there are no rewards or achievements...\n")
-    
-    # Get the latest experiment
-    latest_exp = conn.execute("""
-        SELECT experiment_id, experiment_name
-        FROM experiments
-        ORDER BY created_at DESC
-        LIMIT 1
-    """).fetchone()
-    
-    if not latest_exp:
-        print("No experiments found")
-        return
-        
-    exp_id, exp_name = latest_exp
-    print(f"📊 Latest Experiment: {exp_name}")
-    print(f"   ID: {exp_id}\n")
-    
-    # Get all sessions from latest experiment
-    sessions = conn.execute("""
-        SELECT DISTINCT e.session_id
-        FROM events e
-        JOIN session_timesteps st ON e.timestep_id = st.id
-        WHERE st.experiment_id = ?
-        ORDER BY e.session_id
-    """, [exp_id]).fetchall()
-    
-    print(f"Found {len(sessions)} sessions\n")
-    
-    # Analyze each session
-    for i, (session_id,) in enumerate(sessions[:3]):  # First 3 sessions
-        print(f"\n{'='*60}")
-        print(f"SESSION {i}: {session_id}")
-        print(f"{'='*60}")
-        
-        # Get all events for this session
-        events = conn.execute("""
-            SELECT event_type, metadata, system_state_after, reward
-            FROM events
-            WHERE session_id = ?
-            ORDER BY id
-        """, [session_id]).fetchall()
-        
-        # Track actions and their results
-        action_results = []
-        total_reward = 0
-        achievements_timeline = []
-        inventory_timeline = []
-        
-        for event_type, metadata_str, state_after_str, reward in events:
-            if event_type == 'runtime' and metadata_str:
-                metadata = json.loads(metadata_str)
-                action_name = metadata.get('action_name', 'unknown')
-                
-                # Look for the corresponding environment event
-                result = {
-                    'action': action_name,
-                    'reward': reward or 0,
-                    'achievements_unlocked': []
-                }
-                
-                if state_after_str:
-                    state_after = json.loads(state_after_str)
-                    if 'public_state' in state_after:
-                        ps = state_after['public_state']
-                        
-                        # Check achievements
-                        if 'achievements_status' in ps:
-                            unlocked = [k for k, v in ps['achievements_status'].items() if v]
-                            achievements_timeline.append((action_name, unlocked))
-                            if len(unlocked) > len(result['achievements_unlocked']):
-                                result['achievements_unlocked'] = unlocked
-                        
-                        # Check inventory
-                        if 'inventory' in ps:
-                            inv = ps['inventory']
-                            non_zero = {k: v for k, v in inv.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
-                            if non_zero:
-                                inventory_timeline.append((action_name, non_zero))
-                
-                action_results.append(result)
-                if reward:
-                    total_reward += reward
-        
-        # Analyze action effectiveness
-        print(f"\n📊 ACTION ANALYSIS")
-        print(f"Total actions: {len(action_results)}")
-        print(f"Total reward: {total_reward}")
-        
-        # Count actions by type
-        action_counts = Counter([r['action'] for r in action_results])
-        print(f"\nAction distribution:")
-        for action, count in action_counts.most_common(10):
-            print(f"  {action:20} {count:3}")
-        
-        # Check for successful resource collection
-        print(f"\n📦 RESOURCE COLLECTION")
-        successful_collections = []
-        for i, (action, inv) in enumerate(inventory_timeline):
-            if i > 0 and inv != inventory_timeline[i-1][1]:
-                # Inventory changed
-                prev_inv = inventory_timeline[i-1][1] if i > 0 else {}
-                new_items = {k: v for k, v in inv.items() if v > prev_inv.get(k, 0)}
-                if new_items:
-                    successful_collections.append((action, new_items))
-        
-        if successful_collections:
-            print("Successful collections:")
-            for action, items in successful_collections[:5]:
-                print(f"  After '{action}': gained {items}")
-        else:
-            print("No successful resource collections detected!")
-        
-        # Check specific action sequences
-        print(f"\n🔍 ACTION SEQUENCE ANALYSIS")
-        # Look for 'do' actions and their context
-        do_actions = []
-        for i, result in enumerate(action_results):
-            if result['action'] == 'do':
-                context = {
-                    'prev_action': action_results[i-1]['action'] if i > 0 else 'start',
-                    'next_action': action_results[i+1]['action'] if i < len(action_results)-1 else 'end',
-                    'reward': result['reward']
-                }
-                do_actions.append(context)
-        
-        if do_actions:
-            print(f"Found {len(do_actions)} 'do' actions")
-            # Check what happened before 'do' actions
-            prev_action_counts = Counter([d['prev_action'] for d in do_actions])
-            print("Actions before 'do':")
-            for action, count in prev_action_counts.most_common(5):
-                print(f"  {action}: {count}")
-        
-        # Check for make_wood_pickaxe attempts
-        pickaxe_attempts = [r for r in action_results if r['action'] == 'make_wood_pickaxe']
-        if pickaxe_attempts:
-            print(f"\n🔨 PICKAXE CRAFTING")
-            print(f"Attempted to make wood pickaxe {len(pickaxe_attempts)} times")
-        
-        # Final inventory check
-        if inventory_timeline:
-            final_inv = inventory_timeline[-1][1]
-            print(f"\n📦 FINAL INVENTORY: {final_inv if final_inv else 'Empty'}")
-        
-        # Achievement check
-        if achievements_timeline:
-            final_achievements = achievements_timeline[-1][1]
-            print(f"🏆 FINAL ACHIEVEMENTS: {final_achievements if final_achievements else 'None'}")
-    
-    # Check for any rewards across all sessions
-    print(f"\n\n{'='*60}")
-    print("📊 OVERALL REWARD ANALYSIS")
-    print(f"{'='*60}")
-    
-    total_rewards = conn.execute("""
-        SELECT SUM(reward) as total, COUNT(*) as count
-        FROM events
-        WHERE reward IS NOT NULL AND reward != 0
-        AND session_id IN (
-            SELECT DISTINCT e.session_id
-            FROM events e
-            JOIN session_timesteps st ON e.timestep_id = st.id
-            WHERE st.experiment_id = ?
-        )
-    """, [exp_id]).fetchone()
-    
-    if total_rewards:
-        total, count = total_rewards
-        print(f"Total non-zero rewards: {total or 0}")
-        print(f"Number of reward events: {count or 0}")
-    
-    # Check if rewards are being recorded at all
-    print("\n🔍 REWARD RECORDING CHECK")
-    sample_rewards = conn.execute("""
-        SELECT reward, metadata
-        FROM events
-        WHERE event_type = 'environment'
-        AND session_id IN (
-            SELECT DISTINCT e.session_id
-            FROM events e
-            JOIN session_timesteps st ON e.timestep_id = st.id
-            WHERE st.experiment_id = ?
-        )
-        LIMIT 20
-    """, [exp_id]).fetchall()
-    
-    reward_values = [r for r, _ in sample_rewards if r is not None]
-    print(f"Sample reward values: {reward_values[:10]}")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_no_rewards(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/analyze_trace_issue.py

@@ -1,206 +0,0 @@
-#!/usr/bin/env python3
-"""Analyze trace data to understand why no achievements were unlocked."""
-
-import duckdb
-import json
-from pathlib import Path
-from collections import defaultdict, Counter
-
-def analyze_traces(db_path: str):
-    """Analyze trace data to identify issues."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Analyzing trace data...\n")
-    
-    # 1. Check basic statistics
-    print("📊 BASIC STATISTICS")
-    print("=" * 50)
-    
-    # First, show available tables
-    tables = conn.execute("SHOW TABLES").fetchall()
-    print("Available tables:")
-    for table in tables:
-        print(f"  - {table[0]}")
-    
-    # Count sessions
-    session_count = conn.execute("SELECT COUNT(DISTINCT session_id) FROM session_traces").fetchone()[0]
-    print(f"\nTotal sessions: {session_count}")
-    
-    # Count events
-    event_count = conn.execute("SELECT COUNT(*) FROM session_traces").fetchone()[0]
-    print(f"Total events: {event_count}")
-    
-    # Check event types
-    print("\n📋 EVENT TYPE DISTRIBUTION")
-    print("-" * 30)
-    event_types = conn.execute("""
-        SELECT event_type, COUNT(*) as count 
-        FROM session_traces 
-        GROUP BY event_type 
-        ORDER BY count DESC
-    """).fetchall()
-    
-    for event_type, count in event_types:
-        print(f"{event_type}: {count}")
-    
-    # 2. Analyze agent decisions
-    print("\n🤖 AGENT DECISIONS ANALYSIS")
-    print("=" * 50)
-    
-    # Get all generation completion events
-    completions = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'generation_completion'
-        LIMIT 50
-    """).fetchall()
-    
-    if completions:
-        print(f"Found {len(completions)} generation completions (showing first 50)")
-        
-        # Analyze first few completions
-        for i, (event_data,) in enumerate(completions[:5]):
-            data = json.loads(event_data)
-            print(f"\n--- Completion {i+1} ---")
-            
-            # Extract response
-            if 'response' in data and data['response']:
-                response = data['response']
-                if 'content' in response:
-                    print(f"Content preview: {response['content'][:200]}...")
-                if 'tool_calls' in response:
-                    print(f"Tool calls: {response['tool_calls']}")
-            else:
-                print("No response found in event data")
-    else:
-        print("❌ No generation completion events found!")
-    
-    # 3. Analyze runtime events (actions taken)
-    print("\n🎮 RUNTIME EVENTS (ACTIONS)")
-    print("=" * 50)
-    
-    runtime_events = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'runtime_event'
-        LIMIT 100
-    """).fetchall()
-    
-    if runtime_events:
-        action_counter = Counter()
-        
-        for (event_data,) in runtime_events:
-            data = json.loads(event_data)
-            if 'metadata' in data and 'action_name' in data['metadata']:
-                action_counter[data['metadata']['action_name']] += 1
-        
-        print(f"Found {len(runtime_events)} runtime events")
-        print("\nAction distribution:")
-        for action, count in action_counter.most_common():
-            print(f"  {action}: {count}")
-    else:
-        print("❌ No runtime events found!")
-    
-    # 4. Analyze environment events (results)
-    print("\n🌍 ENVIRONMENT EVENTS")
-    print("=" * 50)
-    
-    env_events = conn.execute("""
-        SELECT event_data 
-        FROM session_traces 
-        WHERE event_type = 'environment_event'
-        LIMIT 100
-    """).fetchall()
-    
-    if env_events:
-        reward_sum = 0
-        achievements_found = []
-        
-        for (event_data,) in env_events:
-            data = json.loads(event_data)
-            
-            # Check rewards
-            if 'reward' in data:
-                reward_sum += data['reward'] or 0
-            
-            # Check for achievements in state
-            if 'system_state_after' in data:
-                state = data['system_state_after']
-                if 'public_state' in state and 'achievements' in state['public_state']:
-                    achievements = state['public_state']['achievements']
-                    for ach, unlocked in achievements.items():
-                        if unlocked:
-                            achievements_found.append(ach)
-        
-        print(f"Found {len(env_events)} environment events")
-        print(f"Total reward across all events: {reward_sum}")
-        print(f"Achievements found: {set(achievements_found) if achievements_found else 'None'}")
-    else:
-        print("❌ No environment events found!")
-    
-    # 5. Check for errors
-    print("\n⚠️ ERROR CHECK")
-    print("=" * 50)
-    
-    # Look for error messages in events
-    error_events = conn.execute("""
-        SELECT event_type, event_data 
-        FROM session_traces 
-        WHERE event_data LIKE '%error%' OR event_data LIKE '%Error%'
-        LIMIT 10
-    """).fetchall()
-    
-    if error_events:
-        print(f"Found {len(error_events)} events with potential errors:")
-        for event_type, event_data in error_events[:3]:
-            print(f"\n{event_type}:")
-            data = json.loads(event_data)
-            print(json.dumps(data, indent=2)[:500])
-    else:
-        print("No obvious errors found in events")
-    
-    # 6. Sample a full episode flow
-    print("\n📖 SAMPLE EPISODE FLOW")
-    print("=" * 50)
-    
-    # Get events from first session
-    first_session = conn.execute("SELECT DISTINCT session_id FROM session_traces LIMIT 1").fetchone()
-    if first_session:
-        session_id = first_session[0]
-        print(f"Analyzing session: {session_id}")
-        
-        session_events = conn.execute("""
-            SELECT event_type, event_data, created_at
-            FROM session_traces 
-            WHERE session_id = ?
-            ORDER BY created_at
-            LIMIT 20
-        """, [session_id]).fetchall()
-        
-        print(f"\nFirst 20 events in session:")
-        for i, (event_type, event_data, created_at) in enumerate(session_events):
-            data = json.loads(event_data)
-            print(f"\n{i+1}. {event_type} at {created_at}")
-            
-            # Show relevant info based on event type
-            if event_type == 'generation_completion':
-                if 'response' in data and 'tool_calls' in data['response']:
-                    print(f"   Tool calls: {data['response']['tool_calls']}")
-            elif event_type == 'runtime_event':
-                if 'metadata' in data:
-                    print(f"   Action: {data['metadata'].get('action_name', 'Unknown')}")
-            elif event_type == 'environment_event':
-                if 'reward' in data:
-                    print(f"   Reward: {data['reward']}")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        analyze_traces(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")
-        print("Available databases:")
-        for db in Path(".").glob("**/traces.duckdb"):
-            print(f"  - {db}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_db_schema.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-"""Check DuckDB schema to understand table structure."""
-
-import duckdb
-from pathlib import Path
-
-def check_schema(db_path: str):
-    """Check database schema."""
-    conn = duckdb.connect(db_path, read_only=True)
-    
-    print("🔍 Checking database schema...\n")
-    
-    # Get all tables
-    tables = conn.execute("SHOW TABLES").fetchall()
-    print("📋 Tables in database:")
-    for table in tables:
-        print(f"  - {table[0]}")
-    
-    # Check schema of key tables
-    key_tables = ['session_traces', 'events', 'messages', 'session_timesteps']
-    
-    for table_name in key_tables:
-        if any(t[0] == table_name for t in tables):
-            print(f"\n📊 Schema for {table_name}:")
-            print("-" * 50)
-            schema = conn.execute(f"DESCRIBE {table_name}").fetchall()
-            for col_name, col_type, _, _, _, _ in schema:
-                print(f"  {col_name}: {col_type}")
-            
-            # Show sample data
-            print(f"\n📄 Sample data from {table_name} (first 2 rows):")
-            sample = conn.execute(f"SELECT * FROM {table_name} LIMIT 2").fetchall()
-            if sample:
-                # Get column names
-                cols = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
-                print(f"  Columns: {cols}")
-                for row in sample:
-                    print(f"  {row}")
-            else:
-                print("  (No data)")
-    
-    conn.close()
-
-if __name__ == "__main__":
-    db_path = "./traces_v2_synth/traces.duckdb"
-    if Path(db_path).exists():
-        check_schema(db_path)
-    else:
-        print(f"❌ Database not found at {db_path}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/check_latest_results.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-"""Check the latest results JSON file."""
-
-import json
-import glob
-from pathlib import Path
-
-# Find the latest results file
-result_files = glob.glob("crafter_lm_synth_results_*.json")
-if not result_files:
-    print("No result files found")
-    exit(1)
-
-# Get the most recent file
-latest_file = max(result_files, key=lambda f: Path(f).stat().st_mtime)
-print(f"📊 Checking latest results: {latest_file}\n")
-
-with open(latest_file) as f:
-    data = json.load(f)
-
-# Extract key metrics
-total_episodes = data.get('total_episodes', 0)
-total_steps = data.get('total_steps', 0)
-model = data.get('model', 'unknown')
-
-print(f"Model: {model}")
-print(f"Episodes: {total_episodes}")
-print(f"Total Steps: {total_steps}")
-
-# Check episode results
-episodes = data.get('episodes', [])
-if episodes:
-    print(f"\n📊 Episode Summary:")
-    for i, ep in enumerate(episodes):
-        steps = ep.get('steps', 0)
-        reward = ep.get('total_reward', 0)
-        achievements = ep.get('achievements_unlocked', [])
-        
-        print(f"\nEpisode {i}:")
-        print(f"  Steps: {steps}")
-        print(f"  Reward: {reward}")
-        print(f"  Achievements: {len(achievements)}")
-        if achievements:
-            print(f"    - {', '.join(achievements[:5])}")
-            if len(achievements) > 5:
-                print(f"    ... and {len(achievements) - 5} more")
-        
-        # Check inventory at end
-        inventory = ep.get('final_inventory', {})
-        non_zero = {k: v for k, v in inventory.items() if v > 0 and k not in ['health', 'food', 'drink', 'energy']}
-        if non_zero:
-            print(f"  Final inventory: {non_zero}")
-        
-        # Sample actions
-        if 'action_history' in ep and ep['action_history']:
-            print(f"  Sample actions: {ep['action_history'][:5]}")
-
-# Overall statistics
-avg_reward = sum(ep.get('total_reward', 0) for ep in episodes) / len(episodes) if episodes else 0
-total_achievements = sum(len(ep.get('achievements_unlocked', [])) for ep in episodes)
-
-print(f"\n📊 Overall Statistics:")
-print(f"Average reward per episode: {avg_reward:.2f}")
-print(f"Total achievements unlocked: {total_achievements}")

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/old/debug_agent_responses.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""Debug why agent is not using multiple actions."""
-
-import json
-import sys
-from pathlib import Path
-
-# Read the latest run output from stdin or file
-if len(sys.argv) > 1:
-    with open(sys.argv[1]) as f:
-        output = f.read()
-else:
-    # Check for latest log file
-    log_files = list(Path(".").glob("crafter_run_*.log"))
-    if not log_files:
-        print("No log files found. Run with: python test_crafter_react_agent_lm_synth.py --model 'Qwen/Qwen2.5-14B-Instruct' --episodes 1 --max-steps 10 --verbose 2>&1 | tee crafter_run.log")
-        exit(1)
-    
-    latest_log = max(log_files, key=lambda f: f.stat().st_mtime)
-    with open(latest_log) as f:
-        output = f.read()
-
-# Parse tool calls
-tool_calls = []
-single_action_count = 0
-multi_action_count = 0
-
-for line in output.split('\n'):
-    if "🔧 Turn" in line and "Tool Call:" in line:
-        turn_info = {"turn": line}
-        tool_calls.append(turn_info)
-    elif "Actions:" in line and tool_calls:
-        actions_str = line.strip().split("Actions:")[1].strip()
-        try:
-            # Parse the action list
-            actions = eval(actions_str) if actions_str else []
-            tool_calls[-1]["actions"] = actions
-            tool_calls[-1]["action_count"] = len(actions)
-            
-            if len(actions) == 1:
-                single_action_count += 1
-            elif len(actions) > 1:
-                multi_action_count += 1
-        except:
-            tool_calls[-1]["actions"] = "parse_error"
-            tool_calls[-1]["action_count"] = 0
-
-print("🔍 AGENT ACTION ANALYSIS\n")
-print(f"Total tool calls: {len(tool_calls)}")
-print(f"Single action calls: {single_action_count}")
-print(f"Multi-action calls: {multi_action_count}")
-print(f"Average actions per call: {sum(tc.get('action_count', 0) for tc in tool_calls) / len(tool_calls) if tool_calls else 0:.2f}")
-
-# Show distribution
-action_counts = {}
-for tc in tool_calls:
-    count = tc.get('action_count', 0)
-    action_counts[count] = action_counts.get(count, 0) + 1
-
-print("\nAction count distribution:")
-for count in sorted(action_counts.keys()):
-    print(f"  {count} actions: {action_counts[count]} times")
-
-# Show examples of multi-action calls
-print("\n📋 Multi-action examples:")
-multi_examples = [tc for tc in tool_calls if tc.get('action_count', 0) > 1]
-for example in multi_examples[:5]:
-    print(f"  {example['turn']}")
-    print(f"    Actions: {example['actions']}")
-
-# Check for response parsing issues
-print("\n🔍 Response preview analysis:")
-response_previews = []
-for line in output.split('\n'):
-    if "📝 Raw response preview:" in line:
-        preview = line.split("preview:")[1].strip()
-        response_previews.append(preview)
-
-if response_previews:
-    print(f"Found {len(response_previews)} response previews")
-    # Check if responses mention multiple actions
-    multi_action_mentions = 0
-    for preview in response_previews[:5]:
-        if any(word in preview.lower() for word in ['multiple', 'sequence', 'then', 'after']):
-            multi_action_mentions += 1
-        print(f"  - {preview[:100]}...")
-    
-    print(f"\nResponses mentioning sequences: {multi_action_mentions}/{len(response_previews[:5])}")