synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/eval_by_difficulty.py

@@ -1,126 +0,0 @@
-#!/usr/bin/env python3
-"""
-Evaluate traces grouped by difficulty level.
-"""
-
-import json
-from pathlib import Path
-from collections import defaultdict
-from trace_eval import evaluate_trace, WEIGHTS
-
-def get_trace_difficulty(trace_path: Path) -> str:
-    """Extract difficulty from trace metadata."""
-    try:
-        with open(trace_path, 'r') as f:
-            data = json.load(f)
-        
-        # Try to find difficulty in metadata
-        metadata = data.get('metadata', {})
-        if 'difficulty' in metadata:
-            return metadata['difficulty']
-        
-        # Try to find in task instance metadata
-        if 'task_instance' in metadata:
-            task_metadata = metadata['task_instance'].get('metadata', {})
-            if 'difficulty' in task_metadata:
-                return task_metadata['difficulty']
-        
-        return 'unknown'
-    except:
-        return 'unknown'
-
-def main():
-    traces_dir = Path("traces")
-    if not traces_dir.exists():
-        print(f"Traces directory not found: {traces_dir}")
-        return
-    
-    # Group traces by difficulty
-    traces_by_difficulty = defaultdict(list)
-    
-    for trace_file in traces_dir.glob("*.json"):
-        difficulty = get_trace_difficulty(trace_file)
-        result = evaluate_trace(trace_file)
-        traces_by_difficulty[difficulty].append(result)
-    
-    # Sort difficulties
-    difficulty_order = ['easy', 'medium', 'hard', 'unknown']
-    
-    print("=" * 80)
-    print("CRAFTER EVALUATION BY DIFFICULTY")
-    print("=" * 80)
-    
-    for difficulty in difficulty_order:
-        traces = traces_by_difficulty[difficulty]
-        if not traces:
-            continue
-        
-        print(f"\n{difficulty.upper()} ({len(traces)} traces)")
-        print("-" * 40)
-        
-        # Calculate statistics
-        scores = [t['total_score'] for t in traces]
-        avg_score = sum(scores) / len(scores) if scores else 0
-        max_score = max(scores) if scores else 0
-        min_score = min(scores) if scores else 0
-        
-        # Count achievements and invalid actions
-        total_easy = sum(t['counts'].get('easy_achievement', 0) for t in traces)
-        total_medium = sum(t['counts'].get('medium_achievement', 0) for t in traces)
-        total_hard = sum(t['counts'].get('hard_achievement', 0) for t in traces)
-        total_invalid = sum(t['counts'].get('invalid_action', 0) for t in traces)
-        
-        print(f"Average Score: {avg_score:.2f}")
-        print(f"Score Range: {min_score:.2f} to {max_score:.2f}")
-        print(f"\nAchievements per trace:")
-        print(f"  Easy:   {total_easy / len(traces):.2f}")
-        print(f"  Medium: {total_medium / len(traces):.2f}")
-        print(f"  Hard:   {total_hard / len(traces):.2f}")
-        print(f"\nInvalid actions per trace: {total_invalid / len(traces):.2f}")
-        
-        # Show score distribution
-        positive_scores = [s for s in scores if s > 0]
-        zero_scores = [s for s in scores if s == 0]
-        negative_scores = [s for s in scores if s < 0]
-        
-        print(f"\nScore distribution:")
-        print(f"  Positive: {len(positive_scores)} ({len(positive_scores)/len(scores)*100:.1f}%)")
-        print(f"  Zero:     {len(zero_scores)} ({len(zero_scores)/len(scores)*100:.1f}%)")
-        print(f"  Negative: {len(negative_scores)} ({len(negative_scores)/len(scores)*100:.1f}%)")
-        
-        # Show top 3 traces
-        traces_sorted = sorted(traces, key=lambda x: x['total_score'], reverse=True)
-        print(f"\nTop 3 traces:")
-        for i, trace in enumerate(traces_sorted[:3], 1):
-            print(f"  {i}. Score: {trace['total_score']:.2f}, Trajectory: {trace['trajectory'][:50]}")
-    
-    # Overall summary
-    print("\n" + "=" * 80)
-    print("OVERALL SUMMARY")
-    print("=" * 80)
-    
-    all_traces = []
-    for traces in traces_by_difficulty.values():
-        all_traces.extend(traces)
-    
-    if all_traces:
-        all_scores = [t['total_score'] for t in all_traces]
-        print(f"Total traces evaluated: {len(all_traces)}")
-        print(f"Overall average score: {sum(all_scores) / len(all_scores):.2f}")
-        
-        # Achievement type distribution
-        total_achievements = defaultdict(int)
-        for trace in all_traces:
-            for achievement_type in ['easy_achievement', 'medium_achievement', 'hard_achievement']:
-                total_achievements[achievement_type] += trace['counts'].get(achievement_type, 0)
-        
-        print(f"\nTotal achievements unlocked:")
-        print(f"  Easy:   {total_achievements['easy_achievement']} (worth {total_achievements['easy_achievement'] * WEIGHTS['easy_achievement']:.1f} points)")
-        print(f"  Medium: {total_achievements['medium_achievement']} (worth {total_achievements['medium_achievement'] * WEIGHTS['medium_achievement']:.1f} points)")  
-        print(f"  Hard:   {total_achievements['hard_achievement']} (worth {total_achievements['hard_achievement'] * WEIGHTS['hard_achievement']:.1f} points)")
-        
-        total_invalid = sum(t['counts'].get('invalid_action', 0) for t in all_traces)
-        print(f"\nTotal invalid actions: {total_invalid} (penalty: {total_invalid * WEIGHTS['invalid_action']:.1f} points)")
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_classic/agent_demos/old/eval_example.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-"""
-Example of using the trace evaluation system programmatically.
-"""
-
-from pathlib import Path
-from trace_eval import evaluate_trace, evaluate_all_traces, print_trace_evaluation, print_evaluation_summary
-
-def main():
-    # Example 1: Evaluate a single trace
-    print("=" * 60)
-    print("Example 1: Evaluating a single trace")
-    print("=" * 60)
-    
-    # Pick a high-scoring trace
-    trace_path = Path("traces/session_crafter_episode_1_f2cea96d-34b6-46a3-9991-fe74ef263462_20250724_162140.json")
-    if trace_path.exists():
-        result = evaluate_trace(trace_path)
-        print_trace_evaluation(result)
-    else:
-        print(f"Trace file not found: {trace_path}")
-    
-    # Example 2: Evaluate all traces and show top 5
-    print("\n" + "=" * 60)
-    print("Example 2: Top 5 traces by score")
-    print("=" * 60)
-    
-    traces_dir = Path("traces")
-    if traces_dir.exists():
-        all_results = evaluate_all_traces(traces_dir)
-        
-        # Show only top 5
-        print(f"\nFound {len(all_results)} traces. Showing top 5:\n")
-        for i, result in enumerate(all_results[:5], 1):
-            print(f"{i}. {result['trace_file']}")
-            print(f"   Score: {result['total_score']:.2f}")
-            print(f"   Trajectory: {result['trajectory']}")
-            if result['counts']:
-                print("   Breakdown:")
-                for score_type, count in result['counts'].items():
-                    weight = {
-                        'easy_achievement': 1.0,
-                        'medium_achievement': 2.5,
-                        'hard_achievement': 5.0,
-                        'invalid_action': -0.05
-                    }[score_type]
-                    print(f"     {score_type}: {count} × {weight} = {count * weight:.2f}")
-            print()
-    
-    # Example 3: Score distribution analysis
-    print("=" * 60)
-    print("Example 3: Score distribution analysis")
-    print("=" * 60)
-    
-    if traces_dir.exists():
-        all_results = evaluate_all_traces(traces_dir)
-        scores = [r['total_score'] for r in all_results]
-        
-        # Group by score ranges
-        score_ranges = {
-            "Negative (<0)": 0,
-            "Low (0-0.5)": 0,
-            "Medium (0.5-1.5)": 0,
-            "High (1.5-2.5)": 0,
-            "Very High (>2.5)": 0
-        }
-        
-        for score in scores:
-            if score < 0:
-                score_ranges["Negative (<0)"] += 1
-            elif score <= 0.5:
-                score_ranges["Low (0-0.5)"] += 1
-            elif score <= 1.5:
-                score_ranges["Medium (0.5-1.5)"] += 1
-            elif score <= 2.5:
-                score_ranges["High (1.5-2.5)"] += 1
-            else:
-                score_ranges["Very High (>2.5)"] += 1
-        
-        print(f"\nScore distribution across {len(scores)} traces:")
-        for range_name, count in score_ranges.items():
-            percentage = (count / len(scores) * 100) if scores else 0
-            bar = "█" * int(percentage / 2)  # Scale to 50 chars max
-            print(f"  {range_name:<20} {count:3d} ({percentage:5.1f}%) {bar}")
-        
-        # Additional statistics
-        if scores:
-            print(f"\nStatistics:")
-            print(f"  Mean score: {sum(scores) / len(scores):.2f}")
-            print(f"  Median score: {sorted(scores)[len(scores)//2]:.2f}")
-            print(f"  Std deviation: {(sum((x - sum(scores)/len(scores))**2 for x in scores) / len(scores))**0.5:.2f}")
-
-if __name__ == "__main__":
-    main()

synth_ai/environments/examples/crafter_classic/agent_demos/old/explore_saved_states.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python3
-"""
-Explore and visualize saved Crafter states
-==========================================
-"""
-
-import gzip
-import pickle
-import json
-from pathlib import Path
-from datetime import datetime
-import asyncio
-from uuid import uuid4
-
-from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
-from synth_ai.environments.examples.crafter_classic.taskset import CrafterTaskInstance, CrafterTaskInstanceMetadata
-from synth_ai.environments.tasks.core import Impetus, Intent
-
-STATES_DIR = Path("synth_ai/environments/examples/crafter_classic/env_states")
-
-
-def list_sessions():
-    """List all saved MCTS sessions."""
-    if not STATES_DIR.exists():
-        print("No env_states directory found!")
-        return []
-    
-    sessions = []
-    for session_dir in sorted(STATES_DIR.iterdir()):
-        if session_dir.is_dir() and session_dir.name.startswith("mcts_"):
-            metadata_file = session_dir / "session_metadata.json"
-            if metadata_file.exists():
-                metadata = json.loads(metadata_file.read_text())
-                sessions.append((session_dir, metadata))
-    
-    return sessions
-
-
-async def explore_state_file(state_file: Path):
-    """Load and display info about a single state file."""
-    print(f"\n📄 State file: {state_file.name}")
-    print(f"   Size: {state_file.stat().st_size / 1024:.1f} KB")
-    
-    # Load the state
-    try:
-        env_blob = gzip.decompress(state_file.read_bytes())
-        env_snapshot = pickle.loads(env_blob)
-        
-        # Create dummy task for deserialization
-        task = CrafterTaskInstance(
-            id=uuid4(),
-            impetus=Impetus(instructions="Analysis"),
-            intent=Intent(rubric={"goal": "Analysis"}, gold_trajectories=None, gold_state_diff={}),
-            metadata=CrafterTaskInstanceMetadata(
-                difficulty="easy", seed=0, num_trees_radius=0, 
-                num_cows_radius=0, num_hostiles_radius=0
-            ),
-            is_reproducible=True,
-            initial_engine_snapshot=None
-        )
-        
-        env = await CrafterClassicEnvironment._deserialize_engine(env_snapshot, task)
-        pub = env.engine._get_public_state_from_env()
-        priv = env.engine._get_private_state_from_env(0, False, False)
-        
-        print(f"   Position: {pub.player_position}")
-        print(f"   Health: {priv.player_internal_stats.get('health', 0)}")
-        print(f"   Steps: {pub.num_steps_taken}")
-        
-        # Show achievements
-        achievements = [k for k, v in pub.achievements_status.items() if v]
-        if achievements:
-            print(f"   Achievements ({len(achievements)}): {', '.join(achievements[:3])}{'...' if len(achievements) > 3 else ''}")
-        
-        # Show inventory
-        inventory = [(k, v) for k, v in pub.inventory.items() 
-                    if v > 0 and k not in ['health', 'food', 'drink', 'energy']]
-        if inventory:
-            print(f"   Inventory: {dict(inventory)}")
-            
-    except Exception as e:
-        print(f"   ❌ Error loading state: {e}")
-
-
-async def main():
-    print("🔍 Exploring Saved Crafter States")
-    print("=" * 60)
-    
-    sessions = list_sessions()
-    
-    if not sessions:
-        print("No saved sessions found!")
-        return
-    
-    print(f"Found {len(sessions)} saved sessions:\n")
-    
-    for i, (session_dir, metadata) in enumerate(sessions):
-        print(f"{i+1}. {session_dir.name}")
-        print(f"   Seed: {metadata['seed']}")
-        print(f"   Time: {metadata['timestamp']}")
-        print(f"   States: {len(list(session_dir.glob('*.snapshot.gz')))}")
-    
-    # Explore the most recent session
-    if sessions:
-        latest_session, latest_metadata = sessions[-1]
-        print(f"\n📂 Exploring latest session: {latest_session.name}")
-        print("-" * 60)
-        
-        # Get all state files
-        state_files = sorted(latest_session.glob("*.snapshot.gz"))
-        print(f"Total states saved: {len(state_files)}")
-        
-        # Show the root state
-        root_id = latest_metadata['root_node_id']
-        root_file = latest_session / f"{root_id}.snapshot.gz"
-        if root_file.exists():
-            print("\n🌱 Root state:")
-            await explore_state_file(root_file)
-        
-        # Show a few random states
-        import random
-        sample_size = min(3, len(state_files) - 1)
-        if sample_size > 0:
-            print("\n🎲 Random sample states:")
-            for state_file in random.sample(state_files[1:], sample_size):
-                await explore_state_file(state_file)
-        
-        # Show summary if it exists
-        summary_file = latest_session / "summary.txt"
-        if summary_file.exists():
-            print("\n📊 Session Summary:")
-            print("-" * 40)
-            print(summary_file.read_text())
-    
-    print("\n✅ Done exploring saved states!")
-    print(f"\n💡 Tip: States are saved in: {STATES_DIR}")
-    print("   Each .snapshot.gz file contains a complete game state")
-    print("   You can analyze these to understand MCTS exploration patterns")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/filter_traces_sft.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-"""
-DEPRECATED: This script reads individual JSON trace files which are no longer generated.
-Please use filter_traces_sft_duckdb.py instead, which reads from DuckDB.
-
-Original functionality:
-- Filter traces to create OpenAI SFT-ready .jsonl files
-- Supports trajectory-level and window-based filtering
-
-Migration guide:
-1. Run your agent with DuckDB enabled (this happens automatically now)
-2. Use filter_traces_sft_duckdb.py to filter and extract training data
-3. Example: python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -o training.jsonl
-
-This file is kept for reference only and will be removed in a future version.
-"""
-
-import sys
-print("=" * 80)
-print("DEPRECATED: This script is no longer supported.")
-print("Please use filter_traces_sft_duckdb.py instead.")
-print("=" * 80)
-print("\nExample usage:")
-print("  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -o training.jsonl")
-print("\nSee DUCKDB_FILTERING_GUIDE.md for more information.")
-sys.exit(1)