synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/eval_by_difficulty.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""
+Evaluate traces grouped by difficulty level.
+"""
+
+import json
+from pathlib import Path
+from collections import defaultdict
+from trace_eval import evaluate_trace, WEIGHTS
+
+def get_trace_difficulty(trace_path: Path) -> str:
+    """Extract difficulty from trace metadata."""
+    try:
+        with open(trace_path, 'r') as f:
+            data = json.load(f)
+        
+        # Try to find difficulty in metadata
+        metadata = data.get('metadata', {})
+        if 'difficulty' in metadata:
+            return metadata['difficulty']
+        
+        # Try to find in task instance metadata
+        if 'task_instance' in metadata:
+            task_metadata = metadata['task_instance'].get('metadata', {})
+            if 'difficulty' in task_metadata:
+                return task_metadata['difficulty']
+        
+        return 'unknown'
+    except:
+        return 'unknown'
+
+def main():
+    traces_dir = Path("traces")
+    if not traces_dir.exists():
+        print(f"Traces directory not found: {traces_dir}")
+        return
+    
+    # Group traces by difficulty
+    traces_by_difficulty = defaultdict(list)
+    
+    for trace_file in traces_dir.glob("*.json"):
+        difficulty = get_trace_difficulty(trace_file)
+        result = evaluate_trace(trace_file)
+        traces_by_difficulty[difficulty].append(result)
+    
+    # Sort difficulties
+    difficulty_order = ['easy', 'medium', 'hard', 'unknown']
+    
+    print("=" * 80)
+    print("CRAFTER EVALUATION BY DIFFICULTY")
+    print("=" * 80)
+    
+    for difficulty in difficulty_order:
+        traces = traces_by_difficulty[difficulty]
+        if not traces:
+            continue
+        
+        print(f"\n{difficulty.upper()} ({len(traces)} traces)")
+        print("-" * 40)
+        
+        # Calculate statistics
+        scores = [t['total_score'] for t in traces]
+        avg_score = sum(scores) / len(scores) if scores else 0
+        max_score = max(scores) if scores else 0
+        min_score = min(scores) if scores else 0
+        
+        # Count achievements and invalid actions
+        total_easy = sum(t['counts'].get('easy_achievement', 0) for t in traces)
+        total_medium = sum(t['counts'].get('medium_achievement', 0) for t in traces)
+        total_hard = sum(t['counts'].get('hard_achievement', 0) for t in traces)
+        total_invalid = sum(t['counts'].get('invalid_action', 0) for t in traces)
+        
+        print(f"Average Score: {avg_score:.2f}")
+        print(f"Score Range: {min_score:.2f} to {max_score:.2f}")
+        print(f"\nAchievements per trace:")
+        print(f"  Easy:   {total_easy / len(traces):.2f}")
+        print(f"  Medium: {total_medium / len(traces):.2f}")
+        print(f"  Hard:   {total_hard / len(traces):.2f}")
+        print(f"\nInvalid actions per trace: {total_invalid / len(traces):.2f}")
+        
+        # Show score distribution
+        positive_scores = [s for s in scores if s > 0]
+        zero_scores = [s for s in scores if s == 0]
+        negative_scores = [s for s in scores if s < 0]
+        
+        print(f"\nScore distribution:")
+        print(f"  Positive: {len(positive_scores)} ({len(positive_scores)/len(scores)*100:.1f}%)")
+        print(f"  Zero:     {len(zero_scores)} ({len(zero_scores)/len(scores)*100:.1f}%)")
+        print(f"  Negative: {len(negative_scores)} ({len(negative_scores)/len(scores)*100:.1f}%)")
+        
+        # Show top 3 traces
+        traces_sorted = sorted(traces, key=lambda x: x['total_score'], reverse=True)
+        print(f"\nTop 3 traces:")
+        for i, trace in enumerate(traces_sorted[:3], 1):
+            print(f"  {i}. Score: {trace['total_score']:.2f}, Trajectory: {trace['trajectory'][:50]}")
+    
+    # Overall summary
+    print("\n" + "=" * 80)
+    print("OVERALL SUMMARY")
+    print("=" * 80)
+    
+    all_traces = []
+    for traces in traces_by_difficulty.values():
+        all_traces.extend(traces)
+    
+    if all_traces:
+        all_scores = [t['total_score'] for t in all_traces]
+        print(f"Total traces evaluated: {len(all_traces)}")
+        print(f"Overall average score: {sum(all_scores) / len(all_scores):.2f}")
+        
+        # Achievement type distribution
+        total_achievements = defaultdict(int)
+        for trace in all_traces:
+            for achievement_type in ['easy_achievement', 'medium_achievement', 'hard_achievement']:
+                total_achievements[achievement_type] += trace['counts'].get(achievement_type, 0)
+        
+        print(f"\nTotal achievements unlocked:")
+        print(f"  Easy:   {total_achievements['easy_achievement']} (worth {total_achievements['easy_achievement'] * WEIGHTS['easy_achievement']:.1f} points)")
+        print(f"  Medium: {total_achievements['medium_achievement']} (worth {total_achievements['medium_achievement'] * WEIGHTS['medium_achievement']:.1f} points)")  
+        print(f"  Hard:   {total_achievements['hard_achievement']} (worth {total_achievements['hard_achievement'] * WEIGHTS['hard_achievement']:.1f} points)")
+        
+        total_invalid = sum(t['counts'].get('invalid_action', 0) for t in all_traces)
+        print(f"\nTotal invalid actions: {total_invalid} (penalty: {total_invalid * WEIGHTS['invalid_action']:.1f} points)")
+
+if __name__ == "__main__":
+    main()

synth_ai/environments/examples/crafter_classic/agent_demos/old/eval_example.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""
+Example of using the trace evaluation system programmatically.
+"""
+
+from pathlib import Path
+from trace_eval import evaluate_trace, evaluate_all_traces, print_trace_evaluation, print_evaluation_summary
+
+def main():
+    # Example 1: Evaluate a single trace
+    print("=" * 60)
+    print("Example 1: Evaluating a single trace")
+    print("=" * 60)
+    
+    # Pick a high-scoring trace
+    trace_path = Path("traces/session_crafter_episode_1_f2cea96d-34b6-46a3-9991-fe74ef263462_20250724_162140.json")
+    if trace_path.exists():
+        result = evaluate_trace(trace_path)
+        print_trace_evaluation(result)
+    else:
+        print(f"Trace file not found: {trace_path}")
+    
+    # Example 2: Evaluate all traces and show top 5
+    print("\n" + "=" * 60)
+    print("Example 2: Top 5 traces by score")
+    print("=" * 60)
+    
+    traces_dir = Path("traces")
+    if traces_dir.exists():
+        all_results = evaluate_all_traces(traces_dir)
+        
+        # Show only top 5
+        print(f"\nFound {len(all_results)} traces. Showing top 5:\n")
+        for i, result in enumerate(all_results[:5], 1):
+            print(f"{i}. {result['trace_file']}")
+            print(f"   Score: {result['total_score']:.2f}")
+            print(f"   Trajectory: {result['trajectory']}")
+            if result['counts']:
+                print("   Breakdown:")
+                for score_type, count in result['counts'].items():
+                    weight = {
+                        'easy_achievement': 1.0,
+                        'medium_achievement': 2.5,
+                        'hard_achievement': 5.0,
+                        'invalid_action': -0.05
+                    }[score_type]
+                    print(f"     {score_type}: {count} × {weight} = {count * weight:.2f}")
+            print()
+    
+    # Example 3: Score distribution analysis
+    print("=" * 60)
+    print("Example 3: Score distribution analysis")
+    print("=" * 60)
+    
+    if traces_dir.exists():
+        all_results = evaluate_all_traces(traces_dir)
+        scores = [r['total_score'] for r in all_results]
+        
+        # Group by score ranges
+        score_ranges = {
+            "Negative (<0)": 0,
+            "Low (0-0.5)": 0,
+            "Medium (0.5-1.5)": 0,
+            "High (1.5-2.5)": 0,
+            "Very High (>2.5)": 0
+        }
+        
+        for score in scores:
+            if score < 0:
+                score_ranges["Negative (<0)"] += 1
+            elif score <= 0.5:
+                score_ranges["Low (0-0.5)"] += 1
+            elif score <= 1.5:
+                score_ranges["Medium (0.5-1.5)"] += 1
+            elif score <= 2.5:
+                score_ranges["High (1.5-2.5)"] += 1
+            else:
+                score_ranges["Very High (>2.5)"] += 1
+        
+        print(f"\nScore distribution across {len(scores)} traces:")
+        for range_name, count in score_ranges.items():
+            percentage = (count / len(scores) * 100) if scores else 0
+            bar = "█" * int(percentage / 2)  # Scale to 50 chars max
+            print(f"  {range_name:<20} {count:3d} ({percentage:5.1f}%) {bar}")
+        
+        # Additional statistics
+        if scores:
+            print(f"\nStatistics:")
+            print(f"  Mean score: {sum(scores) / len(scores):.2f}")
+            print(f"  Median score: {sorted(scores)[len(scores)//2]:.2f}")
+            print(f"  Std deviation: {(sum((x - sum(scores)/len(scores))**2 for x in scores) / len(scores))**0.5:.2f}")
+
+if __name__ == "__main__":
+    main()

synth_ai/environments/examples/crafter_classic/agent_demos/old/explore_saved_states.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Explore and visualize saved Crafter states
+==========================================
+"""
+
+import gzip
+import pickle
+import json
+from pathlib import Path
+from datetime import datetime
+import asyncio
+from uuid import uuid4
+
+from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
+from synth_ai.environments.examples.crafter_classic.taskset import CrafterTaskInstance, CrafterTaskInstanceMetadata
+from synth_ai.environments.tasks.core import Impetus, Intent
+
+STATES_DIR = Path("synth_ai/environments/examples/crafter_classic/env_states")
+
+
+def list_sessions():
+    """List all saved MCTS sessions."""
+    if not STATES_DIR.exists():
+        print("No env_states directory found!")
+        return []
+    
+    sessions = []
+    for session_dir in sorted(STATES_DIR.iterdir()):
+        if session_dir.is_dir() and session_dir.name.startswith("mcts_"):
+            metadata_file = session_dir / "session_metadata.json"
+            if metadata_file.exists():
+                metadata = json.loads(metadata_file.read_text())
+                sessions.append((session_dir, metadata))
+    
+    return sessions
+
+
+async def explore_state_file(state_file: Path):
+    """Load and display info about a single state file."""
+    print(f"\n📄 State file: {state_file.name}")
+    print(f"   Size: {state_file.stat().st_size / 1024:.1f} KB")
+    
+    # Load the state
+    try:
+        env_blob = gzip.decompress(state_file.read_bytes())
+        env_snapshot = pickle.loads(env_blob)
+        
+        # Create dummy task for deserialization
+        task = CrafterTaskInstance(
+            id=uuid4(),
+            impetus=Impetus(instructions="Analysis"),
+            intent=Intent(rubric={"goal": "Analysis"}, gold_trajectories=None, gold_state_diff={}),
+            metadata=CrafterTaskInstanceMetadata(
+                difficulty="easy", seed=0, num_trees_radius=0, 
+                num_cows_radius=0, num_hostiles_radius=0
+            ),
+            is_reproducible=True,
+            initial_engine_snapshot=None
+        )
+        
+        env = await CrafterClassicEnvironment._deserialize_engine(env_snapshot, task)
+        pub = env.engine._get_public_state_from_env()
+        priv = env.engine._get_private_state_from_env(0, False, False)
+        
+        print(f"   Position: {pub.player_position}")
+        print(f"   Health: {priv.player_internal_stats.get('health', 0)}")
+        print(f"   Steps: {pub.num_steps_taken}")
+        
+        # Show achievements
+        achievements = [k for k, v in pub.achievements_status.items() if v]
+        if achievements:
+            print(f"   Achievements ({len(achievements)}): {', '.join(achievements[:3])}{'...' if len(achievements) > 3 else ''}")
+        
+        # Show inventory
+        inventory = [(k, v) for k, v in pub.inventory.items() 
+                    if v > 0 and k not in ['health', 'food', 'drink', 'energy']]
+        if inventory:
+            print(f"   Inventory: {dict(inventory)}")
+            
+    except Exception as e:
+        print(f"   ❌ Error loading state: {e}")
+
+
+async def main():
+    print("🔍 Exploring Saved Crafter States")
+    print("=" * 60)
+    
+    sessions = list_sessions()
+    
+    if not sessions:
+        print("No saved sessions found!")
+        return
+    
+    print(f"Found {len(sessions)} saved sessions:\n")
+    
+    for i, (session_dir, metadata) in enumerate(sessions):
+        print(f"{i+1}. {session_dir.name}")
+        print(f"   Seed: {metadata['seed']}")
+        print(f"   Time: {metadata['timestamp']}")
+        print(f"   States: {len(list(session_dir.glob('*.snapshot.gz')))}")
+    
+    # Explore the most recent session
+    if sessions:
+        latest_session, latest_metadata = sessions[-1]
+        print(f"\n📂 Exploring latest session: {latest_session.name}")
+        print("-" * 60)
+        
+        # Get all state files
+        state_files = sorted(latest_session.glob("*.snapshot.gz"))
+        print(f"Total states saved: {len(state_files)}")
+        
+        # Show the root state
+        root_id = latest_metadata['root_node_id']
+        root_file = latest_session / f"{root_id}.snapshot.gz"
+        if root_file.exists():
+            print("\n🌱 Root state:")
+            await explore_state_file(root_file)
+        
+        # Show a few random states
+        import random
+        sample_size = min(3, len(state_files) - 1)
+        if sample_size > 0:
+            print("\n🎲 Random sample states:")
+            for state_file in random.sample(state_files[1:], sample_size):
+                await explore_state_file(state_file)
+        
+        # Show summary if it exists
+        summary_file = latest_session / "summary.txt"
+        if summary_file.exists():
+            print("\n📊 Session Summary:")
+            print("-" * 40)
+            print(summary_file.read_text())
+    
+    print("\n✅ Done exploring saved states!")
+    print(f"\n💡 Tip: States are saved in: {STATES_DIR}")
+    print("   Each .snapshot.gz file contains a complete game state")
+    print("   You can analyze these to understand MCTS exploration patterns")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/filter_traces_sft.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""
+DEPRECATED: This script reads individual JSON trace files which are no longer generated.
+Please use filter_traces_sft_duckdb.py instead, which reads from DuckDB.
+
+Original functionality:
+- Filter traces to create OpenAI SFT-ready .jsonl files
+- Supports trajectory-level and window-based filtering
+
+Migration guide:
+1. Run your agent with DuckDB enabled (this happens automatically now)
+2. Use filter_traces_sft_duckdb.py to filter and extract training data
+3. Example: python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -o training.jsonl
+
+This file is kept for reference only and will be removed in a future version.
+"""
+
+import sys
+print("=" * 80)
+print("DEPRECATED: This script is no longer supported.")
+print("Please use filter_traces_sft_duckdb.py instead.")
+print("=" * 80)
+print("\nExample usage:")
+print("  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -o training.jsonl")
+print("\nSee DUCKDB_FILTERING_GUIDE.md for more information.")
+sys.exit(1)