synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.5__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/run_and_eval.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Run Crafter agent evaluation and automatically evaluate traces.
+"""
+
+import subprocess
+import sys
+import time
+from pathlib import Path
+from trace_eval import evaluate_all_traces, print_evaluation_summary, print_trace_evaluation
+
+def main():
+    # Run the agent evaluation
+    print("🎮 Running Crafter Agent Evaluation...")
+    print("=" * 60)
+    
+    # Pass all arguments to the test script
+    cmd = [sys.executable, "test_crafter_react_agent_openai.py"] + sys.argv[1:]
+    
+    # Record start time
+    start_time = time.time()
+    
+    # Run the evaluation
+    result = subprocess.run(cmd)
+    
+    if result.returncode != 0:
+        print(f"\n❌ Agent evaluation failed with return code {result.returncode}")
+        return
+    
+    # Wait a moment for files to be written
+    time.sleep(1)
+    
+    # Find recent trace files
+    print("\n" + "=" * 80)
+    print("📊 TRACE EVALUATION")
+    print("=" * 80)
+    
+    trace_dir = Path("traces")
+    if not trace_dir.exists():
+        print("❌ No traces directory found")
+        return
+    
+    # Find traces created since we started
+    recent_traces = []
+    for trace_file in trace_dir.glob("*.json"):
+        if trace_file.stat().st_mtime >= start_time:
+            recent_traces.append(trace_file)
+    
+    if not recent_traces:
+        print("❌ No new trace files found")
+        return
+    
+    print(f"Found {len(recent_traces)} new trace files")
+    
+    # Evaluate all recent traces
+    results = []
+    for trace_file in recent_traces:
+        from trace_eval import evaluate_trace
+        result = evaluate_trace(trace_file)
+        results.append(result)
+    
+    # Sort by score
+    results.sort(key=lambda x: x['total_score'], reverse=True)
+    
+    # Show individual evaluations if not too many
+    if len(results) <= 5:
+        for result in results:
+            print_trace_evaluation(result)
+    
+    # Always show summary
+    print_evaluation_summary(results)
+    
+    # Show achievement distribution
+    print("\n" + "=" * 80)
+    print("📊 ACHIEVEMENT DISTRIBUTION")
+    print("=" * 80)
+    
+    total_easy = sum(r['counts'].get('easy_achievement', 0) for r in results)
+    total_medium = sum(r['counts'].get('medium_achievement', 0) for r in results)
+    total_hard = sum(r['counts'].get('hard_achievement', 0) for r in results)
+    total_invalid = sum(r['counts'].get('invalid_action', 0) for r in results)
+    
+    print(f"Easy achievements:    {total_easy} total ({total_easy/len(results):.1f} per episode)")
+    print(f"Medium achievements:  {total_medium} total ({total_medium/len(results):.1f} per episode)")
+    print(f"Hard achievements:    {total_hard} total ({total_hard/len(results):.1f} per episode)")
+    print(f"Invalid actions:      {total_invalid} total ({total_invalid/len(results):.1f} per episode)")
+    
+    # Score interpretation
+    avg_score = sum(r['total_score'] for r in results) / len(results)
+    print(f"\nAverage Score: {avg_score:.2f}")
+    
+    if avg_score >= 2.0:
+        print("🎉 Excellent performance!")
+    elif avg_score >= 1.0:
+        print("✅ Good performance")
+    elif avg_score >= 0.0:
+        print("📈 Room for improvement")
+    else:
+        print("⚠️  Many invalid actions detected")
+
+if __name__ == "__main__":
+    main()

synth_ai/environments/examples/crafter_classic/agent_demos/old/run_comparison.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""
+Run comparison between OpenAI and LM implementations to verify trace equivalence.
+"""
+
+import asyncio
+import subprocess
+import json
+from pathlib import Path
+import sys
+
+async def run_tests():
+    """Run both tests and compare results."""
+    print("🚀 Running Crafter v2 tracing comparison test")
+    print("=" * 80)
+    
+    # Test parameters
+    model = "gpt-4o-mini"
+    episodes = 2
+    max_turns = 5
+    
+    # Run OpenAI implementation
+    print("\n📍 Running OpenAI implementation...")
+    openai_cmd = [
+        sys.executable,
+        "synth_ai/environments/examples/crafter_classic/agent_demos/test_crafter_react_agent_openai.py",
+        "--episodes", str(episodes),
+        "--model", model,
+        "--max-turns", str(max_turns)
+    ]
+    openai_result = subprocess.run(openai_cmd, capture_output=True, text=True)
+    
+    if openai_result.returncode != 0:
+        print(f"❌ OpenAI test failed: {openai_result.stderr}")
+        return
+    
+    print("✅ OpenAI test completed")
+    
+    # Run LM implementation
+    print("\n📍 Running LM implementation...")
+    lm_cmd = [
+        sys.executable,
+        "synth_ai/environments/examples/crafter_classic/agent_demos/test_crafter_react_agent_lm.py",
+        "--episodes", str(episodes),
+        "--model", model,
+        "--max-turns", str(max_turns)
+    ]
+    lm_result = subprocess.run(lm_cmd, capture_output=True, text=True)
+    
+    if lm_result.returncode != 0:
+        print(f"❌ LM test failed: {lm_result.stderr}")
+        return
+    
+    print("✅ LM test completed")
+    
+    # Compare results
+    print("\n📊 Comparing results...")
+    
+    # Load OpenAI results
+    openai_results_path = Path("traces/results.json")
+    if openai_results_path.exists():
+        with open(openai_results_path) as f:
+            openai_results = json.load(f)
+        print(f"\nOpenAI Results:")
+        print(f"  Episodes: {openai_results['summary']['successful']}/{episodes}")
+        print(f"  Avg Reward: {openai_results['summary']['avg_reward']:.2f}")
+        print(f"  Avg Steps: {openai_results['summary']['avg_steps']:.1f}")
+    else:
+        print("❌ OpenAI results not found")
+        
+    # Load LM results
+    lm_results_path = Path("traces_v2_lm/results.json")
+    if lm_results_path.exists():
+        with open(lm_results_path) as f:
+            lm_results = json.load(f)
+        print(f"\nLM Results:")
+        print(f"  Episodes: {lm_results['summary']['successful']}/{episodes}")
+        print(f"  Avg Reward: {lm_results['summary']['avg_reward']:.2f}")
+        print(f"  Avg Steps: {lm_results['summary']['avg_steps']:.1f}")
+    else:
+        print("❌ LM results not found")
+    
+    # Compare trace structures
+    print("\n🔍 Comparing trace structures...")
+    
+    openai_trace = Path("traces/trace_episode_0.json")
+    lm_trace = Path("traces_v2_lm/trace_episode_0.json")
+    
+    if openai_trace.exists() and lm_trace.exists():
+        with open(openai_trace) as f:
+            openai_data = json.load(f)
+        with open(lm_trace) as f:
+            lm_data = json.load(f)
+            
+        # Check key structures
+        print(f"\nOpenAI trace:")
+        print(f"  Messages: {len(openai_data.get('message_history', []))}")
+        print(f"  Events: {len(openai_data.get('event_history', []))}")
+        print(f"  Timesteps: {len(openai_data.get('session_time_steps', []))}")
+        
+        print(f"\nLM trace:")
+        print(f"  Messages: {len(lm_data.get('message_history', []))}")
+        print(f"  Events: {len(lm_data.get('event_history', []))}")
+        print(f"  Timesteps: {len(lm_data.get('session_time_steps', []))}")
+        
+        # Check for AI events
+        openai_ai_events = [e for e in openai_data.get('event_history', []) 
+                           if 'gen_ai.request.model' in e.get('system_state_before', {})]
+        lm_ai_events = [e for e in lm_data.get('event_history', [])
+                       if 'gen_ai.request.model' in e.get('system_state_before', {})]
+        
+        print(f"\nAI Events:")
+        print(f"  OpenAI: {len(openai_ai_events)}")
+        print(f"  LM: {len(lm_ai_events)}")
+        
+        if len(openai_ai_events) == len(lm_ai_events):
+            print("✅ Same number of AI events captured")
+        else:
+            print("⚠️  Different number of AI events")
+            
+    else:
+        print("❌ Trace files not found")
+    
+    print("\n" + "=" * 80)
+    print("✅ Comparison complete!")
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())