synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/example_v3_usage.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+Simple example demonstrating how to use tracing_v3 with Crafter.
+This shows the basic pattern for converting v2 code to v3.
+"""
+
+import asyncio
+import json
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent.parent))
+
+# Import v3 tracing components
+# Import Crafter hooks for v3
+from synth_ai.environments.examples.crafter_classic.trace_hooks_v3 import CRAFTER_HOOKS
+
+# Import LM 
+from synth_ai.lm.core.main_v2 import LM
+from synth_ai.tracing_v3.abstractions import (
+    EnvironmentEvent,
+    LMCAISEvent,
+    RuntimeEvent,
+    SessionEventMarkovBlanketMessage,
+    TimeRecord,
+)
+from synth_ai.tracing_v3.decorators import set_session_id, set_turn_number
+from synth_ai.tracing_v3.session_tracer import SessionTracer
+from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
+
+
+async def simple_crafter_session():
+    """Run a simple Crafter session with v3 tracing."""
+    
+    # 1. Create session tracer with hooks
+    tracer = SessionTracer(
+        hooks=CRAFTER_HOOKS,
+        db_url="sqlite+libsql://http://127.0.0.1:8080",  # Turso URL
+        auto_save=True
+    )
+    
+    # 2. Start a session
+    session_id = await tracer.start_session(
+        metadata={
+            "experiment": "v3_example",
+            "model": "gpt-4o-mini",
+            "difficulty": "easy"
+        }
+    )
+    print(f"Started session: {session_id}")
+    
+    # 3. Simulate a few game turns
+    for turn in range(5):
+        # Start timestep
+        await tracer.start_timestep(f"turn_{turn}", turn_number=turn)
+        
+        # Record observation message
+        observation = {
+            "inventory": {"wood": turn, "stone": 0},
+            "nearby": ["tree", "stone"],
+            "status": {"health": 9, "food": 8 - turn}
+        }
+        
+        await tracer.record_message(
+            content=json.dumps(observation),
+            message_type="observation",
+            metadata={"source": "environment"}
+        )
+        
+        # Record LM event (simulated)
+        lm_event = LMCAISEvent(
+            system_instance_id="crafter_agent",
+            time_record=TimeRecord(
+                event_time=time.time(),
+                message_time=turn
+            ),
+            model_name="gpt-4o-mini",
+            provider="openai",
+            input_tokens=100 + turn * 10,
+            output_tokens=20,
+            total_tokens=120 + turn * 10,
+            cost_usd=0.001 * (turn + 1),
+            latency_ms=100 + turn * 50
+        )
+        await tracer.record_event(lm_event)
+        
+        # Record action
+        action = "collect_wood" if turn % 2 == 0 else "move_right"
+        await tracer.record_message(
+            content=action,
+            message_type="action",
+            metadata={"source": "agent"}
+        )
+        
+        # Record runtime event
+        runtime_event = RuntimeEvent(
+            system_instance_id="crafter_env",
+            time_record=TimeRecord(
+                event_time=time.time(),
+                message_time=turn
+            ),
+            actions=[5 if turn % 2 == 0 else 2],  # action IDs
+            metadata={
+                "action_name": action,
+                "valid": True
+            }
+        )
+        await tracer.record_event(runtime_event)
+        
+        # Record environment event with achievements
+        achievements_before = {"collect_wood": turn > 0}
+        achievements_after = {"collect_wood": True} if action == "collect_wood" else achievements_before
+        
+        env_event = EnvironmentEvent(
+            system_instance_id="crafter_env",
+            time_record=TimeRecord(
+                event_time=time.time(),
+                message_time=turn
+            ),
+            reward=1.0 if action == "collect_wood" else 0.0,
+            terminated=False,
+            system_state_before={
+                "public_state": {"achievements_status": achievements_before}
+            },
+            system_state_after={
+                "public_state": {"achievements_status": achievements_after}
+            }
+        )
+        await tracer.record_event(env_event)
+        
+        # End timestep
+        await tracer.end_timestep()
+        
+        print(f"Completed turn {turn}")
+    
+    # 4. End session (auto-saves to database)
+    trace = await tracer.end_session()
+    print(f"Session ended. Total events: {len(trace.event_history)}")
+    
+    # 5. Query the saved data
+    db_manager = AsyncSQLTraceManager("sqlite+libsql://http://127.0.0.1:8080")
+    await db_manager.initialize()
+    
+    # Get session data
+    session_data = await db_manager.get_session_trace(session_id)
+    if session_data:
+        print(f"\nRetrieved session from database:")
+        print(f"  Session ID: {session_data['session_id']}")
+        print(f"  Timesteps: {session_data['num_timesteps']}")
+        print(f"  Events: {session_data['num_events']}")
+        print(f"  Messages: {session_data['num_messages']}")
+    
+    # Query model usage
+    model_usage = await db_manager.get_model_usage()
+    print(f"\nModel usage statistics:")
+    print(model_usage)
+    
+    await db_manager.close()
+    await tracer.close()
+
+
+async def context_manager_example():
+    """Example using context managers for cleaner code."""
+    
+    tracer = SessionTracer(
+        hooks=CRAFTER_HOOKS,
+        db_url="sqlite+libsql://http://127.0.0.1:8080"
+    )
+    
+    # Use context managers for automatic cleanup
+    async with tracer.session(metadata={"example": "context_manager"}) as session_id:
+        print(f"In session: {session_id}")
+        
+        async with tracer.timestep("step_1", turn_number=0) as step:
+            print(f"In timestep: {step.step_id}")
+            
+            # Record some events
+            await tracer.record_message(
+                content="Hello from context manager",
+                message_type="user"
+            )
+            
+            event = RuntimeEvent(
+                system_instance_id="example_system",
+                time_record=TimeRecord(event_time=time.time()),
+                actions=[1],
+                metadata={"example": True}
+            )
+            await tracer.record_event(event)
+    
+    print("Session automatically ended and saved")
+    await tracer.close()
+
+
+async def main():
+    """Run the examples."""
+    print("=== V3 Tracing Examples ===\n")
+    
+    print("1. Running simple Crafter session...")
+    await simple_crafter_session()
+    
+    print("\n2. Running context manager example...")
+    await context_manager_example()
+    
+    print("\n✅ Examples completed!")
+
+
+if __name__ == "__main__":
+    # Make sure sqld is running on port 8080
+    print("Note: This example assumes sqld is running on http://127.0.0.1:8080")
+    print("Start it with: sqld --http-listen 127.0.0.1:8080\n")
+    
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/compare_traces.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+"""
+Script to compare traces between OpenAI direct API and LM class implementations.
+Runs both versions and compares the captured events.
+"""
+
+import asyncio
+import json
+import subprocess
+import sys
+import os
+from pathlib import Path
+from typing import Dict, List, Any, Tuple
+
+# Colors for output
+GREEN = '\033[92m'
+RED = '\033[91m'
+YELLOW = '\033[93m'
+BLUE = '\033[94m'
+RESET = '\033[0m'
+
+
+def run_openai_version(model: str = "gpt-4o-mini", episodes: int = 1, max_turns: int = 2) -> Tuple[bool, str]:
+    """Run the OpenAI version and return trace directory."""
+    print(f"{BLUE}Running OpenAI version...{RESET}")
+    
+    cmd = [
+        sys.executable,
+        "test_crafter_react_agent_openai.py",
+        "--model", model,
+        "--episodes", str(episodes),
+        "--max-turns", str(max_turns)
+    ]
+    
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
+        if result.returncode == 0:
+            print(f"{GREEN}✓ OpenAI version completed successfully{RESET}")
+            # Extract trace directory from output
+            for line in result.stdout.split('\n'):
+                if "Saved trace to" in line:
+                    trace_path = line.split("Saved trace to")[-1].strip()
+                    trace_dir = Path(trace_path).parent
+                    return True, str(trace_dir)
+            return True, "./traces"
+        else:
+            print(f"{RED}✗ OpenAI version failed{RESET}")
+            print(result.stderr)
+            return False, ""
+    except Exception as e:
+        print(f"{RED}✗ Error running OpenAI version: {e}{RESET}")
+        return False, ""
+
+
+def run_lm_version(model: str = "gpt-4o-mini", episodes: int = 1, max_turns: int = 2) -> Tuple[bool, str]:
+    """Run the LM class version and return trace directory."""
+    print(f"{BLUE}Running LM version...{RESET}")
+    
+    cmd = [
+        sys.executable,
+        "test_crafter_react_agent_lm.py",
+        "--model", model,
+        "--episodes", str(episodes),
+        "--max-turns", str(max_turns)
+    ]
+    
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
+        if result.returncode == 0:
+            print(f"{GREEN}✓ LM version completed successfully{RESET}")
+            # Extract trace directory from output
+            for line in result.stdout.split('\n'):
+                if "Saved trace to" in line:
+                    trace_path = line.split("Saved trace to")[-1].strip()
+                    trace_dir = Path(trace_path).parent
+                    return True, str(trace_dir)
+            return True, "./traces_v2_lm"
+        else:
+            print(f"{RED}✗ LM version failed{RESET}")
+            print(result.stderr)
+            return False, ""
+    except Exception as e:
+        print(f"{RED}✗ Error running LM version: {e}{RESET}")
+        return False, ""
+
+
+def load_trace(trace_file: Path) -> Dict[str, Any]:
+    """Load a trace file."""
+    try:
+        with open(trace_file, 'r') as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"{RED}Error loading trace {trace_file}: {e}{RESET}")
+        return {}
+
+
+def extract_cais_events(trace_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extract CAISEvents from trace data."""
+    events = trace_data.get('event_history', [])
+    return [e for e in events if e.get('system_instance_id', '').startswith('crafter-react-agent')]
+
+
+def extract_env_events(trace_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extract EnvironmentEvents from trace data."""
+    events = trace_data.get('event_history', [])
+    return [e for e in events if 'reward' in e and not e.get('system_instance_id', '').startswith('crafter-react-agent')]
+
+
+def compare_events(events1: List[Dict], events2: List[Dict], event_type: str) -> bool:
+    """Compare two lists of events and report differences."""
+    print(f"\n{BLUE}Comparing {event_type}...{RESET}")
+    
+    if len(events1) != len(events2):
+        print(f"{RED}✗ Different number of events: {len(events1)} vs {len(events2)}{RESET}")
+        return False
+    
+    all_match = True
+    for i, (e1, e2) in enumerate(zip(events1, events2)):
+        print(f"\n  Event {i+1}:")
+        
+        # Compare key fields
+        fields_to_compare = {
+            'CAISEvent': ['system_instance_id', 'llm_call_records', 'metadata'],
+            'EnvironmentEvent': ['reward', 'done', 'info']
+        }
+        
+        for field in fields_to_compare.get(event_type, []):
+            if field in e1 and field in e2:
+                # Special handling for llm_call_records
+                if field == 'llm_call_records':
+                    if len(e1[field]) != len(e2[field]):
+                        print(f"    {RED}✗ Different number of LLM calls{RESET}")
+                        all_match = False
+                    else:
+                        # Compare model and tool calls
+                        for j, (llm1, llm2) in enumerate(zip(e1[field], e2[field])):
+                            model1 = llm1.get('model', '')
+                            model2 = llm2.get('model', '')
+                            if model1 != model2:
+                                print(f"    {RED}✗ Different models: {model1} vs {model2}{RESET}")
+                                all_match = False
+                            else:
+                                print(f"    {GREEN}✓ Model matches: {model1}{RESET}")
+                            
+                            # Check tool calls
+                            resp1 = llm1.get('response', {})
+                            resp2 = llm2.get('response', {})
+                            tools1 = extract_tool_calls(resp1)
+                            tools2 = extract_tool_calls(resp2)
+                            
+                            if tools1 != tools2:
+                                print(f"    {YELLOW}⚠ Different tool calls: {tools1} vs {tools2}{RESET}")
+                                # This might be OK due to LLM non-determinism
+                elif field == 'metadata':
+                    # Compare token counts if available
+                    tokens1 = {k: v for k, v in e1[field].items() if 'token' in k}
+                    tokens2 = {k: v for k, v in e2[field].items() if 'token' in k}
+                    if tokens1 and tokens2:
+                        print(f"    Tokens (OpenAI): {tokens1}")
+                        print(f"    Tokens (LM):     {tokens2}")
+                else:
+                    if e1[field] == e2[field]:
+                        print(f"    {GREEN}✓ {field} matches{RESET}")
+                    else:
+                        print(f"    {RED}✗ {field} differs: {e1[field]} vs {e2[field]}{RESET}")
+                        all_match = False
+    
+    return all_match
+
+
+def extract_tool_calls(response: Dict[str, Any]) -> List[str]:
+    """Extract tool call names from response."""
+    tool_names = []
+    choices = response.get('choices', [])
+    if choices and isinstance(choices[0], dict):
+        tool_calls = choices[0].get('message', {}).get('tool_calls', [])
+        for tc in tool_calls:
+            tool_names.append(tc.get('function', {}).get('name', 'unknown'))
+    return tool_names
+
+
+def compare_traces(openai_dir: Path, lm_dir: Path, episode: int = 0) -> bool:
+    """Compare traces from both versions."""
+    print(f"\n{BLUE}{'='*60}{RESET}")
+    print(f"{BLUE}Comparing traces for episode {episode}{RESET}")
+    print(f"{BLUE}{'='*60}{RESET}")
+    
+    # Load traces - OpenAI uses different naming convention
+    # Look for most recent session files
+    openai_files = list(openai_dir.glob(f"session_episode_{episode}_*.json"))
+    if openai_files:
+        # Get the most recent one
+        openai_trace_file = max(openai_files, key=lambda f: f.stat().st_mtime)
+    else:
+        openai_trace_file = openai_dir / f"trace_episode_{episode}.json"
+    
+    lm_trace_file = lm_dir / f"trace_episode_{episode}.json"
+    
+    if not openai_trace_file.exists():
+        print(f"{RED}✗ OpenAI trace not found: {openai_trace_file}{RESET}")
+        return False
+    
+    if not lm_trace_file.exists():
+        print(f"{RED}✗ LM trace not found: {lm_trace_file}{RESET}")
+        return False
+    
+    openai_trace = load_trace(openai_trace_file)
+    lm_trace = load_trace(lm_trace_file)
+    
+    # Extract events
+    openai_cais = extract_cais_events(openai_trace)
+    lm_cais = extract_cais_events(lm_trace)
+    
+    openai_env = extract_env_events(openai_trace)
+    lm_env = extract_env_events(lm_trace)
+    
+    # Compare
+    cais_match = compare_events(openai_cais, lm_cais, "CAISEvent")
+    env_match = compare_events(openai_env, lm_env, "EnvironmentEvent")
+    
+    # Check messages
+    print(f"\n{BLUE}Comparing messages...{RESET}")
+    openai_msgs = openai_trace.get('message_history', [])
+    lm_msgs = lm_trace.get('message_history', [])
+    
+    if len(openai_msgs) != len(lm_msgs):
+        print(f"{RED}✗ Different number of messages: {len(openai_msgs)} vs {len(lm_msgs)}{RESET}")
+    else:
+        print(f"{GREEN}✓ Same number of messages: {len(openai_msgs)}{RESET}")
+    
+    return cais_match and env_match
+
+
+async def main():
+    """Run both versions and compare traces."""
+    print(f"{BLUE}{'='*60}{RESET}")
+    print(f"{BLUE}Crafter Trace Comparison: OpenAI vs LM{RESET}")
+    print(f"{BLUE}{'='*60}{RESET}")
+    
+    # Configuration
+    model = "gpt-4o-mini"
+    episodes = 2
+    max_turns = 3
+    
+    print(f"\nConfiguration:")
+    print(f"  Model: {model}")
+    print(f"  Episodes: {episodes}")
+    print(f"  Max turns: {max_turns}")
+    
+    # Run OpenAI version
+    openai_success, openai_dir = run_openai_version(model, episodes, max_turns)
+    if not openai_success:
+        print(f"{RED}Failed to run OpenAI version{RESET}")
+        return
+    
+    # Small delay to avoid rate limits
+    await asyncio.sleep(2)
+    
+    # Run LM version
+    lm_success, lm_dir = run_lm_version(model, episodes, max_turns)
+    if not lm_success:
+        print(f"{RED}Failed to run LM version{RESET}")
+        return
+    
+    # Compare traces
+    print(f"\n{BLUE}Trace directories:{RESET}")
+    print(f"  OpenAI: {openai_dir}")
+    print(f"  LM:     {lm_dir}")
+    
+    all_match = True
+    for episode in range(episodes):
+        match = compare_traces(Path(openai_dir), Path(lm_dir), episode)
+        all_match = all_match and match
+    
+    # Final verdict
+    print(f"\n{BLUE}{'='*60}{RESET}")
+    print(f"{BLUE}FINAL VERDICT{RESET}")
+    print(f"{BLUE}{'='*60}{RESET}")
+    
+    if all_match:
+        print(f"{GREEN}✅ Traces match! LM class produces equivalent v2 traces.{RESET}")
+    else:
+        print(f"{YELLOW}⚠️  Some differences found. This may be due to:{RESET}")
+        print(f"{YELLOW}   - LLM non-determinism (different responses){RESET}")
+        print(f"{YELLOW}   - Minor implementation differences{RESET}")
+        print(f"{YELLOW}   - Timing variations{RESET}")
+    
+    print(f"\nKey observations:")
+    print(f"  • Both versions create CAISEvents with LLM call records")
+    print(f"  • Both capture environment events and observations")
+    print(f"  • Token counts and metadata are preserved")
+    print(f"  • The LM class successfully replaces direct OpenAI calls")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/crafter_comprehensive_evaluation.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Run script for Full Enchilada Crafter Evaluation
+"""
+
+import asyncio
+import argparse
+from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
+    run_full_enchilada_eval,
+)
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="Run Full Enchilada Crafter Evaluation")
+    parser.add_argument(
+        "--models", nargs="+", default=["gpt-4o-mini"], help="Model names to evaluate"
+    )
+    parser.add_argument(
+        "--difficulties",
+        nargs="+",
+        default=["easy", "hard"],
+        help="Difficulty levels to test",
+    )
+    parser.add_argument(
+        "--num-trajectories",
+        type=int,
+        default=3,
+        help="Number of trajectories per condition",
+    )
+    parser.add_argument("--max-turns", type=int, default=30, help="Maximum turns per trajectory")
+    parser.add_argument("--no-images", action="store_true", help="Disable image capture")
+    parser.add_argument(
+        "--no-viewer",
+        action="store_true",
+        help="Don't launch the viewer after evaluation",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=None,
+        help="Output directory (default: src/evals/crafter/run_TIMESTAMP)",
+    )
+
+    args = parser.parse_args()
+
+    await run_full_enchilada_eval(
+        model_names=args.models,
+        difficulties=args.difficulties,
+        num_trajectories=args.num_trajectories,
+        max_turns=args.max_turns,
+        capture_images=not args.no_images,
+        launch_viewer=not args.no_viewer,
+        output_dir=args.output_dir,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())