synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/example_v3_usage.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple example demonstrating how to use tracing_v3 with Crafter.
-This shows the basic pattern for converting v2 code to v3.
-"""
-
-import asyncio
-import json
-import sys
-import time
-from datetime import datetime
-from pathlib import Path
-
-# Add parent directory to path
-sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent.parent))
-
-# Import v3 tracing components
-# Import Crafter hooks for v3
-from synth_ai.environments.examples.crafter_classic.trace_hooks_v3 import CRAFTER_HOOKS
-
-# Import LM 
-from synth_ai.lm.core.main_v2 import LM
-from synth_ai.tracing_v3.abstractions import (
-    EnvironmentEvent,
-    LMCAISEvent,
-    RuntimeEvent,
-    SessionEventMarkovBlanketMessage,
-    TimeRecord,
-)
-from synth_ai.tracing_v3.decorators import set_session_id, set_turn_number
-from synth_ai.tracing_v3.session_tracer import SessionTracer
-from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
-
-
-async def simple_crafter_session():
-    """Run a simple Crafter session with v3 tracing."""
-    
-    # 1. Create session tracer with hooks
-    tracer = SessionTracer(
-        hooks=CRAFTER_HOOKS,
-        db_url="sqlite+libsql://http://127.0.0.1:8080",  # Turso URL
-        auto_save=True
-    )
-    
-    # 2. Start a session
-    session_id = await tracer.start_session(
-        metadata={
-            "experiment": "v3_example",
-            "model": "gpt-4o-mini",
-            "difficulty": "easy"
-        }
-    )
-    print(f"Started session: {session_id}")
-    
-    # 3. Simulate a few game turns
-    for turn in range(5):
-        # Start timestep
-        await tracer.start_timestep(f"turn_{turn}", turn_number=turn)
-        
-        # Record observation message
-        observation = {
-            "inventory": {"wood": turn, "stone": 0},
-            "nearby": ["tree", "stone"],
-            "status": {"health": 9, "food": 8 - turn}
-        }
-        
-        await tracer.record_message(
-            content=json.dumps(observation),
-            message_type="observation",
-            metadata={"source": "environment"}
-        )
-        
-        # Record LM event (simulated)
-        lm_event = LMCAISEvent(
-            system_instance_id="crafter_agent",
-            time_record=TimeRecord(
-                event_time=time.time(),
-                message_time=turn
-            ),
-            model_name="gpt-4o-mini",
-            provider="openai",
-            input_tokens=100 + turn * 10,
-            output_tokens=20,
-            total_tokens=120 + turn * 10,
-            cost_usd=0.001 * (turn + 1),
-            latency_ms=100 + turn * 50
-        )
-        await tracer.record_event(lm_event)
-        
-        # Record action
-        action = "collect_wood" if turn % 2 == 0 else "move_right"
-        await tracer.record_message(
-            content=action,
-            message_type="action",
-            metadata={"source": "agent"}
-        )
-        
-        # Record runtime event
-        runtime_event = RuntimeEvent(
-            system_instance_id="crafter_env",
-            time_record=TimeRecord(
-                event_time=time.time(),
-                message_time=turn
-            ),
-            actions=[5 if turn % 2 == 0 else 2],  # action IDs
-            metadata={
-                "action_name": action,
-                "valid": True
-            }
-        )
-        await tracer.record_event(runtime_event)
-        
-        # Record environment event with achievements
-        achievements_before = {"collect_wood": turn > 0}
-        achievements_after = {"collect_wood": True} if action == "collect_wood" else achievements_before
-        
-        env_event = EnvironmentEvent(
-            system_instance_id="crafter_env",
-            time_record=TimeRecord(
-                event_time=time.time(),
-                message_time=turn
-            ),
-            reward=1.0 if action == "collect_wood" else 0.0,
-            terminated=False,
-            system_state_before={
-                "public_state": {"achievements_status": achievements_before}
-            },
-            system_state_after={
-                "public_state": {"achievements_status": achievements_after}
-            }
-        )
-        await tracer.record_event(env_event)
-        
-        # End timestep
-        await tracer.end_timestep()
-        
-        print(f"Completed turn {turn}")
-    
-    # 4. End session (auto-saves to database)
-    trace = await tracer.end_session()
-    print(f"Session ended. Total events: {len(trace.event_history)}")
-    
-    # 5. Query the saved data
-    db_manager = AsyncSQLTraceManager("sqlite+libsql://http://127.0.0.1:8080")
-    await db_manager.initialize()
-    
-    # Get session data
-    session_data = await db_manager.get_session_trace(session_id)
-    if session_data:
-        print(f"\nRetrieved session from database:")
-        print(f"  Session ID: {session_data['session_id']}")
-        print(f"  Timesteps: {session_data['num_timesteps']}")
-        print(f"  Events: {session_data['num_events']}")
-        print(f"  Messages: {session_data['num_messages']}")
-    
-    # Query model usage
-    model_usage = await db_manager.get_model_usage()
-    print(f"\nModel usage statistics:")
-    print(model_usage)
-    
-    await db_manager.close()
-    await tracer.close()
-
-
-async def context_manager_example():
-    """Example using context managers for cleaner code."""
-    
-    tracer = SessionTracer(
-        hooks=CRAFTER_HOOKS,
-        db_url="sqlite+libsql://http://127.0.0.1:8080"
-    )
-    
-    # Use context managers for automatic cleanup
-    async with tracer.session(metadata={"example": "context_manager"}) as session_id:
-        print(f"In session: {session_id}")
-        
-        async with tracer.timestep("step_1", turn_number=0) as step:
-            print(f"In timestep: {step.step_id}")
-            
-            # Record some events
-            await tracer.record_message(
-                content="Hello from context manager",
-                message_type="user"
-            )
-            
-            event = RuntimeEvent(
-                system_instance_id="example_system",
-                time_record=TimeRecord(event_time=time.time()),
-                actions=[1],
-                metadata={"example": True}
-            )
-            await tracer.record_event(event)
-    
-    print("Session automatically ended and saved")
-    await tracer.close()
-
-
-async def main():
-    """Run the examples."""
-    print("=== V3 Tracing Examples ===\n")
-    
-    print("1. Running simple Crafter session...")
-    await simple_crafter_session()
-    
-    print("\n2. Running context manager example...")
-    await context_manager_example()
-    
-    print("\n✅ Examples completed!")
-
-
-if __name__ == "__main__":
-    # Make sure sqld is running on port 8080
-    print("Note: This example assumes sqld is running on http://127.0.0.1:8080")
-    print("Start it with: sqld --http-listen 127.0.0.1:8080\n")
-    
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/compare_traces.py

@@ -1,296 +0,0 @@
-#!/usr/bin/env python3
-"""
-Script to compare traces between OpenAI direct API and LM class implementations.
-Runs both versions and compares the captured events.
-"""
-
-import asyncio
-import json
-import subprocess
-import sys
-import os
-from pathlib import Path
-from typing import Dict, List, Any, Tuple
-
-# Colors for output
-GREEN = '\033[92m'
-RED = '\033[91m'
-YELLOW = '\033[93m'
-BLUE = '\033[94m'
-RESET = '\033[0m'
-
-
-def run_openai_version(model: str = "gpt-4o-mini", episodes: int = 1, max_turns: int = 2) -> Tuple[bool, str]:
-    """Run the OpenAI version and return trace directory."""
-    print(f"{BLUE}Running OpenAI version...{RESET}")
-    
-    cmd = [
-        sys.executable,
-        "test_crafter_react_agent_openai.py",
-        "--model", model,
-        "--episodes", str(episodes),
-        "--max-turns", str(max_turns)
-    ]
-    
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
-        if result.returncode == 0:
-            print(f"{GREEN}✓ OpenAI version completed successfully{RESET}")
-            # Extract trace directory from output
-            for line in result.stdout.split('\n'):
-                if "Saved trace to" in line:
-                    trace_path = line.split("Saved trace to")[-1].strip()
-                    trace_dir = Path(trace_path).parent
-                    return True, str(trace_dir)
-            return True, "./traces"
-        else:
-            print(f"{RED}✗ OpenAI version failed{RESET}")
-            print(result.stderr)
-            return False, ""
-    except Exception as e:
-        print(f"{RED}✗ Error running OpenAI version: {e}{RESET}")
-        return False, ""
-
-
-def run_lm_version(model: str = "gpt-4o-mini", episodes: int = 1, max_turns: int = 2) -> Tuple[bool, str]:
-    """Run the LM class version and return trace directory."""
-    print(f"{BLUE}Running LM version...{RESET}")
-    
-    cmd = [
-        sys.executable,
-        "test_crafter_react_agent_lm.py",
-        "--model", model,
-        "--episodes", str(episodes),
-        "--max-turns", str(max_turns)
-    ]
-    
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, cwd=Path(__file__).parent)
-        if result.returncode == 0:
-            print(f"{GREEN}✓ LM version completed successfully{RESET}")
-            # Extract trace directory from output
-            for line in result.stdout.split('\n'):
-                if "Saved trace to" in line:
-                    trace_path = line.split("Saved trace to")[-1].strip()
-                    trace_dir = Path(trace_path).parent
-                    return True, str(trace_dir)
-            return True, "./traces_v2_lm"
-        else:
-            print(f"{RED}✗ LM version failed{RESET}")
-            print(result.stderr)
-            return False, ""
-    except Exception as e:
-        print(f"{RED}✗ Error running LM version: {e}{RESET}")
-        return False, ""
-
-
-def load_trace(trace_file: Path) -> Dict[str, Any]:
-    """Load a trace file."""
-    try:
-        with open(trace_file, 'r') as f:
-            return json.load(f)
-    except Exception as e:
-        print(f"{RED}Error loading trace {trace_file}: {e}{RESET}")
-        return {}
-
-
-def extract_cais_events(trace_data: Dict[str, Any]) -> List[Dict[str, Any]]:
-    """Extract CAISEvents from trace data."""
-    events = trace_data.get('event_history', [])
-    return [e for e in events if e.get('system_instance_id', '').startswith('crafter-react-agent')]
-
-
-def extract_env_events(trace_data: Dict[str, Any]) -> List[Dict[str, Any]]:
-    """Extract EnvironmentEvents from trace data."""
-    events = trace_data.get('event_history', [])
-    return [e for e in events if 'reward' in e and not e.get('system_instance_id', '').startswith('crafter-react-agent')]
-
-
-def compare_events(events1: List[Dict], events2: List[Dict], event_type: str) -> bool:
-    """Compare two lists of events and report differences."""
-    print(f"\n{BLUE}Comparing {event_type}...{RESET}")
-    
-    if len(events1) != len(events2):
-        print(f"{RED}✗ Different number of events: {len(events1)} vs {len(events2)}{RESET}")
-        return False
-    
-    all_match = True
-    for i, (e1, e2) in enumerate(zip(events1, events2)):
-        print(f"\n  Event {i+1}:")
-        
-        # Compare key fields
-        fields_to_compare = {
-            'CAISEvent': ['system_instance_id', 'llm_call_records', 'metadata'],
-            'EnvironmentEvent': ['reward', 'done', 'info']
-        }
-        
-        for field in fields_to_compare.get(event_type, []):
-            if field in e1 and field in e2:
-                # Special handling for llm_call_records
-                if field == 'llm_call_records':
-                    if len(e1[field]) != len(e2[field]):
-                        print(f"    {RED}✗ Different number of LLM calls{RESET}")
-                        all_match = False
-                    else:
-                        # Compare model and tool calls
-                        for j, (llm1, llm2) in enumerate(zip(e1[field], e2[field])):
-                            model1 = llm1.get('model', '')
-                            model2 = llm2.get('model', '')
-                            if model1 != model2:
-                                print(f"    {RED}✗ Different models: {model1} vs {model2}{RESET}")
-                                all_match = False
-                            else:
-                                print(f"    {GREEN}✓ Model matches: {model1}{RESET}")
-                            
-                            # Check tool calls
-                            resp1 = llm1.get('response', {})
-                            resp2 = llm2.get('response', {})
-                            tools1 = extract_tool_calls(resp1)
-                            tools2 = extract_tool_calls(resp2)
-                            
-                            if tools1 != tools2:
-                                print(f"    {YELLOW}⚠ Different tool calls: {tools1} vs {tools2}{RESET}")
-                                # This might be OK due to LLM non-determinism
-                elif field == 'metadata':
-                    # Compare token counts if available
-                    tokens1 = {k: v for k, v in e1[field].items() if 'token' in k}
-                    tokens2 = {k: v for k, v in e2[field].items() if 'token' in k}
-                    if tokens1 and tokens2:
-                        print(f"    Tokens (OpenAI): {tokens1}")
-                        print(f"    Tokens (LM):     {tokens2}")
-                else:
-                    if e1[field] == e2[field]:
-                        print(f"    {GREEN}✓ {field} matches{RESET}")
-                    else:
-                        print(f"    {RED}✗ {field} differs: {e1[field]} vs {e2[field]}{RESET}")
-                        all_match = False
-    
-    return all_match
-
-
-def extract_tool_calls(response: Dict[str, Any]) -> List[str]:
-    """Extract tool call names from response."""
-    tool_names = []
-    choices = response.get('choices', [])
-    if choices and isinstance(choices[0], dict):
-        tool_calls = choices[0].get('message', {}).get('tool_calls', [])
-        for tc in tool_calls:
-            tool_names.append(tc.get('function', {}).get('name', 'unknown'))
-    return tool_names
-
-
-def compare_traces(openai_dir: Path, lm_dir: Path, episode: int = 0) -> bool:
-    """Compare traces from both versions."""
-    print(f"\n{BLUE}{'='*60}{RESET}")
-    print(f"{BLUE}Comparing traces for episode {episode}{RESET}")
-    print(f"{BLUE}{'='*60}{RESET}")
-    
-    # Load traces - OpenAI uses different naming convention
-    # Look for most recent session files
-    openai_files = list(openai_dir.glob(f"session_episode_{episode}_*.json"))
-    if openai_files:
-        # Get the most recent one
-        openai_trace_file = max(openai_files, key=lambda f: f.stat().st_mtime)
-    else:
-        openai_trace_file = openai_dir / f"trace_episode_{episode}.json"
-    
-    lm_trace_file = lm_dir / f"trace_episode_{episode}.json"
-    
-    if not openai_trace_file.exists():
-        print(f"{RED}✗ OpenAI trace not found: {openai_trace_file}{RESET}")
-        return False
-    
-    if not lm_trace_file.exists():
-        print(f"{RED}✗ LM trace not found: {lm_trace_file}{RESET}")
-        return False
-    
-    openai_trace = load_trace(openai_trace_file)
-    lm_trace = load_trace(lm_trace_file)
-    
-    # Extract events
-    openai_cais = extract_cais_events(openai_trace)
-    lm_cais = extract_cais_events(lm_trace)
-    
-    openai_env = extract_env_events(openai_trace)
-    lm_env = extract_env_events(lm_trace)
-    
-    # Compare
-    cais_match = compare_events(openai_cais, lm_cais, "CAISEvent")
-    env_match = compare_events(openai_env, lm_env, "EnvironmentEvent")
-    
-    # Check messages
-    print(f"\n{BLUE}Comparing messages...{RESET}")
-    openai_msgs = openai_trace.get('message_history', [])
-    lm_msgs = lm_trace.get('message_history', [])
-    
-    if len(openai_msgs) != len(lm_msgs):
-        print(f"{RED}✗ Different number of messages: {len(openai_msgs)} vs {len(lm_msgs)}{RESET}")
-    else:
-        print(f"{GREEN}✓ Same number of messages: {len(openai_msgs)}{RESET}")
-    
-    return cais_match and env_match
-
-
-async def main():
-    """Run both versions and compare traces."""
-    print(f"{BLUE}{'='*60}{RESET}")
-    print(f"{BLUE}Crafter Trace Comparison: OpenAI vs LM{RESET}")
-    print(f"{BLUE}{'='*60}{RESET}")
-    
-    # Configuration
-    model = "gpt-4o-mini"
-    episodes = 2
-    max_turns = 3
-    
-    print(f"\nConfiguration:")
-    print(f"  Model: {model}")
-    print(f"  Episodes: {episodes}")
-    print(f"  Max turns: {max_turns}")
-    
-    # Run OpenAI version
-    openai_success, openai_dir = run_openai_version(model, episodes, max_turns)
-    if not openai_success:
-        print(f"{RED}Failed to run OpenAI version{RESET}")
-        return
-    
-    # Small delay to avoid rate limits
-    await asyncio.sleep(2)
-    
-    # Run LM version
-    lm_success, lm_dir = run_lm_version(model, episodes, max_turns)
-    if not lm_success:
-        print(f"{RED}Failed to run LM version{RESET}")
-        return
-    
-    # Compare traces
-    print(f"\n{BLUE}Trace directories:{RESET}")
-    print(f"  OpenAI: {openai_dir}")
-    print(f"  LM:     {lm_dir}")
-    
-    all_match = True
-    for episode in range(episodes):
-        match = compare_traces(Path(openai_dir), Path(lm_dir), episode)
-        all_match = all_match and match
-    
-    # Final verdict
-    print(f"\n{BLUE}{'='*60}{RESET}")
-    print(f"{BLUE}FINAL VERDICT{RESET}")
-    print(f"{BLUE}{'='*60}{RESET}")
-    
-    if all_match:
-        print(f"{GREEN}✅ Traces match! LM class produces equivalent v2 traces.{RESET}")
-    else:
-        print(f"{YELLOW}⚠️  Some differences found. This may be due to:{RESET}")
-        print(f"{YELLOW}   - LLM non-determinism (different responses){RESET}")
-        print(f"{YELLOW}   - Minor implementation differences{RESET}")
-        print(f"{YELLOW}   - Timing variations{RESET}")
-    
-    print(f"\nKey observations:")
-    print(f"  • Both versions create CAISEvents with LLM call records")
-    print(f"  • Both capture environment events and observations")
-    print(f"  • Token counts and metadata are preserved")
-    print(f"  • The LM class successfully replaces direct OpenAI calls")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/crafter_comprehensive_evaluation.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-"""
-Run script for Full Enchilada Crafter Evaluation
-"""
-
-import asyncio
-import argparse
-from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
-    run_full_enchilada_eval,
-)
-
-
-async def main():
-    parser = argparse.ArgumentParser(description="Run Full Enchilada Crafter Evaluation")
-    parser.add_argument(
-        "--models", nargs="+", default=["gpt-4o-mini"], help="Model names to evaluate"
-    )
-    parser.add_argument(
-        "--difficulties",
-        nargs="+",
-        default=["easy", "hard"],
-        help="Difficulty levels to test",
-    )
-    parser.add_argument(
-        "--num-trajectories",
-        type=int,
-        default=3,
-        help="Number of trajectories per condition",
-    )
-    parser.add_argument("--max-turns", type=int, default=30, help="Maximum turns per trajectory")
-    parser.add_argument("--no-images", action="store_true", help="Disable image capture")
-    parser.add_argument(
-        "--no-viewer",
-        action="store_true",
-        help="Don't launch the viewer after evaluation",
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        default=None,
-        help="Output directory (default: src/evals/crafter/run_TIMESTAMP)",
-    )
-
-    args = parser.parse_args()
-
-    await run_full_enchilada_eval(
-        model_names=args.models,
-        difficulties=args.difficulties,
-        num_trajectories=args.num_trajectories,
-        max_turns=args.max_turns,
-        capture_images=not args.no_images,
-        launch_viewer=not args.no_viewer,
-        output_dir=args.output_dir,
-    )
-
-
-if __name__ == "__main__":
-    asyncio.run(main())