synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server/client.py

@@ -1,429 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple multiprocess client that connects to the server and runs the agent.
-"""
-
-import os
-import sys
-import time
-import base64
-import io
-import requests
-from PIL import Image
-
-# Display-related imports (conditionally used)
-try:
-    import pygame
-    import numpy as np
-    PYGAME_AVAILABLE = True
-except ImportError:
-    PYGAME_AVAILABLE = False
-
-# Add parent directory to path for imports
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from agent import Agent
-from utils.state_formatter import format_state_for_llm
-
-
-def update_display_with_status(screen, font, mode, step_count, additional_info="", frame_surface=None):
-    """
-    Update the display with frame (if provided) and status text overlay.
-    
-    Args:
-        screen: Pygame screen surface
-        font: Pygame font for rendering text
-        mode: Current mode (MANUAL/AGENT/AUTO)
-        step_count: Current step count
-        additional_info: Additional status information to display
-        frame_surface: Optional frame surface to display, if None fills with black
-    """
-    if frame_surface:
-        # Display the frame
-        scaled_surface = pygame.transform.scale(frame_surface, (480, 320))
-        screen.blit(scaled_surface, (0, 0))
-    else:
-        # Fill with black if no frame
-        screen.fill((0, 0, 0))
-    
-    # Create status text
-    status_text = f"{mode} | Steps: {step_count}"
-    if additional_info:
-        status_text += f" | {additional_info}"
-    
-    # Render and display status text
-    text_surface = font.render(status_text, True, (255, 255, 255))
-    screen.blit(text_surface, (10, 290))
-    pygame.display.flip()
-
-
-def run_multiprocess_client(server_port=8000, args=None):
-    """
-    Simple client that gets state from server, processes with agent, sends action back.
-    Supports manual control when pygame display is enabled.
-    
-    Args:
-        server_port: Port the server is running on
-        args: Command line arguments with agent configuration
-    
-    Returns:
-        bool: True if ran successfully
-    """
-    server_url = f"http://localhost:{server_port}"
-    
-    # Initialize the agent (it handles VLM, simple vs 4-module, etc internally)
-    agent = Agent(args)
-    print(f"✅ Agent initialized")
-    print(f"🎮 Client connected to server at {server_url}")
-    
-    # Display setup
-    headless = args and args.headless
-    screen = None
-    clock = None
-    font = None
-    
-    # Control state - three modes: MANUAL, AGENT, AUTO
-    if args and args.manual:
-        mode = "MANUAL"
-    elif args and args.agent_auto:
-        mode = "AUTO"
-    else:
-        mode = "AGENT"
-    
-    last_agent_time = time.time()
-    step_count = 0
-    
-    # Initialize pygame if not headless
-    if not headless and PYGAME_AVAILABLE:
-        pygame.init()
-        screen = pygame.display.set_mode((480, 320))
-        pygame.display.set_caption("Pokemon Emerald")
-        font = pygame.font.Font(None, 24)
-        clock = pygame.time.Clock()
-        print("✅ Display initialized")
-        print("Controls: Tab=Cycle Mode (MANUAL/AGENT/AUTO), Space=Agent Step, M=Show State, Arrows/WASD=Move, Z=A, X=B")
-    elif not headless and not PYGAME_AVAILABLE:
-        print("⚠️ Pygame not available, running in headless mode")
-        headless = True
-    
-    # Auto-display comprehensive state in manual mode for debugging
-    auto_state_timer = None
-    if args and args.manual:
-        auto_state_timer = time.time() + 5  # Display state after 5 seconds in manual mode (allow map to initialize)
-    
-    # Main loop
-    running = True
-    while running:
-        try:
-            # Auto-display comprehensive state in manual mode (one time)
-            if auto_state_timer and time.time() >= auto_state_timer:
-                print("🔍 Auto-displaying comprehensive state in manual mode...")
-                try:
-                    response = requests.get(f"{server_url}/state", timeout=5)
-                    if response.status_code == 200:
-                        state_data = response.json()
-                        print("=" * 80)
-                        print("📊 COMPREHENSIVE STATE (LLM View)")
-                        print("=" * 80)
-                        from utils.state_formatter import format_state_for_llm
-                        formatted_state = format_state_for_llm(state_data)
-                        print(formatted_state)
-                        print("=" * 80)
-                    else:
-                        print(f"❌ Failed to get state: {response.status_code}")
-                except Exception as e:
-                    print(f"❌ Error getting state: {e}")
-                auto_state_timer = None  # Only display once
-            
-            # Handle pygame events and display
-            if not headless:
-                # Process events
-                for event in pygame.event.get():
-                    if event.type == pygame.QUIT:
-                        running = False
-                        break
-                    
-                    elif event.type == pygame.KEYDOWN:
-                        if event.key == pygame.K_ESCAPE:
-                            running = False
-                            break
-                        
-                        # Mode cycle
-                        elif event.key == pygame.K_TAB:
-                            if mode == "MANUAL":
-                                mode = "AGENT"
-                            elif mode == "AGENT":
-                                mode = "AUTO"
-                            else:  # AUTO
-                                mode = "MANUAL"
-                            print(f"🎮 Mode: {mode}")
-                        
-                        # Manual agent step
-                        elif event.key == pygame.K_SPACE and mode in ("AGENT", "AUTO"):
-                            # Force an agent step
-                            response = requests.get(f"{server_url}/state", timeout=5)
-                            if response.status_code == 200:
-                                state_data = response.json()
-                                screenshot_base64 = state_data.get("visual", {}).get("screenshot_base64", "")
-                                if screenshot_base64:
-                                    img_data = base64.b64decode(screenshot_base64)
-                                    screenshot = Image.open(io.BytesIO(img_data))
-                                    game_state = {
-                                        'frame': screenshot,
-                                        'player': state_data.get('player', {}),
-                                        'game': state_data.get('game', {}),
-                                        'map': state_data.get('map', {}),
-                                        'milestones': state_data.get('milestones', {}),
-                                        'visual': state_data.get('visual', {}),
-                                        'step_number': state_data.get('step_number', 0),
-                                        'status': state_data.get('status', ''),
-                                        'action_queue_length': state_data.get('action_queue_length', 0)
-                                    }
-                                    result = agent.step(game_state)
-                                    if result and result.get('action'):
-                                        # Convert action to buttons list format expected by server
-                                        action = result['action']
-                                        if isinstance(action, list):
-                                            buttons = action  # Already a list of buttons
-                                        else:
-                                            # Single action string, convert to list
-                                            buttons = action.split(',') if ',' in action else [action]
-                                            buttons = [btn.strip() for btn in buttons]
-                                        
-                                        try:
-                                            response = requests.post(
-                                                f"{server_url}/action",
-                                                json={"buttons": buttons},
-                                                timeout=5
-                                            )
-                                            if response.status_code == 200:
-                                                print(f"🎮 Agent: {action} (sent successfully)")
-                                            else:
-                                                print(f"🎮 Agent: {action} (server error: {response.status_code})")
-                                        except requests.exceptions.RequestException as e:
-                                            print(f"🎮 Agent: {action} (connection error: {e})")
-                                        step_count += 1
-                                        print(f"🎮 Step {step_count}: {result['action']}")
-                        
-                        # Manual controls (only in manual mode)
-                        elif mode == "MANUAL":
-                            action = None
-                            if event.key in (pygame.K_UP, pygame.K_w):
-                                action = "UP"
-                            elif event.key in (pygame.K_DOWN, pygame.K_s):
-                                action = "DOWN"
-                            elif event.key in (pygame.K_LEFT, pygame.K_a):
-                                action = "LEFT"
-                            elif event.key in (pygame.K_RIGHT, pygame.K_d):
-                                action = "RIGHT"
-                            elif event.key == pygame.K_z:
-                                action = "A"
-                            elif event.key == pygame.K_x:
-                                action = "B"
-                            elif event.key == pygame.K_RETURN:
-                                action = "START"
-                            elif event.key == pygame.K_BACKSPACE:
-                                action = "SELECT"
-                            elif event.key == pygame.K_LSHIFT:
-                                action = "L"
-                            elif event.key == pygame.K_RSHIFT:
-                                action = "R"
-                            elif event.key == pygame.K_1:
-                                # Save state
-                                print("💾 Saving state...")
-                                try:
-                                    response = requests.post(f"{server_url}/save_state", 
-                                                           json={"filepath": ".pokeagent_cache/manual_save.state"}, 
-                                                           timeout=5)
-                                    if response.status_code == 200:
-                                        print("✅ State saved to .pokeagent_cache/manual_save.state")
-                                    else:
-                                        print(f"❌ Failed to save state: {response.status_code}")
-                                except Exception as e:
-                                    print(f"❌ Error saving state: {e}")
-                            elif event.key == pygame.K_2:
-                                # Load state
-                                print("📂 Loading state...")
-                                try:
-                                    response = requests.post(f"{server_url}/load_state", 
-                                                           json={"filepath": ".pokeagent_cache/manual_save.state"}, 
-                                                           timeout=5)
-                                    if response.status_code == 200:
-                                        print("✅ State loaded from .pokeagent_cache/manual_save.state")
-                                    else:
-                                        print(f"❌ Failed to load state: {response.status_code}")
-                                except Exception as e:
-                                    print(f"❌ Error loading state: {e}")
-                            elif event.key == pygame.K_m:
-                                # Display comprehensive state (what LLM sees)
-                                print("🔍 Getting comprehensive state...")
-                                try:
-                                    response = requests.get(f"{server_url}/state", timeout=5)
-                                    if response.status_code == 200:
-                                        state_data = response.json()
-                                        print("=" * 80)
-                                        print("📊 COMPREHENSIVE STATE (LLM View)")
-                                        print("=" * 80)
-                                        
-                                        # Format and display state in a readable way (exactly what LLM sees)
-                                        formatted_state = format_state_for_llm(state_data)
-                                        print(formatted_state)
-                                        
-                                        print("=" * 80)
-                                    else:
-                                        print(f"❌ Failed to get state: {response.status_code}")
-                                except Exception as e:
-                                    print(f"❌ Error getting state: {e}")
-                            
-                            if action:
-                                # Send manual action to server using the same endpoint as agent actions
-                                try:
-                                    response = requests.post(
-                                        f"{server_url}/action",
-                                        json={"buttons": [action]},
-                                        timeout=2
-                                    )
-                                    if response.status_code == 200:
-                                        print(f"🎮 Manual: {action} (sent successfully)")
-                                    else:
-                                        print(f"🎮 Manual: {action} (server error: {response.status_code})")
-                                except requests.exceptions.RequestException as e:
-                                    print(f"🎮 Manual: {action} (connection error: {e})")
-                
-                # Update display
-                try:
-                    response = requests.get(f"{server_url}/screenshot", timeout=0.5)
-                    if response.status_code == 200:
-                        frame_data = response.json().get("screenshot_base64", "")
-                        if frame_data:
-                            img_data = base64.b64decode(frame_data)
-                            img = Image.open(io.BytesIO(img_data))
-                            frame_array = np.array(img)
-                            frame_surface = pygame.surfarray.make_surface(frame_array.swapaxes(0, 1))
-                            update_display_with_status(screen, font, mode, step_count, frame_surface=frame_surface)
-                        else:
-                            update_display_with_status(screen, font, mode, step_count, "No frame data")
-                    else:
-                        update_display_with_status(screen, font, mode, step_count, f"Server error: {response.status_code}")
-                except Exception as e:
-                    update_display_with_status(screen, font, mode, step_count, f"Error: {str(e)[:30]}")
-                
-                clock.tick(30)  # 30 FPS for display
-            
-            # Auto agent processing (both headless and display modes)
-            if mode == "AUTO":
-                current_time = time.time()
-                if current_time - last_agent_time > 3.0:  # Every 3 seconds
-                    # Check if action queue is ready
-                    try:
-                        queue_response = requests.get(f"{server_url}/queue_status", timeout=1)
-                        if queue_response.status_code == 200:
-                            queue_status = queue_response.json()
-                            if queue_status.get("queue_empty", False):
-                                # Get state and process
-                                response = requests.get(f"{server_url}/state", timeout=5)
-                                if response.status_code == 200:
-                                    state_data = response.json()
-                                    screenshot_base64 = state_data.get("visual", {}).get("screenshot_base64", "")
-                                    if screenshot_base64:
-                                        img_data = base64.b64decode(screenshot_base64)
-                                        screenshot = Image.open(io.BytesIO(img_data))
-                                        
-                                        game_state = {
-                                            'frame': screenshot,
-                                            'player': state_data.get('player', {}),
-                                            'game': state_data.get('game', {}),
-                                            'map': state_data.get('map', {}),
-                                            'milestones': state_data.get('milestones', {}),
-                                            'visual': state_data.get('visual', {}),
-                                            'step_number': state_data.get('step_number', 0),
-                                            'status': state_data.get('status', ''),
-                                            'action_queue_length': state_data.get('action_queue_length', 0)
-                                        }
-                                        
-                                        result = agent.step(game_state)
-                                        if result and result.get('action'):
-                                            # Convert action to buttons list format expected by server
-                                            action = result['action']
-                                            if isinstance(action, list):
-                                                buttons = action  # Already a list of buttons
-                                            else:
-                                                # Single action string, convert to list
-                                                buttons = action.split(',') if ',' in action else [action]
-                                                buttons = [btn.strip() for btn in buttons]
-                                            
-                                            try:
-                                                response = requests.post(
-                                                    f"{server_url}/action",
-                                                    json={"buttons": buttons},
-                                                    timeout=5
-                                                )
-                                                if response.status_code == 200:
-                                                    step_count += 1
-                                                    print(f"🎮 Agent: {action} (sent successfully)")
-                                                    print(f"🎮 Step {step_count}: {result['action']}")
-                                                    last_agent_time = current_time
-                                                    
-                                                    # Auto-save checkpoint after each step for persistence
-                                                    try:
-                                                        # Sync client's LLM metrics to server before saving checkpoint
-                                                        try:
-                                                            from utils.llm_logger import get_llm_logger
-                                                            client_llm_logger = get_llm_logger()
-                                                            if client_llm_logger:
-                                                                sync_response = requests.post(
-                                                                    f"{server_url}/sync_llm_metrics",
-                                                                    json={"cumulative_metrics": client_llm_logger.cumulative_metrics},
-                                                                    timeout=5
-                                                                )
-                                                                if sync_response.status_code == 200:
-                                                                    if step_count % 10 == 0:  # Log every 10 steps to avoid spam
-                                                                        print(f"🔄 LLM metrics synced to server")
-                                                        except Exception as e:
-                                                            print(f"⚠️ LLM metrics sync error: {e}")
-                                                        
-                                                        # Save game state checkpoint
-                                                        checkpoint_response = requests.post(
-                                                            f"{server_url}/checkpoint",
-                                                            json={"step_count": step_count},
-                                                            timeout=10
-                                                        )
-                                                        
-                                                        # Save agent history to checkpoint_llm.txt
-                                                        history_response = requests.post(
-                                                            f"{server_url}/save_agent_history",
-                                                            timeout=5
-                                                        )
-                                                        
-                                                        if checkpoint_response.status_code == 200 and history_response.status_code == 200:
-                                                            if step_count % 10 == 0:  # Log every 10 steps to avoid spam
-                                                                print(f"💾 Checkpoint and history saved at step {step_count}")
-                                                        else:
-                                                            print(f"⚠️ Save failed - Checkpoint: {checkpoint_response.status_code}, History: {history_response.status_code}")
-                                                    except requests.exceptions.RequestException as e:
-                                                        print(f"⚠️ Checkpoint/history save error: {e}")
-                                                else:
-                                                    print(f"🎮 Agent: {action} (server error: {response.status_code})")
-                                            except requests.exceptions.RequestException as e:
-                                                print(f"🎮 Agent: {action} (connection error: {e})")
-                    except Exception as e:
-                        print(f"❌ AUTO mode error: {e}")
-                        import traceback
-                        traceback.print_exc()
-            
-            # Small sleep to prevent CPU spinning
-            if headless:
-                time.sleep(0.1)
-            
-        except KeyboardInterrupt:
-            print("\n👋 Shutdown requested")
-            break
-        except Exception as e:
-            print(f"❌ Error: {e}")
-            time.sleep(2)
-    
-    # Cleanup
-    if not headless and PYGAME_AVAILABLE:
-        pygame.quit()
-    
-    return True

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server/frame_server.py

@@ -1,155 +0,0 @@
-#!/usr/bin/env python3
-"""
-Lightweight frame server for stream.html
-Serves only screenshot frames, separate from main game server
-"""
-
-import os
-import sys
-import time
-import json
-import base64
-import threading
-from pathlib import Path
-from PIL import Image
-import numpy as np
-import argparse
-
-# Add parent directory to path for imports
-sys.path.append(str(Path(__file__).parent.parent))
-
-try:
-    from fastapi import FastAPI, Response
-    from fastapi.middleware.cors import CORSMiddleware
-    import uvicorn
-except ImportError:
-    print("❌ FastAPI not available. Install with: pip install fastapi uvicorn")
-    sys.exit(1)
-
-app = FastAPI(title="Pokemon Frame Server")
-
-# Add CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-# Global state
-current_frame = None
-frame_lock = threading.Lock()
-frame_counter = 0
-last_update = time.time()
-
-# Frame cache for shared memory communication
-# Use cache directory instead of /tmp
-CACHE_DIR = ".pokeagent_cache"
-os.makedirs(CACHE_DIR, exist_ok=True)
-FRAME_CACHE_FILE = os.path.join(CACHE_DIR, "frame_cache.json")
-FRAME_UPDATE_INTERVAL = 0.025  # 40 FPS
-
-def load_frame_from_cache():
-    """Load the latest frame from shared cache file"""
-    global current_frame, frame_counter, last_update
-    
-    try:
-        if os.path.exists(FRAME_CACHE_FILE):
-            with open(FRAME_CACHE_FILE, 'r') as f:
-                data = json.load(f)
-                
-            # Check if frame is newer
-            cache_counter = data.get('frame_counter', 0)
-            if cache_counter > frame_counter:
-                with frame_lock:
-                    current_frame = data.get('frame_data')
-                    frame_counter = cache_counter
-                    last_update = time.time()
-                    
-    except Exception as e:
-        pass  # Silently handle cache read errors
-
-def frame_updater():
-    """Background thread to periodically check for new frames"""
-    while True:
-        try:
-            load_frame_from_cache()
-            time.sleep(FRAME_UPDATE_INTERVAL)
-        except Exception:
-            time.sleep(0.1)
-
-@app.get("/health")
-async def health_check():
-    """Simple health check endpoint"""
-    return {"status": "ok", "server": "frame_server"}
-
-@app.get("/frame")
-async def get_frame():
-    """Get the current game frame"""
-    global current_frame, frame_counter, last_update
-    
-    try:
-        load_frame_from_cache()  # Try to get latest frame
-        
-        with frame_lock:
-            if current_frame:
-                # Frame is already base64 encoded
-                return Response(
-                    content=json.dumps({
-                        "frame": current_frame,
-                        "frame_count": frame_counter,
-                        "timestamp": last_update,
-                        "status": "ok"
-                    }),
-                    media_type="application/json"
-                )
-            else:
-                # No frame available
-                return Response(
-                    content=json.dumps({
-                        "frame": None,
-                        "frame_count": 0,
-                        "timestamp": time.time(),
-                        "status": "no_frame"
-                    }),
-                    media_type="application/json"
-                )
-                
-    except Exception as e:
-        return Response(
-            content=json.dumps({
-                "frame": None,
-                "error": str(e),
-                "status": "error"
-            }),
-            media_type="application/json",
-            status_code=500
-        )
-
-@app.get("/status")
-async def get_status():
-    """Get frame server status"""
-    return {
-        "frame_count": frame_counter,
-        "last_update": last_update,
-        "cache_file": FRAME_CACHE_FILE,
-        "cache_exists": os.path.exists(FRAME_CACHE_FILE)
-    }
-
-if __name__ == "__main__":
-    
-    parser = argparse.ArgumentParser(description="Pokemon Frame Server")
-    parser.add_argument("--port", type=int, default=8001, help="Port to run on")
-    parser.add_argument("--host", type=str, default="127.0.0.1", help="Host to bind to")
-    args = parser.parse_args()
-    
-    print(f"🖼️ Starting Pokemon Frame Server on {args.host}:{args.port}")
-    print(f"📁 Frame cache: {FRAME_CACHE_FILE}")
-    
-    # Start background frame updater
-    frame_thread = threading.Thread(target=frame_updater, daemon=True)
-    frame_thread.start()
-    
-    # Start server
-    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/README.md

@@ -1,78 +0,0 @@
-# Pokemon Emerald Emulator Tests
-
-This directory contains tests for the Pokemon Emerald emulator FPS adjustment system.
-
-## Directory Structure
-
-```
-tests/
-├── states/                    # Test state files
-│   ├── simple_test.state     # Base overworld state (30 FPS)
-│   ├── dialog.state          # Dialog state (120 FPS)
-│   ├── dialog2.state         # Dialog state 2 (120 FPS - currently failing)
-│   ├── after_dialog.state    # After dialog state (30 FPS)
-│   └── torchic.state         # Additional test state
-├── test_fps_adjustment_pytest.py  # Main FPS adjustment tests
-├── run_tests.py              # Test runner script
-└── README.md                 # This file
-```
-
-## Test States
-
-### `simple_test.state`
-- **Expected FPS**: 30
-- **Description**: Normal overworld state without dialog
-- **Purpose**: Verify base FPS is working correctly
-
-### `dialog.state`
-- **Expected FPS**: 120 (4x speedup)
-- **Description**: State with active dialog
-- **Purpose**: Verify dialog detection and FPS boost
-
-### `dialog2.state`
-- **Expected FPS**: 120 (4x speedup)
-- **Description**: Another dialog state (currently not detected as dialog)
-- **Purpose**: Test dialog detection robustness
-- **Status**: Currently failing - needs investigation
-
-### `after_dialog.state`
-- **Expected FPS**: 30
-- **Description**: State after dialog has ended
-- **Purpose**: Verify FPS reverts to normal after dialog timeout
-
-## Running Tests
-
-### Using pytest (Recommended)
-```bash
-# Run all FPS adjustment tests
-conda activate mgba && python -m pytest tests/test_fps_adjustment_pytest.py -v
-
-# Run specific test
-conda activate mgba && python -m pytest tests/test_fps_adjustment_pytest.py::test_fps_adjustment_summary -v
-
-# Run all tests in the tests directory
-conda activate mgba && python -m pytest tests/ -v
-```
-
-### Using the test runner
-```bash
-# Run all tests
-conda activate mgba && python tests/run_tests.py
-
-# Run specific test
-conda activate mgba && python tests/run_tests.py test_fps_adjustment_pytest
-```
-
-## Test Results
-
-The FPS adjustment system should:
-1. ✅ Run at 30 FPS in normal overworld state
-2. ✅ Speed up to 120 FPS (4x) when dialog is detected
-3. ❌ Speed up to 120 FPS for dialog2.state (currently failing)
-4. ✅ Revert to 30 FPS when dialog ends
-
-## Notes
-
-- The `dialog2.state` test is expected to fail until the dialog detection logic is improved
-- All state files are now centralized in `tests/states/` for better organization
-- Tests use a timeout-based approach where dialog FPS runs for 5 seconds then reverts