synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/__init__.py

@@ -0,0 +1,120 @@
+"""
+Agent modules for Pokemon Emerald speedrunning agent
+"""
+
+from utils.vlm import VLM
+from .action import action_step
+from .memory import memory_step
+from .perception import perception_step
+from .planning import planning_step
+from .simple import SimpleAgent, get_simple_agent, simple_mode_processing_multiprocess, configure_simple_agent_defaults
+
+
+class Agent:
+    """
+    Unified agent interface that encapsulates all agent logic.
+    The client just calls agent.step(game_state) and gets back an action.
+    """
+    
+    def __init__(self, args=None):
+        """
+        Initialize the agent based on configuration.
+        
+        Args:
+            args: Command line arguments with agent configuration
+        """
+        # Extract configuration
+        backend = args.backend if args else "gemini"
+        model_name = args.model_name if args else "gemini-2.5-flash"
+        simple_mode = args.simple if args else False
+        
+        # Initialize VLM
+        self.vlm = VLM(backend=backend, model_name=model_name)
+        print(f"   VLM: {backend}/{model_name}")
+        
+        # Initialize agent mode
+        self.simple_mode = simple_mode
+        if simple_mode:
+            # Use global SimpleAgent instance to enable checkpoint persistence
+            self.simple_agent = get_simple_agent(self.vlm)
+            print(f"   Mode: Simple (direct frame->action)")
+        else:
+            # Four-module agent context
+            self.context = {
+                'perception_output': None,
+                'planning_output': None,
+                'memory': []
+            }
+            print(f"   Mode: Four-module architecture")
+    
+    def step(self, game_state):
+        """
+        Process a game state and return an action.
+        
+        Args:
+            game_state: Dictionary containing:
+                - screenshot: PIL Image
+                - game_state: Dict with game memory data
+                - visual: Dict with visual observations
+                - audio: Dict with audio observations
+                - progress: Dict with milestone progress
+        
+        Returns:
+            dict: Contains 'action' and optionally 'reasoning'
+        """
+        if self.simple_mode:
+            # Simple mode - delegate to SimpleAgent
+            return self.simple_agent.step(game_state)
+        else:
+            # Four-module processing
+            try:
+                # 1. Perception - understand what's happening
+                perception_output = perception_step(
+                    self.vlm, 
+                    game_state, 
+                    self.context.get('memory', [])
+                )
+                self.context['perception_output'] = perception_output
+                
+                # 2. Planning - decide strategy
+                planning_output = planning_step(
+                    self.vlm, 
+                    perception_output, 
+                    self.context.get('memory', [])
+                )
+                self.context['planning_output'] = planning_output
+                
+                # 3. Memory - update context
+                memory_output = memory_step(
+                    perception_output, 
+                    planning_output, 
+                    self.context.get('memory', [])
+                )
+                self.context['memory'] = memory_output
+                
+                # 4. Action - choose button press
+                action_output = action_step(
+                    self.vlm, 
+                    game_state, 
+                    planning_output,
+                    perception_output
+                )
+                
+                return action_output
+                
+            except Exception as e:
+                print(f"❌ Agent error: {e}")
+                return None
+
+
+__all__ = [
+    'Agent',
+    'action_step',
+    'memory_step', 
+    'perception_step',
+    'planning_step',
+    'SimpleAgent',
+    'get_simple_agent',
+    'simple_mode_processing_multiprocess',
+    'configure_simple_agent_defaults'
+]

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/action.py

@@ -0,0 +1,160 @@
+import logging
+import random
+import sys
+from agent.system_prompt import system_prompt
+from utils.state_formatter import format_state_for_llm, format_state_summary, get_movement_options, get_party_health_summary
+from utils.vlm import VLM
+
+# Set up module logging
+logger = logging.getLogger(__name__)
+
+def action_step(memory_context, current_plan, latest_observation, frame, state_data, recent_actions, vlm):
+    """
+    Decide and perform the next action button(s) based on memory, plan, observation, and comprehensive state.
+    Returns a list of action buttons as strings.
+    """
+    # Get formatted state context and useful summaries
+    state_context = format_state_for_llm(state_data)
+    state_summary = format_state_summary(state_data)
+    movement_options = get_movement_options(state_data)
+    party_health = get_party_health_summary(state_data)
+    
+    logger.info("[ACTION] Starting action decision")
+    logger.info(f"[ACTION] State: {state_summary}")
+    logger.info(f"[ACTION] Party health: {party_health['healthy_count']}/{party_health['total_count']} healthy")
+    if movement_options:
+        logger.info(f"[ACTION] Movement options: {movement_options}")
+    
+    # Build enhanced action context
+    action_context = []
+    
+    # Extract key info for context
+    game_data = state_data.get('game', {})
+    
+    # Battle vs Overworld context
+    if game_data.get('in_battle', False):
+        action_context.append("=== BATTLE MODE ===")
+        battle_info = game_data.get('battle_info', {})
+        if battle_info:
+            if 'player_pokemon' in battle_info:
+                player_pkmn = battle_info['player_pokemon']
+                action_context.append(f"Your Pokemon: {player_pkmn.get('species_name', player_pkmn.get('species', 'Unknown'))} (Lv.{player_pkmn.get('level', '?')}) HP: {player_pkmn.get('current_hp', '?')}/{player_pkmn.get('max_hp', '?')}")
+            if 'opponent_pokemon' in battle_info:
+                opp_pkmn = battle_info['opponent_pokemon']
+                action_context.append(f"Opponent: {opp_pkmn.get('species_name', opp_pkmn.get('species', 'Unknown'))} (Lv.{opp_pkmn.get('level', '?')}) HP: {opp_pkmn.get('current_hp', '?')}/{opp_pkmn.get('max_hp', '?')}")
+    else:
+        action_context.append("=== OVERWORLD MODE ===")
+        
+        # Movement options from utility
+        if movement_options:
+            action_context.append("Movement Options:")
+            for direction, description in movement_options.items():
+                action_context.append(f"  {direction}: {description}")
+    
+    # Party health summary
+    if party_health['total_count'] > 0:
+        action_context.append("=== PARTY STATUS ===")
+        action_context.append(f"Healthy Pokemon: {party_health['healthy_count']}/{party_health['total_count']}")
+        if party_health['critical_pokemon']:
+            action_context.append("Critical Pokemon:")
+            for critical in party_health['critical_pokemon']:
+                action_context.append(f"  {critical}")
+    
+    # Recent actions context
+    if recent_actions:
+        action_context.append(f"Recent Actions: {', '.join(list(recent_actions)[-5:])}")
+    
+    context_str = "\n".join(action_context)
+    
+    action_prompt = f"""
+    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
+    
+    {state_context}
+    
+    ★★★ ENHANCED ACTION CONTEXT ★★★
+    
+    {context_str}
+    
+    ★★★ ACTION DECISION TASK ★★★
+    
+    You are the agent playing Pokemon Emerald with a speedrunning mindset. Make quick, efficient decisions.
+    
+    Memory Context: {memory_context}
+    Current Plan: {current_plan if current_plan else 'No plan yet'}
+    Latest Observation: {latest_observation}
+    
+    Based on the comprehensive state information above, decide your next action(s):
+    
+    BATTLE STRATEGY:
+    - If in battle: Choose moves strategically based on type effectiveness and damage
+    - Consider switching pokemon if current one is weak/low HP
+    - Use items if pokemon is in critical condition
+    
+    NAVIGATION STRATEGY:
+    - Use movement options analysis above for efficient navigation
+    - Avoid blocked tiles (marked as BLOCKED)
+    - Consider tall grass: avoid if party is weak, seek if need to train/catch
+    - Navigate around water unless you have Surf
+    - Use coordinates to track progress toward objectives
+    
+    MENU/DIALOGUE STRATEGY:
+    - If in dialogue: A to advance text, B to cancel/skip if possible
+    - If in menu: Navigate with UP/DOWN/LEFT/RIGHT, A to select, B to cancel/back out
+    - If stuck in menu/interface: B repeatedly to exit to overworld
+    - In Pokemon Center: A to talk to Nurse Joy, A to confirm healing
+    
+    HEALTH MANAGEMENT:
+    - If pokemon are low HP/fainted, head to Pokemon Center
+    - If no healthy pokemon, prioritize healing immediately
+    - Consider terrain: avoid wild encounters if party is weak
+    
+    EFFICIENCY RULES:
+    1. Output sequences of actions when you know what's coming (e.g., "RIGHT, RIGHT, RIGHT, A" to enter a door)
+    2. For dialogue: "A, A, A, A, A" to mash through
+    3. For movement: repeat directions based on movement options (e.g., "UP, UP, UP, UP" if UP shows "Normal path")
+    4. If uncertain, output single action and reassess
+    5. Use traversability data: move toward open paths, avoid obstacles
+    6. If movement doesn't change coordinates (e.g., RIGHT but X doesn't increase), check map for walls (#) blocking your path
+    
+    Valid buttons: A, B, SELECT, START, UP, DOWN, LEFT, RIGHT, L, R
+    - A: Interact with NPCs/objects, confirm selections, advance dialogue, use moves in battle
+    - B: Cancel menus, back out of interfaces, run faster (with running shoes), flee from battle
+    - START: Open main menu (Title sequence, Pokedex, Pokemon, Bag, etc.)
+    - SELECT: Use registered key item (typically unused)
+    - UP/DOWN/LEFT/RIGHT: Move character, navigate menus, select options
+    - L/R: Cycle through pages in some menus, switch Pokemon in battle (rare usage)
+    
+    ⚠️ CRITICAL WARNING: NEVER save the game using the in-game save menu! Saving will crash the entire run and end your progress. If you encounter a save prompt in the game, press B to cancel it immediately!
+    
+    Return ONLY the button name(s) as a comma-separated list, nothing else.
+    Maximum 10 actions in sequence. Avoid repeating same button more than 6 times.
+    """
+    
+    # Construct complete prompt for VLM
+    complete_prompt = system_prompt + action_prompt
+    
+    action_response = vlm.get_text_query(complete_prompt, "ACTION").strip().upper()
+    valid_buttons = ['A', 'B', 'SELECT', 'START', 'UP', 'DOWN', 'LEFT', 'RIGHT', 'L', 'R']
+    
+    # Split the response by commas and clean up
+    actions = [btn.strip() for btn in action_response.split(',') if btn.strip() in valid_buttons]
+    
+    print(f"Parsed actions: {actions}")
+    if len(actions) == 0:
+        print("❌ No valid actions parsed - using default 'A'")
+    print("-" * 80 + "\n")
+    
+    # Limit to maximum 10 actions and prevent excessive repetition
+    actions = actions[:10]
+    
+    # If no valid actions found, make intelligent default based on state
+    if not actions:
+        if game_data.get('in_battle', False):
+            actions = ['A']  # Attack in battle
+        elif party_health['total_count'] == 0:
+            actions = ['A', 'A', 'A']  # Try to progress dialogue/menu
+        else:
+            actions = [random.choice(['A', 'RIGHT', 'UP', 'DOWN', 'LEFT'])]  # Random exploration
+    
+    logger.info(f"[ACTION] Actions decided: {', '.join(actions)}")
+    return actions 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/memory.py

@@ -0,0 +1,155 @@
+import logging
+from collections import deque
+from agent.system_prompt import system_prompt
+from utils.state_formatter import format_state_summary, get_party_health_summary
+from utils.vlm import VLM
+
+# Set up module logging
+logger = logging.getLogger(__name__)
+
+def extract_key_state_info(state_data):
+    """Extract key information from comprehensive state for memory storage using the utility functions"""
+    # Use the state formatter utilities for consistency
+    state_summary = format_state_summary(state_data)
+    party_health = get_party_health_summary(state_data)
+    
+    # Extract additional info
+    player_data = state_data.get('player', {})
+    game_data = state_data.get('game', {})
+    map_info = state_data.get('map', {})
+    
+    key_info = {
+        'state_summary': state_summary,
+        'player_name': player_data.get('name', 'Player'),
+        'money': player_data.get('money') or game_data.get('money', 0),
+        'current_map': player_data.get('location', 'Unknown Location'),
+        'in_battle': game_data.get('in_battle', False),
+        'party_health': f"{party_health['healthy_count']}/{party_health['total_count']}",
+        'critical_pokemon': party_health['critical_pokemon']
+    }
+    
+    # Position info
+    if 'coordinates' in player_data:
+        key_info['position'] = player_data['coordinates']
+    elif 'position' in player_data:
+        key_info['position'] = player_data['position']
+    else:
+        key_info['position'] = {}
+    
+    # Battle opponent
+    if game_data.get('battle_info'):
+        battle = game_data['battle_info']
+        opponent_pokemon = battle.get('opponent_pokemon', {})
+        key_info['battle_opponent'] = opponent_pokemon.get('species_name', opponent_pokemon.get('species', 'Unknown Pokemon'))
+    
+    # Traversability summary
+    if 'traversability' in map_info and map_info['traversability']:
+        traversability = map_info['traversability']
+        total_tiles = sum(len(row) for row in traversability)
+        blocked_count = sum(1 for row in traversability for cell in row if str(cell) in ['0', '0'])
+        passable_tiles = total_tiles - blocked_count
+        key_info['traversability_summary'] = f"{passable_tiles}/{total_tiles} passable"
+    else:
+        key_info['traversability_summary'] = "No data"
+    
+    return key_info
+
+def memory_step(memory_context, current_plan, recent_actions, observation_buffer, vlm):
+    """
+    Maintain a rolling buffer of the previous 50 actions and observations with state information.
+    Returns updated memory_context with the most recent 50 entries and key insights.
+    """
+    # Initialize memory buffer if it doesn't exist
+    if not hasattr(memory_step, 'memory_buffer'):
+        memory_step.memory_buffer = deque(maxlen=50)
+    
+    logger.info(f"[MEMORY] Processing {len(observation_buffer)} new observations")
+    
+    # Add new observations with state info to the buffer
+    for obs in observation_buffer:
+        state_info = extract_key_state_info(obs.get('state', {}))
+        memory_step.memory_buffer.append({
+            "type": "observation",
+            "frame_id": obs["frame_id"],
+            "content": obs["observation"],
+            "state": state_info
+        })
+        logger.info(f"[MEMORY] Added observation frame {obs['frame_id']}: {state_info['state_summary']}")
+    
+    # Add recent actions to the buffer
+    for action in recent_actions:
+        memory_step.memory_buffer.append({
+            "type": "action",
+            "content": action
+        })
+    
+    # Create a formatted memory context from the buffer with state insights
+    memory_entries = []
+    key_events = []
+    
+    # Track significant state changes
+    previous_map = None
+    previous_battle_state = None
+    
+    for i, entry in enumerate(memory_step.memory_buffer):
+        if entry["type"] == "observation":
+            frame_id = entry['frame_id']
+            description = entry['content']
+            state = entry.get('state', {})
+            
+            # Use the consistent state summary
+            state_summary = state.get('state_summary', '')
+            
+            # Check for significant events
+            current_map = state.get('current_map', 'Unknown Location')
+            current_battle = state.get('in_battle', False)
+            
+            if current_map != previous_map and previous_map is not None:
+                key_events.append(f"Moved from {previous_map} to {current_map}")
+                logger.info(f"[MEMORY] Key event: Map change from {previous_map} to {current_map}")
+            
+            if current_battle != previous_battle_state:
+                if current_battle:
+                    opponent = state.get('battle_opponent', 'Unknown Pokemon')
+                    key_events.append(f"Entered battle vs {opponent}")
+                    logger.info(f"[MEMORY] Key event: Entered battle vs {opponent}")
+                else:
+                    key_events.append("Exited battle")
+                    logger.info("[MEMORY] Key event: Exited battle")
+            
+            previous_map = current_map
+            previous_battle_state = current_battle
+            
+            # Format observation entry
+            if isinstance(description, dict):
+                desc_text = description.get('description', str(description))
+            else:
+                desc_text = str(description)
+            
+            memory_entries.append(f"Frame {frame_id}: {desc_text} [{state_summary}]")
+        else:
+            memory_entries.append(f"Action: {entry['content']}")
+    
+    # Get current state summary from the latest observation
+    current_state_summary = ""
+    if observation_buffer:
+        latest_state = extract_key_state_info(observation_buffer[-1].get('state', {}))
+        current_state_summary = latest_state.get('state_summary', 'No state data')
+    
+    # Combine into comprehensive memory context
+    memory_context = f"""★★★ COMPREHENSIVE MEMORY CONTEXT ★★★
+
+CURRENT STATE: {current_state_summary}
+
+CURRENT PLAN: {current_plan if current_plan else 'No plan yet'}
+
+KEY EVENTS: {' -> '.join(key_events[-5:]) if key_events else 'None recently'}
+
+RECENT MEMORY (last 50 entries):
+{chr(10).join(memory_entries[-30:])}"""  # Show last 30 entries to avoid too much text
+    
+    logger.info(f"[MEMORY] Memory context updated with {len(memory_entries)} total entries")
+    logger.info(f"[MEMORY] Current state: {current_state_summary}")
+    logger.info(f"[MEMORY] Key events: {len(key_events)} tracked")
+    
+    return memory_context 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/perception.py

@@ -0,0 +1,69 @@
+import time
+import logging
+from utils.vlm import VLM
+from utils.state_formatter import format_state_for_llm, format_state_summary
+from agent.system_prompt import system_prompt
+
+# Set up module logging
+logger = logging.getLogger(__name__)
+
+def perception_step(frame, state_data, vlm):
+    """
+    Observe and describe your current situation using both visual and comprehensive state data.
+    Returns (observation, slow_thinking_needed)
+    """
+    # Format the comprehensive state context using the utility
+    state_context = format_state_for_llm(state_data)
+    
+    # Log the state data being used
+    state_summary = format_state_summary(state_data)
+    logger.info("[PERCEPTION] Processing frame with comprehensive state data")
+    logger.info(f"[PERCEPTION] State: {state_summary}")
+    logger.info(f"[PERCEPTION] State context length: {len(state_context)} characters")
+    
+    perception_prompt = f"""
+    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
+    
+    {state_context}
+    
+    ★★★ VISUAL ANALYSIS TASK ★★★
+    
+    You are the agent, actively playing Pokemon Emerald. Observe and describe your current situation in detail using both the visual frame and the comprehensive game state data above.
+
+    Based on the visual frame and the above state data, describe your current situation:
+    - CUTSCENE or TITLE SCREEN: What does the cutscene or title screen show?
+    - MAP: You are navigating a terrain (city, forest, grassland, etc.). Are there any interactable locations (NPCs, items, doors)? What are the traversable vs. non-traversable areas? Use your position coordinates to understand where you are.
+    - BATTLE: Analyze the battle situation using both visual and state data. What moves are available? What's the strategy?
+    - DIALOGUE: What is the character telling you? How important is this information? Can you respond to the NPC?
+    - MENU: What menu are you in? What options are available? What should you select based on your current needs?
+    
+    Combine visual observation with the state data to give a complete picture of the current situation.
+    """
+    
+    observation = vlm.get_query(frame, system_prompt + perception_prompt, "PERCEPTION")
+    
+    # Determine if slow thinking is needed based on visual scene and state changes
+    scene_check_prompt = f"""
+    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
+    
+    {state_context}
+    
+    ★★★ SLOW THINKING DECISION ★★★
+    
+    Based on the current state and visual frame above:
+    
+    Does this scene represent a significant change that requires planning? Consider:
+    - Entering/exiting battle
+    - Reaching a new map/location
+    - Encountering important NPCs or story events
+    - Significant changes in pokemon party or game state
+    
+    Answer YES or NO.
+    """
+    scene_response = vlm.get_query(frame, scene_check_prompt, "PERCEPTION-SCENE_CHECK").strip().lower()
+    slow_thinking_needed = ("yes" in scene_response)
+
+    observation = {"description": observation, "state_data": state_context}
+    
+    logger.info(f"[PERCEPTION] Slow thinking needed: {slow_thinking_needed}")
+    return observation, slow_thinking_needed 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/planning.py

@@ -0,0 +1,96 @@
+import logging
+from utils.vlm import VLM
+from utils.state_formatter import format_state_for_llm, format_state_summary
+from agent.system_prompt import system_prompt
+
+# Set up module logging
+logger = logging.getLogger(__name__)
+
+def planning_step(memory_context, current_plan, slow_thinking_needed, state_data, vlm):
+    """
+    Decide and update your high-level plan based on memory context, current state, and the need for slow thinking.
+    Returns updated plan.
+    """
+    # Get formatted state context
+    state_context = format_state_for_llm(state_data)
+    state_summary = format_state_summary(state_data)
+    
+    logger.info("[PLANNING] Starting planning step")
+    logger.info(f"[PLANNING] State: {state_summary}")
+    logger.info(f"[PLANNING] Slow thinking needed: {slow_thinking_needed}")
+    
+    # Check if current plan is accomplished
+    if current_plan:
+        plan_check_prompt = f"""
+        ★★★ COMPREHENSIVE GAME STATE DATA ★★★
+        
+        {state_context}
+        
+        ★★★ PLAN ASSESSMENT TASK ★★★
+        
+        You are the agent playing Pokemon Emerald. Assess your current situation and plan progress.
+        
+        Current Plan: {current_plan}
+        Memory Context: {memory_context}
+        
+        Considering your current location, pokemon party, money, traversability, and recent actions:
+        Have you accomplished your current plan? Answer YES or NO, and explain briefly.
+        
+        Consider these factors:
+        - Did you reach your target location?
+        - Did you complete the intended battle/gym challenge?
+        - Did you acquire the needed pokemon/items?
+        - Are you stuck due to terrain or party status?
+        - Do you need to adapt due to wild encounters or water obstacles?
+        """
+        plan_status = vlm.get_text_query(system_prompt + plan_check_prompt, "PLANNING-ASSESSMENT")
+        if "yes" in plan_status.lower():
+            current_plan = None
+            logger.info("[PLANNING] Current plan marked as completed")
+    
+    # Generate new plan if needed
+    if current_plan is None or slow_thinking_needed:
+        planning_prompt = f"""
+        ★★★ COMPREHENSIVE GAME STATE DATA ★★★
+        
+        {state_context}
+        
+        ★★★ STRATEGIC PLANNING TASK ★★★
+        
+        You are the agent playing Pokemon Emerald with a speedrunning mindset. Create an efficient strategic plan.
+        
+        Memory Context: {memory_context}
+        
+        Analyze your situation and create a strategic plan:
+        
+        1. IMMEDIATE GOAL: What should you focus on right now? Consider:
+           - If in battle: What's your battle strategy based on pokemon HP/levels?
+           - If on map: Navigate efficiently using traversability data
+           - If in menu/dialogue: How to progress efficiently?
+           - Do you need to heal pokemon at Pokemon Center?
+           - Are there terrain obstacles (water, blocked paths) to navigate?
+        
+        2. SHORT-TERM OBJECTIVES (next few actions):
+           - Specific steps to achieve your immediate goal
+           - Account for your current pokemon party health and levels
+           - Consider terrain: avoid/seek tall grass, navigate around obstacles
+           - Money management for items/healing
+        
+        3. LONG-TERM STRATEGY:
+           - How does this fit into beating the game quickly?
+           - What gym leader or major milestone to target next?
+           - Pokemon catching/training priorities based on current party
+           - Route optimization considering terrain types
+        
+        4. EFFICIENCY NOTES:
+           - How to minimize backtracking using map layout
+           - Shortcuts or sequence breaks considering terrain
+           - Wild encounter management (avoid/seek based on needs)
+        
+        Format as a clear, actionable plan focusing on speed and efficiency.
+        """
+        current_plan = vlm.get_text_query(system_prompt + planning_prompt, "PLANNING-CREATION")
+        logger.info("[PLANNING] New plan created")
+    
+    logger.info(f"[PLANNING] Final plan: {current_plan[:300]}..." if len(current_plan) > 300 else f"[PLANNING] Final plan: {current_plan}")
+    return current_plan