synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/__init__.py

@@ -1,120 +0,0 @@
-"""
-Agent modules for Pokemon Emerald speedrunning agent
-"""
-
-from utils.vlm import VLM
-from .action import action_step
-from .memory import memory_step
-from .perception import perception_step
-from .planning import planning_step
-from .simple import SimpleAgent, get_simple_agent, simple_mode_processing_multiprocess, configure_simple_agent_defaults
-
-
-class Agent:
-    """
-    Unified agent interface that encapsulates all agent logic.
-    The client just calls agent.step(game_state) and gets back an action.
-    """
-    
-    def __init__(self, args=None):
-        """
-        Initialize the agent based on configuration.
-        
-        Args:
-            args: Command line arguments with agent configuration
-        """
-        # Extract configuration
-        backend = args.backend if args else "gemini"
-        model_name = args.model_name if args else "gemini-2.5-flash"
-        simple_mode = args.simple if args else False
-        
-        # Initialize VLM
-        self.vlm = VLM(backend=backend, model_name=model_name)
-        print(f"   VLM: {backend}/{model_name}")
-        
-        # Initialize agent mode
-        self.simple_mode = simple_mode
-        if simple_mode:
-            # Use global SimpleAgent instance to enable checkpoint persistence
-            self.simple_agent = get_simple_agent(self.vlm)
-            print(f"   Mode: Simple (direct frame->action)")
-        else:
-            # Four-module agent context
-            self.context = {
-                'perception_output': None,
-                'planning_output': None,
-                'memory': []
-            }
-            print(f"   Mode: Four-module architecture")
-    
-    def step(self, game_state):
-        """
-        Process a game state and return an action.
-        
-        Args:
-            game_state: Dictionary containing:
-                - screenshot: PIL Image
-                - game_state: Dict with game memory data
-                - visual: Dict with visual observations
-                - audio: Dict with audio observations
-                - progress: Dict with milestone progress
-        
-        Returns:
-            dict: Contains 'action' and optionally 'reasoning'
-        """
-        if self.simple_mode:
-            # Simple mode - delegate to SimpleAgent
-            return self.simple_agent.step(game_state)
-        else:
-            # Four-module processing
-            try:
-                # 1. Perception - understand what's happening
-                perception_output = perception_step(
-                    self.vlm, 
-                    game_state, 
-                    self.context.get('memory', [])
-                )
-                self.context['perception_output'] = perception_output
-                
-                # 2. Planning - decide strategy
-                planning_output = planning_step(
-                    self.vlm, 
-                    perception_output, 
-                    self.context.get('memory', [])
-                )
-                self.context['planning_output'] = planning_output
-                
-                # 3. Memory - update context
-                memory_output = memory_step(
-                    perception_output, 
-                    planning_output, 
-                    self.context.get('memory', [])
-                )
-                self.context['memory'] = memory_output
-                
-                # 4. Action - choose button press
-                action_output = action_step(
-                    self.vlm, 
-                    game_state, 
-                    planning_output,
-                    perception_output
-                )
-                
-                return action_output
-                
-            except Exception as e:
-                print(f"❌ Agent error: {e}")
-                return None
-
-
-__all__ = [
-    'Agent',
-    'action_step',
-    'memory_step', 
-    'perception_step',
-    'planning_step',
-    'SimpleAgent',
-    'get_simple_agent',
-    'simple_mode_processing_multiprocess',
-    'configure_simple_agent_defaults'
-]

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/action.py

@@ -1,160 +0,0 @@
-import logging
-import random
-import sys
-from agent.system_prompt import system_prompt
-from utils.state_formatter import format_state_for_llm, format_state_summary, get_movement_options, get_party_health_summary
-from utils.vlm import VLM
-
-# Set up module logging
-logger = logging.getLogger(__name__)
-
-def action_step(memory_context, current_plan, latest_observation, frame, state_data, recent_actions, vlm):
-    """
-    Decide and perform the next action button(s) based on memory, plan, observation, and comprehensive state.
-    Returns a list of action buttons as strings.
-    """
-    # Get formatted state context and useful summaries
-    state_context = format_state_for_llm(state_data)
-    state_summary = format_state_summary(state_data)
-    movement_options = get_movement_options(state_data)
-    party_health = get_party_health_summary(state_data)
-    
-    logger.info("[ACTION] Starting action decision")
-    logger.info(f"[ACTION] State: {state_summary}")
-    logger.info(f"[ACTION] Party health: {party_health['healthy_count']}/{party_health['total_count']} healthy")
-    if movement_options:
-        logger.info(f"[ACTION] Movement options: {movement_options}")
-    
-    # Build enhanced action context
-    action_context = []
-    
-    # Extract key info for context
-    game_data = state_data.get('game', {})
-    
-    # Battle vs Overworld context
-    if game_data.get('in_battle', False):
-        action_context.append("=== BATTLE MODE ===")
-        battle_info = game_data.get('battle_info', {})
-        if battle_info:
-            if 'player_pokemon' in battle_info:
-                player_pkmn = battle_info['player_pokemon']
-                action_context.append(f"Your Pokemon: {player_pkmn.get('species_name', player_pkmn.get('species', 'Unknown'))} (Lv.{player_pkmn.get('level', '?')}) HP: {player_pkmn.get('current_hp', '?')}/{player_pkmn.get('max_hp', '?')}")
-            if 'opponent_pokemon' in battle_info:
-                opp_pkmn = battle_info['opponent_pokemon']
-                action_context.append(f"Opponent: {opp_pkmn.get('species_name', opp_pkmn.get('species', 'Unknown'))} (Lv.{opp_pkmn.get('level', '?')}) HP: {opp_pkmn.get('current_hp', '?')}/{opp_pkmn.get('max_hp', '?')}")
-    else:
-        action_context.append("=== OVERWORLD MODE ===")
-        
-        # Movement options from utility
-        if movement_options:
-            action_context.append("Movement Options:")
-            for direction, description in movement_options.items():
-                action_context.append(f"  {direction}: {description}")
-    
-    # Party health summary
-    if party_health['total_count'] > 0:
-        action_context.append("=== PARTY STATUS ===")
-        action_context.append(f"Healthy Pokemon: {party_health['healthy_count']}/{party_health['total_count']}")
-        if party_health['critical_pokemon']:
-            action_context.append("Critical Pokemon:")
-            for critical in party_health['critical_pokemon']:
-                action_context.append(f"  {critical}")
-    
-    # Recent actions context
-    if recent_actions:
-        action_context.append(f"Recent Actions: {', '.join(list(recent_actions)[-5:])}")
-    
-    context_str = "\n".join(action_context)
-    
-    action_prompt = f"""
-    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
-    
-    {state_context}
-    
-    ★★★ ENHANCED ACTION CONTEXT ★★★
-    
-    {context_str}
-    
-    ★★★ ACTION DECISION TASK ★★★
-    
-    You are the agent playing Pokemon Emerald with a speedrunning mindset. Make quick, efficient decisions.
-    
-    Memory Context: {memory_context}
-    Current Plan: {current_plan if current_plan else 'No plan yet'}
-    Latest Observation: {latest_observation}
-    
-    Based on the comprehensive state information above, decide your next action(s):
-    
-    BATTLE STRATEGY:
-    - If in battle: Choose moves strategically based on type effectiveness and damage
-    - Consider switching pokemon if current one is weak/low HP
-    - Use items if pokemon is in critical condition
-    
-    NAVIGATION STRATEGY:
-    - Use movement options analysis above for efficient navigation
-    - Avoid blocked tiles (marked as BLOCKED)
-    - Consider tall grass: avoid if party is weak, seek if need to train/catch
-    - Navigate around water unless you have Surf
-    - Use coordinates to track progress toward objectives
-    
-    MENU/DIALOGUE STRATEGY:
-    - If in dialogue: A to advance text, B to cancel/skip if possible
-    - If in menu: Navigate with UP/DOWN/LEFT/RIGHT, A to select, B to cancel/back out
-    - If stuck in menu/interface: B repeatedly to exit to overworld
-    - In Pokemon Center: A to talk to Nurse Joy, A to confirm healing
-    
-    HEALTH MANAGEMENT:
-    - If pokemon are low HP/fainted, head to Pokemon Center
-    - If no healthy pokemon, prioritize healing immediately
-    - Consider terrain: avoid wild encounters if party is weak
-    
-    EFFICIENCY RULES:
-    1. Output sequences of actions when you know what's coming (e.g., "RIGHT, RIGHT, RIGHT, A" to enter a door)
-    2. For dialogue: "A, A, A, A, A" to mash through
-    3. For movement: repeat directions based on movement options (e.g., "UP, UP, UP, UP" if UP shows "Normal path")
-    4. If uncertain, output single action and reassess
-    5. Use traversability data: move toward open paths, avoid obstacles
-    6. If movement doesn't change coordinates (e.g., RIGHT but X doesn't increase), check map for walls (#) blocking your path
-    
-    Valid buttons: A, B, SELECT, START, UP, DOWN, LEFT, RIGHT, L, R
-    - A: Interact with NPCs/objects, confirm selections, advance dialogue, use moves in battle
-    - B: Cancel menus, back out of interfaces, run faster (with running shoes), flee from battle
-    - START: Open main menu (Title sequence, Pokedex, Pokemon, Bag, etc.)
-    - SELECT: Use registered key item (typically unused)
-    - UP/DOWN/LEFT/RIGHT: Move character, navigate menus, select options
-    - L/R: Cycle through pages in some menus, switch Pokemon in battle (rare usage)
-    
-    ⚠️ CRITICAL WARNING: NEVER save the game using the in-game save menu! Saving will crash the entire run and end your progress. If you encounter a save prompt in the game, press B to cancel it immediately!
-    
-    Return ONLY the button name(s) as a comma-separated list, nothing else.
-    Maximum 10 actions in sequence. Avoid repeating same button more than 6 times.
-    """
-    
-    # Construct complete prompt for VLM
-    complete_prompt = system_prompt + action_prompt
-    
-    action_response = vlm.get_text_query(complete_prompt, "ACTION").strip().upper()
-    valid_buttons = ['A', 'B', 'SELECT', 'START', 'UP', 'DOWN', 'LEFT', 'RIGHT', 'L', 'R']
-    
-    # Split the response by commas and clean up
-    actions = [btn.strip() for btn in action_response.split(',') if btn.strip() in valid_buttons]
-    
-    print(f"Parsed actions: {actions}")
-    if len(actions) == 0:
-        print("❌ No valid actions parsed - using default 'A'")
-    print("-" * 80 + "\n")
-    
-    # Limit to maximum 10 actions and prevent excessive repetition
-    actions = actions[:10]
-    
-    # If no valid actions found, make intelligent default based on state
-    if not actions:
-        if game_data.get('in_battle', False):
-            actions = ['A']  # Attack in battle
-        elif party_health['total_count'] == 0:
-            actions = ['A', 'A', 'A']  # Try to progress dialogue/menu
-        else:
-            actions = [random.choice(['A', 'RIGHT', 'UP', 'DOWN', 'LEFT'])]  # Random exploration
-    
-    logger.info(f"[ACTION] Actions decided: {', '.join(actions)}")
-    return actions 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/memory.py

@@ -1,155 +0,0 @@
-import logging
-from collections import deque
-from agent.system_prompt import system_prompt
-from utils.state_formatter import format_state_summary, get_party_health_summary
-from utils.vlm import VLM
-
-# Set up module logging
-logger = logging.getLogger(__name__)
-
-def extract_key_state_info(state_data):
-    """Extract key information from comprehensive state for memory storage using the utility functions"""
-    # Use the state formatter utilities for consistency
-    state_summary = format_state_summary(state_data)
-    party_health = get_party_health_summary(state_data)
-    
-    # Extract additional info
-    player_data = state_data.get('player', {})
-    game_data = state_data.get('game', {})
-    map_info = state_data.get('map', {})
-    
-    key_info = {
-        'state_summary': state_summary,
-        'player_name': player_data.get('name', 'Player'),
-        'money': player_data.get('money') or game_data.get('money', 0),
-        'current_map': player_data.get('location', 'Unknown Location'),
-        'in_battle': game_data.get('in_battle', False),
-        'party_health': f"{party_health['healthy_count']}/{party_health['total_count']}",
-        'critical_pokemon': party_health['critical_pokemon']
-    }
-    
-    # Position info
-    if 'coordinates' in player_data:
-        key_info['position'] = player_data['coordinates']
-    elif 'position' in player_data:
-        key_info['position'] = player_data['position']
-    else:
-        key_info['position'] = {}
-    
-    # Battle opponent
-    if game_data.get('battle_info'):
-        battle = game_data['battle_info']
-        opponent_pokemon = battle.get('opponent_pokemon', {})
-        key_info['battle_opponent'] = opponent_pokemon.get('species_name', opponent_pokemon.get('species', 'Unknown Pokemon'))
-    
-    # Traversability summary
-    if 'traversability' in map_info and map_info['traversability']:
-        traversability = map_info['traversability']
-        total_tiles = sum(len(row) for row in traversability)
-        blocked_count = sum(1 for row in traversability for cell in row if str(cell) in ['0', '0'])
-        passable_tiles = total_tiles - blocked_count
-        key_info['traversability_summary'] = f"{passable_tiles}/{total_tiles} passable"
-    else:
-        key_info['traversability_summary'] = "No data"
-    
-    return key_info
-
-def memory_step(memory_context, current_plan, recent_actions, observation_buffer, vlm):
-    """
-    Maintain a rolling buffer of the previous 50 actions and observations with state information.
-    Returns updated memory_context with the most recent 50 entries and key insights.
-    """
-    # Initialize memory buffer if it doesn't exist
-    if not hasattr(memory_step, 'memory_buffer'):
-        memory_step.memory_buffer = deque(maxlen=50)
-    
-    logger.info(f"[MEMORY] Processing {len(observation_buffer)} new observations")
-    
-    # Add new observations with state info to the buffer
-    for obs in observation_buffer:
-        state_info = extract_key_state_info(obs.get('state', {}))
-        memory_step.memory_buffer.append({
-            "type": "observation",
-            "frame_id": obs["frame_id"],
-            "content": obs["observation"],
-            "state": state_info
-        })
-        logger.info(f"[MEMORY] Added observation frame {obs['frame_id']}: {state_info['state_summary']}")
-    
-    # Add recent actions to the buffer
-    for action in recent_actions:
-        memory_step.memory_buffer.append({
-            "type": "action",
-            "content": action
-        })
-    
-    # Create a formatted memory context from the buffer with state insights
-    memory_entries = []
-    key_events = []
-    
-    # Track significant state changes
-    previous_map = None
-    previous_battle_state = None
-    
-    for i, entry in enumerate(memory_step.memory_buffer):
-        if entry["type"] == "observation":
-            frame_id = entry['frame_id']
-            description = entry['content']
-            state = entry.get('state', {})
-            
-            # Use the consistent state summary
-            state_summary = state.get('state_summary', '')
-            
-            # Check for significant events
-            current_map = state.get('current_map', 'Unknown Location')
-            current_battle = state.get('in_battle', False)
-            
-            if current_map != previous_map and previous_map is not None:
-                key_events.append(f"Moved from {previous_map} to {current_map}")
-                logger.info(f"[MEMORY] Key event: Map change from {previous_map} to {current_map}")
-            
-            if current_battle != previous_battle_state:
-                if current_battle:
-                    opponent = state.get('battle_opponent', 'Unknown Pokemon')
-                    key_events.append(f"Entered battle vs {opponent}")
-                    logger.info(f"[MEMORY] Key event: Entered battle vs {opponent}")
-                else:
-                    key_events.append("Exited battle")
-                    logger.info("[MEMORY] Key event: Exited battle")
-            
-            previous_map = current_map
-            previous_battle_state = current_battle
-            
-            # Format observation entry
-            if isinstance(description, dict):
-                desc_text = description.get('description', str(description))
-            else:
-                desc_text = str(description)
-            
-            memory_entries.append(f"Frame {frame_id}: {desc_text} [{state_summary}]")
-        else:
-            memory_entries.append(f"Action: {entry['content']}")
-    
-    # Get current state summary from the latest observation
-    current_state_summary = ""
-    if observation_buffer:
-        latest_state = extract_key_state_info(observation_buffer[-1].get('state', {}))
-        current_state_summary = latest_state.get('state_summary', 'No state data')
-    
-    # Combine into comprehensive memory context
-    memory_context = f"""★★★ COMPREHENSIVE MEMORY CONTEXT ★★★
-
-CURRENT STATE: {current_state_summary}
-
-CURRENT PLAN: {current_plan if current_plan else 'No plan yet'}
-
-KEY EVENTS: {' -> '.join(key_events[-5:]) if key_events else 'None recently'}
-
-RECENT MEMORY (last 50 entries):
-{chr(10).join(memory_entries[-30:])}"""  # Show last 30 entries to avoid too much text
-    
-    logger.info(f"[MEMORY] Memory context updated with {len(memory_entries)} total entries")
-    logger.info(f"[MEMORY] Current state: {current_state_summary}")
-    logger.info(f"[MEMORY] Key events: {len(key_events)} tracked")
-    
-    return memory_context 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/perception.py

@@ -1,69 +0,0 @@
-import time
-import logging
-from utils.vlm import VLM
-from utils.state_formatter import format_state_for_llm, format_state_summary
-from agent.system_prompt import system_prompt
-
-# Set up module logging
-logger = logging.getLogger(__name__)
-
-def perception_step(frame, state_data, vlm):
-    """
-    Observe and describe your current situation using both visual and comprehensive state data.
-    Returns (observation, slow_thinking_needed)
-    """
-    # Format the comprehensive state context using the utility
-    state_context = format_state_for_llm(state_data)
-    
-    # Log the state data being used
-    state_summary = format_state_summary(state_data)
-    logger.info("[PERCEPTION] Processing frame with comprehensive state data")
-    logger.info(f"[PERCEPTION] State: {state_summary}")
-    logger.info(f"[PERCEPTION] State context length: {len(state_context)} characters")
-    
-    perception_prompt = f"""
-    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
-    
-    {state_context}
-    
-    ★★★ VISUAL ANALYSIS TASK ★★★
-    
-    You are the agent, actively playing Pokemon Emerald. Observe and describe your current situation in detail using both the visual frame and the comprehensive game state data above.
-
-    Based on the visual frame and the above state data, describe your current situation:
-    - CUTSCENE or TITLE SCREEN: What does the cutscene or title screen show?
-    - MAP: You are navigating a terrain (city, forest, grassland, etc.). Are there any interactable locations (NPCs, items, doors)? What are the traversable vs. non-traversable areas? Use your position coordinates to understand where you are.
-    - BATTLE: Analyze the battle situation using both visual and state data. What moves are available? What's the strategy?
-    - DIALOGUE: What is the character telling you? How important is this information? Can you respond to the NPC?
-    - MENU: What menu are you in? What options are available? What should you select based on your current needs?
-    
-    Combine visual observation with the state data to give a complete picture of the current situation.
-    """
-    
-    observation = vlm.get_query(frame, system_prompt + perception_prompt, "PERCEPTION")
-    
-    # Determine if slow thinking is needed based on visual scene and state changes
-    scene_check_prompt = f"""
-    ★★★ COMPREHENSIVE GAME STATE DATA ★★★
-    
-    {state_context}
-    
-    ★★★ SLOW THINKING DECISION ★★★
-    
-    Based on the current state and visual frame above:
-    
-    Does this scene represent a significant change that requires planning? Consider:
-    - Entering/exiting battle
-    - Reaching a new map/location
-    - Encountering important NPCs or story events
-    - Significant changes in pokemon party or game state
-    
-    Answer YES or NO.
-    """
-    scene_response = vlm.get_query(frame, scene_check_prompt, "PERCEPTION-SCENE_CHECK").strip().lower()
-    slow_thinking_needed = ("yes" in scene_response)
-
-    observation = {"description": observation, "state_data": state_context}
-    
-    logger.info(f"[PERCEPTION] Slow thinking needed: {slow_thinking_needed}")
-    return observation, slow_thinking_needed 

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/planning.py

@@ -1,96 +0,0 @@
-import logging
-from utils.vlm import VLM
-from utils.state_formatter import format_state_for_llm, format_state_summary
-from agent.system_prompt import system_prompt
-
-# Set up module logging
-logger = logging.getLogger(__name__)
-
-def planning_step(memory_context, current_plan, slow_thinking_needed, state_data, vlm):
-    """
-    Decide and update your high-level plan based on memory context, current state, and the need for slow thinking.
-    Returns updated plan.
-    """
-    # Get formatted state context
-    state_context = format_state_for_llm(state_data)
-    state_summary = format_state_summary(state_data)
-    
-    logger.info("[PLANNING] Starting planning step")
-    logger.info(f"[PLANNING] State: {state_summary}")
-    logger.info(f"[PLANNING] Slow thinking needed: {slow_thinking_needed}")
-    
-    # Check if current plan is accomplished
-    if current_plan:
-        plan_check_prompt = f"""
-        ★★★ COMPREHENSIVE GAME STATE DATA ★★★
-        
-        {state_context}
-        
-        ★★★ PLAN ASSESSMENT TASK ★★★
-        
-        You are the agent playing Pokemon Emerald. Assess your current situation and plan progress.
-        
-        Current Plan: {current_plan}
-        Memory Context: {memory_context}
-        
-        Considering your current location, pokemon party, money, traversability, and recent actions:
-        Have you accomplished your current plan? Answer YES or NO, and explain briefly.
-        
-        Consider these factors:
-        - Did you reach your target location?
-        - Did you complete the intended battle/gym challenge?
-        - Did you acquire the needed pokemon/items?
-        - Are you stuck due to terrain or party status?
-        - Do you need to adapt due to wild encounters or water obstacles?
-        """
-        plan_status = vlm.get_text_query(system_prompt + plan_check_prompt, "PLANNING-ASSESSMENT")
-        if "yes" in plan_status.lower():
-            current_plan = None
-            logger.info("[PLANNING] Current plan marked as completed")
-    
-    # Generate new plan if needed
-    if current_plan is None or slow_thinking_needed:
-        planning_prompt = f"""
-        ★★★ COMPREHENSIVE GAME STATE DATA ★★★
-        
-        {state_context}
-        
-        ★★★ STRATEGIC PLANNING TASK ★★★
-        
-        You are the agent playing Pokemon Emerald with a speedrunning mindset. Create an efficient strategic plan.
-        
-        Memory Context: {memory_context}
-        
-        Analyze your situation and create a strategic plan:
-        
-        1. IMMEDIATE GOAL: What should you focus on right now? Consider:
-           - If in battle: What's your battle strategy based on pokemon HP/levels?
-           - If on map: Navigate efficiently using traversability data
-           - If in menu/dialogue: How to progress efficiently?
-           - Do you need to heal pokemon at Pokemon Center?
-           - Are there terrain obstacles (water, blocked paths) to navigate?
-        
-        2. SHORT-TERM OBJECTIVES (next few actions):
-           - Specific steps to achieve your immediate goal
-           - Account for your current pokemon party health and levels
-           - Consider terrain: avoid/seek tall grass, navigate around obstacles
-           - Money management for items/healing
-        
-        3. LONG-TERM STRATEGY:
-           - How does this fit into beating the game quickly?
-           - What gym leader or major milestone to target next?
-           - Pokemon catching/training priorities based on current party
-           - Route optimization considering terrain types
-        
-        4. EFFICIENCY NOTES:
-           - How to minimize backtracking using map layout
-           - Shortcuts or sequence breaks considering terrain
-           - Wild encounter management (avoid/seek based on needs)
-        
-        Format as a clear, actionable plan focusing on speed and efficiency.
-        """
-        current_plan = vlm.get_text_query(system_prompt + planning_prompt, "PLANNING-CREATION")
-        logger.info("[PLANNING] New plan created")
-    
-    logger.info(f"[PLANNING] Final plan: {current_plan[:300]}..." if len(current_plan) > 300 else f"[PLANNING] Final plan: {current_plan}")
-    return current_plan