droidrun 0.3.9__py3-none-any.whl → 0.3.10.dev3__py3-none-any.whl

droidrun/agent/droid/events.py

@@ -1,19 +1,19 @@
+from typing import Dict, List
+
 from llama_index.core.workflow import Event
-from droidrun.agent.context import Reflection, Task
-from typing import List, Optional
+from pydantic import BaseModel, Field
+
+from droidrun.agent.context import Task
+
 
 class CodeActExecuteEvent(Event):
     task: Task
-    reflection: Optional[Reflection]
 
 class CodeActResultEvent(Event):
     success: bool
     reason: str
     steps: int
 
-class ReasoningLogicEvent(Event):
-    reflection: Optional[Reflection] = None
-    force_planning: bool = False
 
 class FinalizeEvent(Event):
     success: bool
@@ -28,6 +28,88 @@ class FinalizeEvent(Event):
 class TaskRunnerEvent(Event):
     pass
 
-class ReflectionEvent(Event):
-    task: Task
-    pass
+
+
+# ============================================================================
+# DroidAgentState - State model for llama-index Context
+# ============================================================================
+
+class DroidAgentState(BaseModel):
+    """
+    State model for DroidAgent workflow - shared across parent and child workflows.
+    """
+
+    # Task context
+    instruction: str = ""
+
+    # UI State
+    ui_elements_list_before: str = ""
+    ui_elements_list_after: str = ""
+    focused_text: str = ""
+    device_state_text: str = ""
+    width: int = 0
+    height: int = 0
+    screenshot: str | bytes | None = None
+    has_text_to_modify: bool = False
+
+    # Action tracking
+    action_pool: List[Dict] = Field(default_factory=list)
+    action_history: List[Dict] = Field(default_factory=list)
+    summary_history: List[str] = Field(default_factory=list)
+    action_outcomes: List[str] = Field(default_factory=list)  # "A", "B", "C"
+    error_descriptions: List[str] = Field(default_factory=list)
+
+    # Last action info
+    last_action: Dict = Field(default_factory=dict)
+    last_summary: str = ""
+    last_action_thought: str = ""
+
+    # Memory
+    memory: str = ""
+    message_history: List[Dict] = Field(default_factory=list)
+
+    # Planning
+    plan: str = ""
+    completed_plan: str = ""
+    current_subgoal: str = ""
+    finish_thought: str = ""
+    progress_status: str = ""
+    manager_answer: str = ""  # For answer-type tasks
+
+    # Error handling
+    error_flag_plan: bool = False
+    err_to_manager_thresh: int = 2
+
+    # Output
+    output_dir: str = ""
+
+
+# ============================================================================
+# Manager/Executor coordination events
+# ============================================================================
+
+class ManagerInputEvent(Event):
+    """Trigger Manager workflow for planning"""
+    pass
+
+
+class ManagerPlanEvent(Event):
+    """Manager has created a plan"""
+    plan: str
+    current_subgoal: str
+    completed_plan: str
+    thought: str
+    manager_answer: str = ""
+
+
+class ExecutorInputEvent(Event):
+    """Trigger Executor workflow for action execution"""
+    current_subgoal: str
+
+
+class ExecutorResultEvent(Event):
+    """Executor action result"""
+    action: Dict
+    outcome: bool
+    error: str
+    summary: str

droidrun/agent/executor/__init__.py

@@ -0,0 +1,13 @@
+"""
+Executor Agent - Action execution workflow.
+"""
+
+from droidrun.agent.executor.events import ExecutorActionEvent, ExecutorResultEvent
+from droidrun.agent.executor.executor_agent import ExecutorAgent
+
+__all__ = [
+    "ExecutorAgent",
+    "ExecutorThinkingEvent",
+    "ExecutorActionEvent",
+    "ExecutorResultEvent"
+]

droidrun/agent/executor/events.py

@@ -0,0 +1,24 @@
+"""
+Events for the ExecutorAgent workflow.
+"""
+
+from typing import Dict
+
+from llama_index.core.workflow.events import Event
+
+
+class ExecutorActionEvent(Event):
+    """Executor has selected an action to execute"""
+    action_json: str
+    thought: str
+    description: str
+
+
+class ExecutorResultEvent(Event):
+    """Executor action result"""
+    action: Dict
+    outcome: bool
+    error: str
+    summary: str
+    thought: str = ""
+    action_json: str = ""

droidrun/agent/executor/executor_agent.py

@@ -0,0 +1,327 @@
+"""
+ExecutorAgent - Action execution workflow.
+
+This agent is responsible for:
+- Taking a specific subgoal from the Manager
+- Analyzing the current UI state
+- Selecting and executing appropriate actions
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING
+
+from llama_index.core.llms import ChatMessage, ImageBlock, TextBlock
+from llama_index.core.llms.llm import LLM
+from llama_index.core.workflow import Context, StartEvent, StopEvent, Workflow, step
+
+from droidrun.agent.executor.events import ExecutorActionEvent, ExecutorResultEvent
+from droidrun.agent.executor.prompts import build_executor_system_prompt, parse_executor_response
+from droidrun.agent.utils.tools import click, long_press, open_app, swipe, system_button, type
+from droidrun.agent.utils.inference import acall_with_retries
+from droidrun.config_manager import config
+import asyncio
+
+if TYPE_CHECKING:
+    from droidrun.agent.droid.events import DroidAgentState
+
+logger = logging.getLogger("droidrun")
+
+
+class ExecutorAgent(Workflow):
+    """
+    Action execution agent that performs specific actions.
+
+    The Executor:
+    1. Receives a subgoal from the Manager
+    2. Analyzes current UI state and context
+    3. Selects an appropriate action to take
+    4. Executes the action on the device
+    5. Reports the outcome
+    """
+
+    def __init__(
+        self,
+        llm: LLM,
+        vision: bool,
+        tools_instance,
+        shared_state: "DroidAgentState",
+        persona=None,
+        custom_tools: dict = None,
+        debug: bool = False,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.llm = llm
+        self.vision = vision
+        self.tools_instance = tools_instance
+        self.shared_state = shared_state
+        self.persona = persona
+        self.custom_tools = custom_tools or {}
+        self.debug = debug
+
+        logger.info("✅ ExecutorAgent initialized successfully.")
+
+
+    @step
+    async def think(
+        self,
+        ctx: Context,
+        ev: StartEvent
+    ) -> ExecutorActionEvent:
+        """
+        Executor decides which action to take.
+
+        This step:
+        1. Calls LLM with executor prompt and context
+        2. Parses the response for action, thought, description
+        3. Validates action format (blocks answer actions!)
+        4. Returns action event
+        """
+        subgoal = ev.get("subgoal", "")
+        logger.info(f"🧠 Executor thinking about action for: {subgoal}")
+
+        app_card = ""  # TODO: Implement app card retrieval
+
+        system_prompt = build_executor_system_prompt(
+            state=self.shared_state,
+            subgoal=subgoal,
+            app_card=app_card
+        )
+
+        blocks = [TextBlock(text=system_prompt)]
+        if self.vision:
+            screenshot = self.shared_state.screenshot
+            if screenshot is not None:
+                blocks.append(ImageBlock(image=screenshot))
+                logger.debug("📸 Using screenshot for Executor")
+            else:
+                logger.warning("⚠️ Vision enabled but no screenshot available")
+        messages = [ChatMessage(role="user", blocks=blocks)]
+
+        try:
+            response = await acall_with_retries(self.llm, messages)
+            response_text = str(response)
+        except Exception as e:
+            raise RuntimeError(f"Error calling LLM in executor: {e}") from e
+
+        # Parse response
+        try:
+            parsed = parse_executor_response(response_text)
+        except Exception as e:
+            logger.error(f"❌ Failed to parse executor response: {e}")
+            return ExecutorActionEvent(
+                action_json=json.dumps({"action": "invalid"}),
+                thought=f"Failed to parse response: {str(e)}",
+                description="Invalid response format from LLM"
+            )
+
+        logger.info(f"💡 Thought: {parsed['thought']}")
+        logger.info(f"🎯 Action: {parsed['action']}")
+        logger.debug(f"  - Description: {parsed['description']}")
+
+        return ExecutorActionEvent(
+            action_json=parsed["action"],
+            thought=parsed["thought"],
+            description=parsed["description"]
+        )
+
+    @step
+    async def execute(
+        self,
+        ctx: Context,
+        ev: ExecutorActionEvent
+    ) -> ExecutorResultEvent:
+        """
+        Execute the selected action using the tools instance.
+
+        Maps action JSON to appropriate tool calls and handles execution.
+        """
+        logger.info(f"⚡ Executing action: {ev.description}")
+
+        # Parse action JSON
+        try:
+            action_dict = json.loads(ev.action_json)
+        except json.JSONDecodeError as e:
+            logger.error(f"❌ Failed to parse action JSON: {e}")
+            return ExecutorResultEvent(
+                action={"action": "invalid"},
+                outcome=False,
+                error=f"Invalid action JSON: {str(e)}",
+                summary="Failed to parse action",
+                thought=ev.thought,
+                action_json=ev.action_json
+            )
+
+        # Execute the action
+        outcome, error, summary = await self._execute_action(action_dict, ev.description)
+
+        if outcome:
+            await asyncio.sleep(config.agent.after_sleep_action)
+
+        logger.info(f"{'✅' if outcome else '❌'} Execution complete: {summary}")
+
+        return ExecutorResultEvent(
+            action=action_dict,
+            outcome=outcome,
+            error=error,
+            summary=summary,
+            thought=ev.thought,
+            action_json=ev.action_json
+        )
+
+    async def _execute_action(self, action_dict: dict, description: str) -> tuple[bool, str, str]:
+        """
+        Execute a single action based on the action dictionary.
+
+        Args:
+            action_dict: Dictionary containing action type and parameters
+            description: Human-readable description of the action
+
+        Returns:
+            Tuple of (outcome: bool, error: str, summary: str)
+        """
+
+        action_type = action_dict.get("action", "unknown")
+
+        # Check custom_tools first (before atomic actions)
+        if action_type in self.custom_tools:
+            return await self._execute_custom_tool(action_type, action_dict)
+
+        try:
+            if action_type == "click":
+                index = action_dict.get("index")
+                if index is None:
+                    return False, "Missing 'index' parameter", "Failed: click requires index"
+
+                result = click(self.tools_instance, index)
+                return True, "None", f"Clicked element at index {index}"
+
+            elif action_type == "long_press":
+                index = action_dict.get("index")
+                if index is None:
+                    return False, "Missing 'index' parameter", "Failed: long_press requires index"
+
+                success = long_press(self.tools_instance, index)
+                if success:
+                    return True, "None", f"Long pressed element at index {index}"
+                else:
+                    return False, "Long press failed", f"Failed to long press at index {index}"
+
+            elif action_type == "type":
+                text = action_dict.get("text")
+                index = action_dict.get("index", -1)
+
+                if text is None:
+                    return False, "Missing 'text' parameter", "Failed: type requires text"
+
+                result = type(self.tools_instance, text, index)
+                return True, "None", f"Typed '{text}' into element at index {index}"
+
+            elif action_type == "system_button":
+                button = action_dict.get("button")
+                if button is None:
+                    return False, "Missing 'button' parameter", "Failed: system_button requires button"
+
+                result = system_button(self.tools_instance, button)
+                if "Error" in result:
+                    return False, result, f"Failed to press {button} button"
+                return True, "None", f"Pressed {button} button"
+
+            elif action_type == "swipe":
+                coordinate = action_dict.get("coordinate")
+                coordinate2 = action_dict.get("coordinate2")
+
+                if coordinate is None or coordinate2 is None:
+                    return False, "Missing coordinate parameters", "Failed: swipe requires coordinate and coordinate2"
+
+                # Validate coordinate format before calling swipe
+                if not isinstance(coordinate, list) or len(coordinate) != 2:
+                    return False, f"Invalid coordinate format: {coordinate}", "Failed: coordinate must be [x, y]"
+                if not isinstance(coordinate2, list) or len(coordinate2) != 2:
+                    return False, f"Invalid coordinate2 format: {coordinate2}", "Failed: coordinate2 must be [x, y]"
+
+                success = swipe(self.tools_instance, coordinate, coordinate2)
+                if success:
+                    return True, "None", f"Swiped from {coordinate} to {coordinate2}"
+                else:
+                    return False, "Swipe failed", f"Failed to swipe from {coordinate} to {coordinate2}"
+
+            elif action_type == "open_app":
+                text = action_dict.get("text")
+                if text is None:
+                    return False, "Missing 'text' parameter", "Failed: open_app requires text"
+
+                result = open_app(self.tools_instance, text)
+                return True, "None", f"Opened app: {text}"
+
+            else:
+                return False, f"Unknown action type: {action_type}", f"Failed: unknown action '{action_type}'"
+
+        except Exception as e:
+            logger.error(f"❌ Exception during action execution: {e}", exc_info=True)
+            return False, f"Exception: {str(e)}", f"Failed to execute {action_type}: {str(e)}"
+
+    async def _execute_custom_tool(self, action_type: str, action_dict: dict) -> tuple[bool, str, str]:
+        """
+        Execute a custom tool based on the action dictionary.
+
+        Args:
+            action_type: The custom tool name
+            action_dict: Dictionary containing action parameters
+
+        Returns:
+            Tuple of (outcome: bool, error: str, summary: str)
+        """
+        try:
+            tool_spec = self.custom_tools[action_type]
+            tool_func = tool_spec["function"]
+
+            # Extract arguments (exclude 'action' key)
+            tool_args = {k: v for k, v in action_dict.items() if k != "action"}
+
+            # Execute the custom tool function
+            # First argument is always tools_instance (bound in same pattern as atomic actions)
+            if asyncio.iscoroutinefunction(tool_func):
+                result = await tool_func(self.tools_instance, **tool_args)
+            else:
+                result = tool_func(self.tools_instance, **tool_args)
+
+            # Success case
+            summary = f"Executed custom tool '{action_type}'"
+            if result is not None:
+                summary += f": {str(result)}"
+
+            return True, "None", summary
+
+        except TypeError as e:
+            # Likely missing or wrong arguments
+            error_msg = f"Invalid arguments for custom tool '{action_type}': {str(e)}"
+            logger.error(f"❌ {error_msg}")
+            return False, error_msg, f"Failed: {action_type}"
+
+        except Exception as e:
+            # General execution error
+            error_msg = f"Error executing custom tool '{action_type}': {str(e)}"
+            logger.error(f"❌ {error_msg}", exc_info=True)
+            return False, error_msg, f"Failed: {action_type}"
+
+    @step
+    async def finalize(
+        self,
+        ctx: Context,
+        ev: ExecutorResultEvent
+    ) -> StopEvent:
+        """Return executor results to parent workflow."""
+        logger.debug("✅ Executor execution complete")
+
+        return StopEvent(result={
+            "action": ev.action,
+            "outcome": ev.outcome,
+            "error": ev.error,
+            "summary": ev.summary,
+            "thought": ev.thought,
+            "action_json": ev.action_json
+        })

droidrun/agent/executor/prompts.py

@@ -0,0 +1,136 @@
+"""
+Prompts for the ExecutorAgent.
+"""
+
+
+from droidrun.agent.droid.events import DroidAgentState
+from droidrun.agent.utils.tools import ATOMIC_ACTION_SIGNATURES
+
+
+def build_executor_system_prompt(
+    state: "DroidAgentState",
+    subgoal: str,
+    app_card: str = "",
+) -> str:
+    """
+    Build the complete Executor system prompt with all context.
+
+    Args:
+        state: Current DroidAgentState with all context
+        subgoal: Current subgoal to execute
+        app_card: Optional app-specific instructions
+
+    Returns:
+        Complete system prompt for the Executor
+    """
+    prompt = f"""You are a LOW-LEVEL ACTION EXECUTOR for an Android phone. You do NOT answer questions or provide results. You ONLY perform individual atomic actions as specified in the current subgoal. You are part of a larger system - your job is to execute actions, not to think about or answer the user's original question.
+
+### User Request ###
+{state.instruction}
+
+{("App card gives information on how to operate the app and perform actions.\n" + "### App Card ###\n" + app_card.strip() + "\n\n") if app_card.strip() else ""}{(("### Device State ###\n" + state.device_state_text.strip() + "\n\n") if state.device_state_text.strip() else "")}### Overall Plan ###
+{state.plan}
+
+### Current Subgoal ###
+EXECUTE THIS SUBGOAL: {subgoal}
+
+EXECUTION MODE: You are a dumb robot. Find the exact text/element mentioned in the subgoal above and perform the specified action on it. Do not read anything below this line until after you execute the subgoal.
+
+### SUBGOAL PARSING MODE ###
+Read the current subgoal exactly as written. Look for:
+- Action words: "tap", "click", "swipe", "type", "press", "open" etc.
+- Target elements: specific text, buttons, fields, coordinates mentioned
+- Locations: "header", "bottom", "left", "right", specific coordinates
+Convert directly to atomic action:
+- "tap/click" → click action
+- "swipe" → swipe action
+- "type" → type action
+- "press [system button]" → system_button action
+- "open [app]" → open_app action
+Execute the atomic action for the exact target mentioned. Ignore everything else.
+
+### Progress Status ###
+{(state.progress_status + "\n\n") if state.progress_status != "" else "No progress yet.\n\n"}
+
+### Guidelines ###
+General:
+- For any pop-up window, such as a permission request, you need to close it (e.g., by clicking `Don't Allow` or `Accept & continue`) before proceeding. Never choose to add any account or log in.
+Action Related:
+- Use the `open_app` action whenever you want to open an app (nothing will happen if the app is not installed), do not use the app drawer to open an app.
+- Consider exploring the screen by using the `swipe` action with different directions to reveal additional content. Or use search to quickly find a specific entry, if applicable.
+- If you cannot change the page content by swiping in the same direction continuously, the page may have been swiped to the bottom. Please try another operation to display more content.
+- For some horizontally distributed tags, you can swipe horizontally to view more.
+Text Related Operations:
+- Activated input box: If an input box is activated, it may have a cursor inside it and the keyboard is visible. If there is no cursor on the screen but the keyboard is visible, it may be because the cursor is blinking. The color of the activated input box will be highlighted. If you are not sure whether the input box is activated, click it before typing.
+- To input some text: first click the input box that you want to input, make sure the correct input box is activated and the keyboard is visible, then use `type` action to enter the specified text.
+- To clear the text: long press the backspace button in the keyboard.
+- To copy some text: first long press the text you want to copy, then click the `copy` button in bar.
+- To paste text into a text box: first long press the text box, then click the `paste` button in bar.
+
+---
+Execute the current subgoal mechanically. Do NOT examine the screen content or make decisions about what you see. Parse the current subgoal text to identify the required action and execute it exactly as written. You must choose your action from one of the atomic actions.
+
+#### Atomic Actions ####
+The atomic action functions are listed in the format of `action(arguments): description` as follows:
+{chr(10).join(f"- {action_name}({', '.join(action_info['arguments'])}): {action_info['description']}" for action_name, action_info in ATOMIC_ACTION_SIGNATURES.items())}
+\n
+### Latest Action History ###
+{(("Recent actions you took previously and whether they were successful:\n" + "\n".join(
+    (f"Action: {act} | Description: {summ} | Outcome: Successful" if outcome == "A"
+     else f"Action: {act} | Description: {summ} | Outcome: Failed | Feedback: {err_des}")
+    for act, summ, outcome, err_des in zip(
+        state.action_history[-min(5, len(state.action_history)):],
+        state.summary_history[-min(5, len(state.action_history)):],
+        state.action_outcomes[-min(5, len(state.action_history)):],
+        state.error_descriptions[-min(5, len(state.action_history)):], strict=True)
+) + "\n\n")) if state.action_history else "No actions have been taken yet.\n\n"}
+
+---
+### LITERAL EXECUTION RULE ###
+Whatever the current subgoal says to do, do that EXACTLY. Do not substitute with what you think is better. Do not optimize. Do not consider screen state. Parse the subgoal text literally and execute the matching atomic action.
+
+IMPORTANT:
+1. Do NOT repeat previously failed actions multiple times. Try changing to another action.
+2. Must do the current subgoal.
+
+Provide your output in the following format, which contains three parts:
+
+### Thought ###
+Break down the current subgoal into: (1) What atomic action is required? (2) What target/location is specified? (3) What parameters do I need? Do NOT reason about whether this makes sense - just mechanically convert the subgoal text into the appropriate action format.
+
+### Action ###
+Choose only one action or shortcut from the options provided.
+You must provide your decision using a valid JSON format specifying the `action` and the arguments of the action. For example, if you want to open an App, you should write {{ "action":"open_app", "text": "app name" }}.
+
+### Description ###
+A brief description of the chosen action. Do not describe expected outcome.
+"""
+
+
+    return prompt
+
+
+def parse_executor_response(response: str) -> dict:
+    """
+    Parse the Executor LLM response.
+
+    Extracts:
+    - thought: Content between "### Thought" and "### Action"
+    - action: Content between "### Action" and "### Description"
+    - description: Content after "### Description"
+
+    Args:
+        response: Raw LLM response string
+
+    Returns:
+        Dictionary with 'thought', 'action', 'description' keys
+    """
+    thought = response.split("### Thought")[-1].split("### Action")[0].replace("\n", " ").replace("  ", " ").replace("###", "").strip()
+    action = response.split("### Action")[-1].split("### Description")[0].replace("\n", " ").replace("  ", " ").replace("###", "").strip()
+    description = response.split("### Description")[-1].replace("\n", " ").replace("  ", " ").replace("###", "").strip()
+
+    return {
+        "thought": thought,
+        "action": action,
+        "description": description
+    }

droidrun/agent/manager/__init__.py

@@ -0,0 +1,18 @@
+"""
+Manager Agent - Planning and reasoning workflow.
+"""
+
+from droidrun.agent.manager.events import ManagerPlanEvent, ManagerThinkingEvent
+from droidrun.agent.manager.manager_agent import ManagerAgent
+from droidrun.agent.manager.prompts import (
+    build_manager_system_prompt,
+    parse_manager_response,
+)
+
+__all__ = [
+    "ManagerAgent",
+    "ManagerThinkingEvent",
+    "ManagerPlanEvent",
+    "build_manager_system_prompt",
+    "parse_manager_response",
+]

droidrun/agent/manager/events.py

@@ -0,0 +1,20 @@
+"""
+Events for the ManagerAgent workflow.
+"""
+
+from llama_index.core.workflow.events import Event
+
+
+class ManagerThinkingEvent(Event):
+    """Manager is thinking about the plan"""
+    pass
+
+
+class ManagerPlanEvent(Event):
+    """Manager has created a plan"""
+    plan: str
+    current_subgoal: str
+    completed_plan: str
+    thought: str
+    manager_answer: str = ""
+    memory_update: str = ""