droidrun 0.3.8__py3-none-any.whl → 0.3.10.dev2__py3-none-any.whl

droidrun/agent/manager/manager_agent.py

@@ -0,0 +1,459 @@
+"""
+ManagerAgent - Planning and reasoning workflow.
+
+This agent is responsible for:
+- Analyzing the current state
+- Creating plans and subgoals
+- Tracking progress
+- Deciding when tasks are complete
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, List
+
+from llama_index.core.llms.llm import LLM
+from llama_index.core.workflow import Context, StartEvent, StopEvent, Workflow, step
+
+from droidrun.agent.manager.events import ManagerPlanEvent, ManagerThinkingEvent
+from droidrun.agent.manager.prompts import build_manager_system_prompt, parse_manager_response
+from droidrun.agent.utils import convert_messages_to_chatmessages
+from droidrun.agent.utils.chat_utils import remove_empty_messages
+from droidrun.agent.utils.device_state_formatter import get_device_state_exact_format
+from droidrun.agent.utils.inference import acall_with_retries
+from droidrun.agent.utils.tools import build_custom_tool_descriptions
+
+if TYPE_CHECKING:
+    from droidrun.agent.droid.events import DroidAgentState
+    from droidrun.tools import Tools
+
+logger = logging.getLogger("droidrun")
+
+
+class ManagerAgent(Workflow):
+    """
+    Planning and reasoning agent that decides what to do next.
+
+    The Manager:
+    1. Analyzes current device state and action history
+    2. Creates plans with specific subgoals
+    3. Tracks progress and completed steps
+    4. Decides when tasks are complete or need to provide answers
+    """
+
+    def __init__(
+        self,
+        llm: LLM,
+        vision: bool,
+        personas: List,
+        tools_instance: "Tools",
+        shared_state: "DroidAgentState",
+        custom_tools: dict = None,
+        debug: bool = False,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.llm = llm
+        self.vision = vision
+        self.personas = personas
+        self.tools_instance = tools_instance
+        self.shared_state = shared_state
+        self.custom_tools = custom_tools or {}
+        self.debug = debug
+
+        logger.info("✅ ManagerAgent initialized successfully.")
+
+    # ========================================================================
+    # Helper Methods
+    # ========================================================================
+
+    def _build_system_prompt(
+        self,
+        has_text_to_modify: bool
+    ) -> str:
+        """
+        Build system prompt with all context.
+
+        Args:
+            has_text_to_modify: Whether text manipulation mode is enabled
+
+        Returns:
+            Complete system prompt
+        """
+
+        # Get error history if error_flag_plan is set
+        error_history = []
+        if self.shared_state.error_flag_plan:
+            k = self.shared_state.err_to_manager_thresh
+            error_history = [
+                {
+                    "action": act,
+                    "summary": summ,
+                    "error": err_des
+                }
+                for act, summ, err_des in zip(
+                    self.shared_state.action_history[-k:],
+                    self.shared_state.summary_history[-k:],
+                    self.shared_state.error_descriptions[-k:], strict=True
+                )
+            ]
+
+        # Build custom tools descriptions
+        custom_tools_descriptions = build_custom_tool_descriptions(self.custom_tools)
+
+        return build_manager_system_prompt(
+            instruction=self.shared_state.instruction,
+            has_text_to_modify=has_text_to_modify,
+            app_card="",  # TODO: implement app card retrieval system
+            device_date=self.tools_instance.get_date(),
+            important_notes="",  # TODO: expose important_notes in DroidAgentState if needed
+            error_flag=self.shared_state.error_flag_plan,
+            error_history=error_history,
+            custom_tools_descriptions=custom_tools_descriptions
+        )
+
+    def _build_messages_with_context(
+        self,
+        system_prompt: str,
+        screenshot: str = None
+    ) -> list[dict]:
+        """
+        Build messages from history and inject current context.
+
+        Args:
+            system_prompt: System prompt to use
+            screenshot: Path to current screenshot (if vision enabled)
+
+        Returns:
+            List of message dicts ready for conversion
+        """
+        import copy
+
+        # Start with system message
+        messages = [
+            {"role": "system", "content": [{"text": system_prompt}]}
+        ]
+
+        # Add accumulated message history (deep copy to avoid mutation)
+        messages.extend(copy.deepcopy(self.shared_state.message_history))
+
+        # ====================================================================
+        # Inject memory, device state, screenshot to LAST user message
+        # ====================================================================
+        # Find last user message index
+        user_indices = [i for i, msg in enumerate(messages) if msg['role'] == 'user']
+
+        if user_indices:
+            last_user_idx = user_indices[-1]
+
+            # Add memory to last user message
+            current_memory = (self.shared_state.memory or "").strip()
+            if current_memory:
+                if messages[last_user_idx]['content'] and 'text' in messages[last_user_idx]['content'][0]:
+                    messages[last_user_idx]['content'][0]['text'] += f"\n<memory>\n{current_memory}\n</memory>\n"
+                else:
+                    messages[last_user_idx]['content'].insert(0, {"text": f"<memory>\n{current_memory}\n</memory>\n"})
+
+            # Add device state to last user message
+            current_a11y = (self.shared_state.ui_elements_list_after or self.shared_state.device_state_text or "").strip()
+            if current_a11y:
+                if messages[last_user_idx]['content'] and 'text' in messages[last_user_idx]['content'][0]:
+                    messages[last_user_idx]['content'][0]['text'] += f"\n<device_state>\n{current_a11y}\n</device_state>\n"
+                else:
+                    messages[last_user_idx]['content'].insert(0, {"text": f"<device_state>\n{current_a11y}\n</device_state>\n"})
+
+            # Add screenshot to last user message
+            if screenshot and self.vision:
+                messages[last_user_idx]['content'].append({"image": screenshot})
+
+            # Add previous device state to SECOND-TO-LAST user message (if exists)
+            if len(user_indices) >= 2:
+                second_last_user_idx = user_indices[-2]
+                prev_a11y = (self.shared_state.ui_elements_list_before or "").strip()
+
+                if prev_a11y:
+                    if messages[second_last_user_idx]['content'] and 'text' in messages[second_last_user_idx]['content'][0]:
+                        messages[second_last_user_idx]['content'][0]['text'] += f"\n<device_state>\n{prev_a11y}\n</device_state>\n"
+                    else:
+                        messages[second_last_user_idx]['content'].insert(0, {"text": f"<device_state>\n{prev_a11y}\n</device_state>\n"})
+        messages = remove_empty_messages(messages)
+        return messages
+
+    async def _validate_and_retry_llm_call(
+        self,
+        ctx: Context,
+        initial_messages: list[dict],
+        initial_response: str
+    ) -> str:
+        """
+        Validate LLM response and retry if needed.
+
+        Args:
+            ctx: Workflow context
+            initial_messages: Messages sent to LLM
+            initial_response: Initial LLM response
+
+        Returns:
+            Final validated response (may be same as initial or from retry)
+        """
+
+        output_planning = initial_response
+        parsed = parse_manager_response(output_planning)
+
+        max_retries = 3
+        retry_count = 0
+
+        while retry_count < max_retries:
+            # Validation rules
+            error_message = None
+
+            if parsed["answer"] and not parsed["plan"]:
+                # Valid: answer without plan (task complete)
+                break
+            elif parsed["plan"] and parsed["answer"]:
+                error_message = "You cannot use both request_accomplished tag while the plan is not finished. If you want to use request_accomplished tag, please make sure the plan is finished.\nRetry again."
+            elif not parsed["plan"]:
+                error_message = "You must provide a plan to complete the task. Please provide a plan with the correct format."
+            else:
+                # Valid: plan without answer
+                break
+
+            if error_message:
+                retry_count += 1
+                logger.warning(f"Manager response invalid (retry {retry_count}/{max_retries}): {error_message}")
+
+                # Retry with error message
+                retry_messages = initial_messages + [
+                    {"role": "assistant", "content": [{"text": output_planning}]},
+                    {"role": "user", "content": [{"text": error_message}]}
+                ]
+
+                chat_messages = convert_messages_to_chatmessages(retry_messages)
+
+                try:
+                    response = await acall_with_retries(self.llm, chat_messages)
+                    output_planning = response.message.content
+                    parsed = parse_manager_response(output_planning)
+                except Exception as e:
+                    logger.error(f"LLM retry failed: {e}")
+                    break  # Give up retrying
+
+        return output_planning
+
+    # ========================================================================
+    # Workflow Steps
+    # ========================================================================
+
+    @step
+    async def prepare_input(
+        self,
+        ctx: Context,
+        ev: StartEvent
+    ) -> ManagerThinkingEvent:
+        """
+        Gather context and prepare manager prompt.
+
+        This step:
+        1. Gets current device state (UI elements, screenshot)
+        2. Detects text manipulation mode
+        3. Builds message history entry with last action
+        4. Stores context for think() step
+        """
+        logger.info("💬 Preparing manager input...")
+
+        # ====================================================================
+        # Step 1: Get device state (UI elements accessibility tree)
+        # ====================================================================
+        device_state_text, focused_text = get_device_state_exact_format(self.tools_instance.get_state())
+
+        # ====================================================================
+        # Step 2: Capture screenshot if vision enabled
+        # ====================================================================
+        screenshot = None
+        if self.vision:
+            try:
+                result = self.tools_instance.take_screenshot()
+                if isinstance(result, tuple):
+                    success, screenshot = result
+                    if not success:
+                        screenshot = None
+                else:
+                    screenshot = result
+                logger.debug("📸 Screenshot captured for Manager")
+            except Exception as e:
+                logger.warning(f"Failed to capture screenshot: {e}")
+                screenshot = None
+
+        # ====================================================================
+        # Step 3: Detect text manipulation mode
+        # ====================================================================
+        focused_text = focused_text or ""
+        focused_text_clean = focused_text.replace("'", "").strip()
+
+        # Check if focused text differs from last typed text
+        # last_typed_text = ""
+        # if self.shared_state.action_history:
+        #     recent_actions = self.shared_state.action_history[-1:] if len(self.shared_state.action_history) >= 1 else []
+        #     for action in reversed(recent_actions):
+        #         if isinstance(action, dict) and action.get('action') == 'type':
+        #             last_typed_text = action.get('text', '')
+        #             break
+
+        has_text_to_modify = (focused_text_clean != "")
+
+        # ====================================================================
+        # Step 4: Update state with device info
+        # ====================================================================
+        self.shared_state.device_state_text = device_state_text
+        self.shared_state.focused_text = focused_text
+        # Shift UI elements: before ← after, after ← current
+        self.shared_state.ui_elements_list_before = self.shared_state.ui_elements_list_after
+        self.shared_state.ui_elements_list_after = device_state_text
+
+        # ====================================================================
+        # Step 5: Build user message entry
+        # ====================================================================
+        parts = []
+
+        # Add context from last action
+        if self.shared_state.finish_thought:
+            parts.append(f"<thought>\n{self.shared_state.finish_thought}\n</thought>\n")
+
+        if self.shared_state.last_action:
+            import json
+            action_str = json.dumps(self.shared_state.last_action)
+            parts.append(f"<last_action>\n{action_str}\n</last_action>\n")
+
+        if self.shared_state.last_summary:
+            parts.append(f"<last_action_description>\n{self.shared_state.last_summary}\n</last_action_description>\n")
+
+        
+        self.shared_state.message_history.append({
+            "role": "user",
+            "content": [{"text": "".join(parts)}]
+        })
+
+        # Store has_text_to_modify and screenshot for next step
+        self.shared_state.has_text_to_modify = has_text_to_modify
+        self.shared_state.screenshot = screenshot
+
+        logger.debug(f"  - Device state prepared (text_modify={has_text_to_modify}, screenshot={screenshot is not None})")
+        return ManagerThinkingEvent()
+
+    @step
+    async def think(
+        self,
+        ctx: Context,
+        ev: ManagerThinkingEvent
+    ) -> ManagerPlanEvent:
+        """
+        Manager reasons and creates plan.
+
+        This step:
+        1. Builds system prompt with all context
+        2. Builds messages from history with injected context
+        3. Calls LLM
+        4. Validates and retries if needed
+        5. Parses response
+        6. Updates state (memory, message history)
+        """
+        logger.info("🧠 Manager thinking about the plan...")
+
+        has_text_to_modify = self.shared_state.has_text_to_modify
+        screenshot = self.shared_state.screenshot
+
+        # ====================================================================
+        # Step 1: Build system prompt
+        # ====================================================================
+        system_prompt = self._build_system_prompt(has_text_to_modify)
+
+        # ====================================================================
+        # Step 2: Build messages with context
+        # ====================================================================
+        messages = self._build_messages_with_context(
+            system_prompt=system_prompt,
+            screenshot=screenshot
+        )
+
+        # ====================================================================
+        # Step 3: Convert messages and call LLM
+        # ====================================================================
+        chat_messages = convert_messages_to_chatmessages(messages)
+
+        try:
+            response = await acall_with_retries(self.llm, chat_messages)
+            output_planning = response.message.content
+        except Exception as e:
+            logger.error(f"LLM call failed: {e}")
+            raise RuntimeError(f"Error calling LLM in manager: {e}") from e
+
+        # ====================================================================
+        # Step 4: Validate and retry if needed
+        # ====================================================================
+        output_planning = await self._validate_and_retry_llm_call(
+            ctx=ctx,
+            initial_messages=messages,
+            initial_response=output_planning
+        )
+
+        # ====================================================================
+        # Step 5: Parse response
+        # ====================================================================
+        parsed = parse_manager_response(output_planning)
+
+        # ====================================================================
+        # Step 6: Update state
+        # ====================================================================
+        memory_update = parsed.get("memory", "").strip()
+
+        # Update memory (append, not replace)
+        if memory_update:
+            if self.shared_state.memory:
+                self.shared_state.memory += "\n" + memory_update
+            else:
+                self.shared_state.memory = memory_update
+
+        # Append assistant response to message history
+        self.shared_state.message_history.append({
+            "role": "assistant",
+            "content": [{"text": output_planning}]
+        })
+
+        # Update planning fields
+        self.shared_state.plan = parsed["plan"]
+        self.shared_state.current_subgoal = parsed["current_subgoal"]
+        self.shared_state.completed_plan = parsed.get("completed_subgoal", "No completed subgoal.")
+        self.shared_state.finish_thought = parsed["thought"]
+        self.shared_state.manager_answer = parsed["answer"]
+
+        logger.info(f"📝 Plan: {parsed['plan'][:100]}...")
+        logger.debug(f"  - Current subgoal: {parsed['current_subgoal']}")
+        logger.debug(f"  - Manager answer: {parsed['answer'][:50] if parsed['answer'] else 'None'}")
+
+        return ManagerPlanEvent(
+            plan=parsed["plan"],
+            current_subgoal=parsed["current_subgoal"],
+            completed_plan=parsed.get("completed_subgoal", "No completed subgoal."),
+            thought=parsed["thought"],
+            manager_answer=parsed["answer"],
+            memory_update=memory_update
+        )
+
+    @step
+    async def finalize(
+        self,
+        ctx: Context,
+        ev: ManagerPlanEvent
+    ) -> StopEvent:
+        """Return manager results to parent workflow."""
+        logger.debug("✅ Manager planning complete")
+
+        return StopEvent(result={
+            "plan": ev.plan,
+            "current_subgoal": ev.current_subgoal,
+            "completed_plan": ev.completed_plan,
+            "thought": ev.thought,
+            "manager_answer": ev.manager_answer,
+            "memory_update": ev.memory_update
+        })

droidrun/agent/manager/prompts.py

@@ -0,0 +1,223 @@
+"""
+Prompts for the ManagerAgent.
+"""
+
+import re
+
+
+def build_manager_system_prompt(
+    instruction: str,
+    has_text_to_modify: bool = False,
+    app_card: str = "",
+    device_date: str = "",
+    important_notes: str = "",
+    error_flag: bool = False,
+    error_history: list = [],  # noqa: B006
+    custom_tools_descriptions: str = ""
+) -> str:
+    """
+    Build the manager system prompt with all context.
+
+    Args:
+        instruction: User's goal/task
+        has_text_to_modify: Whether focused text field has editable content
+        app_card: App-specific instructions (TODO: implement app card system)
+        device_date: Current device date (TODO: implement via adb shell date)
+        important_notes: Additional important information
+        error_flag: Whether consecutive errors occurred
+        error_history: List of recent errors if error_flag=True
+        custom_tools_descriptions: Formatted descriptions of custom tools available to executor
+
+    Returns:
+        Complete system prompt for Manager
+    """
+    prompt = (
+        "You are an agent who can operate an Android phone on behalf of a user. "
+        "Your goal is to track progress and devise high-level plans to achieve the user's requests.\n\n"
+        "<user_request>\n"
+        f"{instruction}\n"
+        "</user_request>\n\n"
+    )
+
+
+    if device_date.strip():
+        prompt += f"<device_date>\n{device_date}\n</device_date>\n\n"
+
+
+
+
+    if app_card.strip():
+        prompt += "App card gives information on how to operate the app and perform actions.\n"
+        prompt += f"<app_card>\n{app_card.strip()}\n</app_card>\n\n"
+
+    # Important notes
+    if important_notes:
+        prompt += "<important_notes>\n"
+        prompt += f"{important_notes}\n"
+        prompt += "</important_notes>\n\n"
+
+    # Error escalation
+    if error_flag and error_history:
+        prompt += (
+            "<potentially_stuck>\n"
+            "You have encountered several failed attempts. Here are some logs:\n"
+        )
+        for error in error_history:
+            prompt += (
+                f"- Attempt: Action: {error['action']} | "
+                f"Description: {error['summary']} | "
+                f"Outcome: Failed | "
+                f"Feedback: {error['error']}\n"
+            )
+        prompt += "</potentially_stuck>\n\n"
+
+    # Guidelines
+    prompt += """<guidelines>
+The following guidelines will help you plan this request.
+General:
+1. Use the `open_app` action whenever you want to open an app, do not use the app drawer to open an app.
+2. Use search to quickly find a file or entry with a specific name, if search function is applicable.
+3. Only use copy to clipboard actions when the task specifically requires copying text to clipboard. Do not copy text just to use it later - use the Memory section instead.
+4. When you need to remember information for later use, store it in the Memory section (using <add_memory> tags) with step context (e.g., "At step X, I obtained [information] from [source]").
+5. File names in the user request must always match the exact file name you are working with, make that reflect in the plan too.
+6. Make sure names and titles are not cutoff. If the request is to check who sent a message, make sure to check the message sender's full name not just what appears in the notification because it might be cut off.
+7. Dates and file names must match the user query exactly.
+8. Don't do more than what the user asks for."""
+
+    # Text manipulation guidelines (conditional)
+    if has_text_to_modify:
+        prompt += """
+
+<text_manipulation>
+1. Use **TEXT_TASK:** prefix in your plan when you need to modify text in the currently focused text input field
+2. TEXT_TASK is for editing, formatting, or transforming existing text content in text boxes using Python code
+3. Do not use TEXT_TASK for extracting text from messages, typing new text, or composing messages
+4. The focused text field contains editable text that you can modify
+5. Example plan item: 'TEXT_TASK: Add "Hello World" at the beginning of the text'
+6. Always use TEXT_TASK for modifying text, do not try to select the text to copy/cut/paste or adjust the text
+</text_manipulation>"""
+
+    prompt += """
+
+Memory Usage:
+- Always include step context: "At step [number], I obtained [actual content] from [source]"
+- Store the actual content you observe, not just references (e.g., store full recipe text, not "found recipes")
+- Use memory instead of copying text unless specifically requested
+- Memory is append-only: whatever you put in <add_memory> tags gets added to existing memory, not replaced
+- Update memory to track progress on multi-step tasks
+
+</guidelines>"""
+
+    # Add custom tools section if custom tools are provided
+    if custom_tools_descriptions.strip():
+        prompt += """
+
+<custom_actions>
+The executor has access to these additional custom actions beyond the standard actions (click, type, swipe, etc.):
+""" + custom_tools_descriptions + """
+
+You can reference these custom actions or tell the Executer agent to use them in your plan when they help achieve the user's goal.
+</custom_actions>"""
+
+    prompt += """
+---
+Carefully assess the current status and the provided screenshot. Check if the current plan needs to be revised.
+Determine if the user request has been fully completed. If you are confident that no further actions are required, use the request_accomplished tag with a message in it. If the user request is not finished, update the plan and don't use it. If you are stuck with errors, think step by step about whether the overall plan needs to be revised to address the error.
+NOTE: 1. If the current situation prevents proceeding with the original plan or requires clarification from the user, make reasonable assumptions and revise the plan accordingly. Act as though you are the user in such cases. 2. Please refer to the helpful information and steps in the Guidelines first for planning. 3. If the first subgoal in plan has been completed, please update the plan in time according to the screenshot and progress to ensure that the next subgoal is always the first item in the plan. 4. If the first subgoal is not completed, please copy the previous round's plan or update the plan based on the completion of the subgoal.
+Provide your output in the following format, which contains four or five parts:
+
+<thought>
+An explanation of your rationale for the updated plan and current subgoal.
+</thought>
+
+<add_memory>
+Store important information here with step context for later reference. Always include "At step X, I obtained [actual content] from [source]".
+Examples:
+- At step 5, I obtained recipe details from recipes.jpg: Recipe 1 "Chicken Pasta" - ingredients: chicken, pasta, cream. Instructions: Cook pasta, sauté chicken, add cream.
+or
+- At step 12, I successfully added Recipe 1 to Broccoli app. Still need to add Recipe 2 and Recipe 3 from memory.
+Store important information here with step context for later reference.
+</add_memory>
+
+<plan>
+Please update or copy the existing plan according to the current page and progress. Please pay close attention to the historical operations. Please do not repeat the plan of completed content unless you can judge from the screen status that a subgoal is indeed not completed.
+</plan>
+
+<request_accomplished>
+Use this tag ONLY after actually completing the user's request through concrete actions, not at the beginning or for planning.
+
+1. Always include a message inside this tag confirming what you accomplished
+2. Ensure both opening and closing tags are present
+3. Use exclusively for signaling completed user requests
+</request_accomplished>"""
+
+    return prompt
+
+
+def parse_manager_response(response: str) -> dict:
+    """
+    Parse manager LLM response into structured dict.
+
+    Extracts XML-style tags from the response:
+    - <thought>...</thought>
+    - <add_memory>...</add_memory>
+    - <plan>...</plan>
+    - <request_accomplished>...</request_accomplished> (answer)
+    - <historical_operations>...</historical_operations> (optional, for completed plan)
+
+    Also derives:
+    - current_subgoal: first line of plan (with list markers removed)
+
+    Args:
+        response: Raw LLM response text
+
+    Returns:
+        Dict with keys:
+            - thought: str
+            - memory: str
+            - plan: str
+            - current_subgoal: str (first line of plan, cleaned)
+            - completed_subgoal: str
+            - answer: str (from request_accomplished tag)
+    """
+    def extract(tag: str) -> str:
+        """Extract content between XML-style tags."""
+        if f"<{tag}>" in response and f"</{tag}>" in response:
+            return response.split(f"<{tag}>", 1)[-1].split(f"</{tag}>", 1)[0].strip()
+        return ""
+
+    thought = extract("thought")
+    memory_section = extract("add_memory")
+    plan = extract("plan")
+    answer = extract("request_accomplished")
+
+    # Extract completed subgoal (optional historical_operations tag)
+    if "<historical_operations>" in response:
+        completed_subgoal = extract("historical_operations")
+    else:
+        completed_subgoal = "No completed subgoal."
+
+    # Parse current subgoal from first line of plan
+    current_goal_text = plan
+    # Prefer newline-separated plans; take the first non-empty line
+    plan_lines = [line.strip() for line in current_goal_text.splitlines() if line.strip()]
+    if plan_lines:
+        first_line = plan_lines[0]
+    else:
+        first_line = current_goal_text.strip()
+
+    # Remove common list markers like "1.", "-", "*", or bullet characters
+    first_line = re.sub(r"^\s*\d+\.\s*", "", first_line)  # Remove "1. ", "2. ", etc.
+    first_line = re.sub(r"^\s*[-*]\s*", "", first_line)    # Remove "- " or "* "
+    first_line = re.sub(r"^\s*•\s*", "", first_line)      # Remove bullet "• "
+
+    current_subgoal = first_line.strip()
+
+    return {
+        "thought": thought,
+        "completed_subgoal": completed_subgoal,
+        "plan": plan,
+        "memory": memory_section,
+        "current_subgoal": current_subgoal,
+        "answer": answer,
+    }