droidrun 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

droidrun/agent/droid/droid_agent.py

@@ -0,0 +1,418 @@
+"""
+DroidAgent - A wrapper class that coordinates the planning and execution of tasks
+to achieve a user's goal on an Android device.
+"""
+
+import asyncio
+import logging
+from typing import Dict, Any, List, Tuple
+
+from llama_index.core.base.llms.types import ChatMessage
+from llama_index.core.llms.llm import LLM
+from llama_index.core.memory import ChatMemoryBuffer
+from ..codeact import CodeActAgent
+from ..planner import PlannerAgent, TaskManager
+from ..utils.executer import SimpleCodeExecutor
+from ...tools import Tools
+
+logger = logging.getLogger("droidrun")
+
+class DroidAgent:
+    """
+    A wrapper class that coordinates between PlannerAgent (creates plans) and 
+    CodeActAgent (executes tasks) to achieve a user's goal.
+    """
+    
+    def __init__(
+        self, 
+        goal: str,
+        llm: LLM,
+        tools_instance: 'Tools' = None,
+        tool_list: Dict[str, Any] = None,
+        max_steps: int = 15,
+        vision: bool = False,
+        timeout: int = 1000,
+        max_retries: int = 3,
+        reasoning: bool = True,
+        enable_tracing: bool = False,
+        debug: bool = False,
+        device_serial: str = None,
+        **kwargs
+    ):
+        """
+        Initialize the DroidAgent wrapper.
+        
+        Args:
+            goal: The user's goal or command to execute
+            llm: The language model to use for both agents
+            tools_instance: An instance of the Tools class
+            tool_list: Dictionary of available tools
+            max_steps: Maximum number of steps for both agents
+            vision: Whether to enable vision capabilities
+            timeout: Timeout for agent execution in seconds
+            max_retries: Maximum number of retries for failed tasks
+            reasoning: Whether to use the PlannerAgent for complex reasoning (True) 
+                      or send tasks directly to CodeActAgent (False)
+            enable_tracing: Whether to enable Arize Phoenix tracing
+            debug: Whether to enable verbose debug logging
+            device_serial: Target Android device serial number
+            **kwargs: Additional keyword arguments to pass to the agents
+        """
+        self.goal = goal
+        self.llm = llm
+        self.max_steps = max_steps
+        self.vision = vision
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.task_manager = TaskManager()
+        self.reasoning = reasoning
+        self.debug = debug
+        self.device_serial = device_serial
+        
+        logger.info("🤖 Initializing DroidAgent wrapper...")
+        
+        self.tools_instance = tools_instance
+        self.tool_list = tool_list
+        
+        # Ensure remember tool is in the tool_list if available
+        if hasattr(tools_instance, 'remember') and 'remember' not in tool_list:
+            logger.debug("📝 Adding 'remember' tool to the available tools")
+            tool_list['remember'] = tools_instance.remember
+        
+        # Create code executor
+        logger.debug("🔧 Initializing Code Executor...")
+        loop = asyncio.get_event_loop()
+        self.executor = SimpleCodeExecutor(
+            loop=loop,
+            locals={},
+            tools=tool_list,
+            globals={"__builtins__": __builtins__}
+        )
+        
+        # Create memory buffer for the planning agent if reasoning is enabled
+        if self.reasoning:
+            self.planning_memory = ChatMemoryBuffer.from_defaults(llm=llm)
+        
+        # Create CodeActAgent
+        logger.info("🧠 Initializing CodeAct Agent...")
+        self.codeact_agent = CodeActAgent(
+            llm=llm,
+            code_execute_fn=self.executor.execute,
+            available_tools=tool_list.values(),
+            tools=tools_instance,
+            max_steps=999999, 
+            vision=vision,
+            debug=debug,
+            timeout=timeout
+        )
+        
+        if self.reasoning:
+            logger.info("📝 Initializing Planner Agent...")
+            self.planner_agent = PlannerAgent(
+                goal=goal,
+                llm=llm,
+                agent=self.codeact_agent, 
+                tools_instance=tools_instance,
+                timeout=timeout,
+                max_retries=max_retries,
+                enable_tracing=enable_tracing,
+                debug=debug
+            )
+            
+            # Give task manager to the planner
+            self.planner_agent.task_manager = self.task_manager
+        else:
+            logger.debug("🚫 Planning disabled - will execute tasks directly with CodeActAgent")
+            self.planner_agent = None
+        
+        logger.info("✅ DroidAgent initialized successfully.")
+    
+    async def _get_plan_from_planner(self) -> List[Dict]:
+        """
+        Get a plan (list of tasks) from the PlannerAgent.
+        
+        Returns:
+            List of task dictionaries
+        """
+        logger.info("📋 Planning steps to accomplish the goal...")
+        
+        # Create system and user messages
+        system_msg = ChatMessage(role="system", content=self.planner_agent.system_prompt)
+        user_msg = ChatMessage(role="user", content=self.planner_agent.user_prompt)
+        
+        # Check if we have task history to add to the prompt
+        task_history = ""
+        # Use the persistent task history methods to get ALL completed and failed tasks
+        completed_tasks = self.task_manager.get_all_completed_tasks()
+        failed_tasks = self.task_manager.get_all_failed_tasks()
+        
+        # Show any remembered information in task history
+        remembered_info = ""
+        if hasattr(self.tools_instance, 'memory') and self.tools_instance.memory:
+            remembered_info = "\n### Remembered Information:\n"
+            for idx, item in enumerate(self.tools_instance.memory, 1):
+                remembered_info += f"{idx}. {item}\n"
+        
+        if completed_tasks or failed_tasks or remembered_info:
+            task_history = "### Task Execution History:\n"
+            
+            if completed_tasks:
+                task_history += "✅ Completed Tasks:\n"
+                for task in completed_tasks:
+                    task_history += f"- {task['description']}\n"
+
+            if failed_tasks:
+                task_history += "\n❌ Failed Tasks:\n"
+                for task in failed_tasks:
+                    failure_reason = task.get('failure_reason', 'Unknown reason')
+                    task_history += f"- {task['description']} (Failed: {failure_reason})\n"
+            
+            if remembered_info:
+                task_history += remembered_info
+                
+            # Add a reminder to use this information
+            task_history += "\n⚠️ Please use the above information in your planning. For example, if specific dates or locations were found, include them explicitly in your next tasks instead of just referring to 'the dates' or 'the location'.\n"
+            
+            # Append task history to user prompt
+            user_msg = ChatMessage(
+                role="user", 
+                content=f"{self.planner_agent.user_prompt}\n\n{task_history}\n\nPlease consider the above task history and discovered information when creating your next plan. Incorporate specific data (dates, locations, etc.) directly into tasks rather than referring to them generally. Remember that previously completed or failed tasks will not be repeated."
+            )
+        
+        # Create message list
+        messages = [system_msg, user_msg]
+        if self.debug:
+            logger.debug(f"Sending {len(messages)} messages to planner: {[msg.role for msg in messages]}")
+        
+        # Get response from LLM
+        llm_response = await self.planner_agent._get_llm_response(messages)
+        code, thoughts = self.planner_agent._extract_code_and_thought(llm_response.message.content)
+        
+        # Execute the planning code (which should call set_tasks)
+        if code:
+            try:
+                planning_tools = {
+                    "set_tasks": self.task_manager.set_tasks,
+                    "add_task": self.task_manager.add_task,
+                    "get_all_tasks": self.task_manager.get_all_tasks,
+                    "clear_tasks": self.task_manager.clear_tasks,
+                    "complete_goal": self.task_manager.complete_goal
+                }
+                planning_executor = SimpleCodeExecutor(
+                    loop=asyncio.get_event_loop(),
+                    globals={},
+                    locals={},
+                    tools=planning_tools
+                )
+                await planning_executor.execute(code)
+            except Exception as e:
+                logger.error(f"Error executing planning code: {e}")
+                # If there's an error, create a simple default task
+                self.task_manager.set_tasks([f"Achieve the goal: {self.goal}"])
+        
+        # Get and display the tasks
+        tasks = self.task_manager.get_all_tasks()
+        if tasks:
+            logger.info("📝 Plan created:")
+            for i, task in enumerate(tasks, 1):
+                if task["status"] == self.task_manager.STATUS_PENDING:
+                    logger.info(f"  {i}. {task['description']}")
+        else:
+            logger.warning("No tasks were generated in the plan")
+            
+        return tasks
+
+    async def _execute_task_with_codeact(self, task: Dict) -> Tuple[bool, str]:
+        """
+        Execute a single task using the CodeActAgent.
+        
+        Args:
+            task: Task dictionary with description and status
+            
+        Returns:
+            Tuple of (success, reason)
+        """
+        task_description = task["description"]
+        logger.info(f"🔧 Executing task: {task_description}")
+        
+        # Update task status
+        task["status"] = self.task_manager.STATUS_ATTEMPTING
+        
+        # Run the CodeActAgent
+        try:
+            # Reset the tools finished flag before execution
+            self.tools_instance.finished = False
+            self.tools_instance.success = None
+            self.tools_instance.reason = None
+            
+            # Execute the CodeActAgent with the task description as input
+            # Pass input as a keyword argument, not as a dictionary
+            result = await self.codeact_agent.run(input=task_description)
+            
+            # Check if the tools instance was marked as finished by the 'complete' function
+            if self.tools_instance.finished:
+                if self.tools_instance.success:
+                    task["status"] = self.task_manager.STATUS_COMPLETED
+                    if self.debug:
+                        logger.debug(f"Task completed successfully: {self.tools_instance.reason}")
+                    return True, self.tools_instance.reason or "Task completed successfully"
+                else:
+                    task["status"] = self.task_manager.STATUS_FAILED
+                    task["failure_reason"] = self.tools_instance.reason or "Task failed without specific reason"
+                    logger.warning(f"Task failed: {task['failure_reason']}")
+                    return False, self.tools_instance.reason or "Task failed without specific reason"
+            
+            # If tools instance wasn't marked as finished, check the result directly
+            if result and isinstance(result, dict) and "success" in result and result["success"]:
+                task["status"] = self.task_manager.STATUS_COMPLETED
+                if self.debug:
+                    logger.debug(f"Task completed with result: {result}")
+                return True, result.get("reason", "Task completed successfully")
+            else:
+                failure_reason = result.get("reason", "Unknown failure") if isinstance(result, dict) else "Task execution failed"
+                task["status"] = self.task_manager.STATUS_FAILED
+                task["failure_reason"] = failure_reason
+                logger.warning(f"Task failed: {failure_reason}")
+                return False, failure_reason
+                
+        except Exception as e:
+            logger.error(f"Error during task execution: {e}")
+            if self.debug:
+                import traceback
+                logger.error(traceback.format_exc())
+            task["status"] = self.task_manager.STATUS_FAILED
+            task["failure_reason"] = f"Error: {str(e)}"
+            return False, f"Error: {str(e)}"
+
+    async def run(self) -> Dict[str, Any]:
+        """
+        Main execution loop that coordinates between planning and execution.
+        
+        Returns:
+            Dict containing the execution result
+        """
+        logger.info(f"🚀 Running DroidAgent to achieve goal: {self.goal}")
+        
+        step_counter = 0
+        retry_counter = 0
+        overall_success = False
+        final_message = ""
+        
+        try:
+            # If reasoning is disabled, directly execute the goal as a single task in CodeActAgent
+            if not self.reasoning:
+                logger.info(f"🔄 Direct execution mode - executing goal: {self.goal}")
+                # Create a simple task for the goal
+                task = {
+                    "description": self.goal,
+                    "status": self.task_manager.STATUS_PENDING
+                }
+                
+                # Execute the task directly with CodeActAgent
+                success, reason = await self._execute_task_with_codeact(task)
+                
+                return {
+                    "success": success,
+                    "reason": reason,
+                    "steps": 1,
+                    "task_history": [task]  # Single task history
+                }
+            
+            # Standard reasoning mode with planning
+            while step_counter < self.max_steps:
+                step_counter += 1
+                if self.debug:
+                    logger.debug(f"Planning step {step_counter}/{self.max_steps}")
+                
+                # 1. Get a plan from the planner
+                tasks = await self._get_plan_from_planner()
+                
+                if self.task_manager.task_completed:
+                    # Task is marked as complete by the planner
+                    logger.info(f"✅ Goal completed: {self.task_manager.message}")
+                    overall_success = True
+                    final_message = self.task_manager.message
+                    break
+                
+                if not tasks:
+                    logger.warning("No tasks generated by planner")
+                    final_message = "Planner did not generate any tasks"
+                    break
+                
+                # 2. Execute each task in the plan sequentially
+                for task in tasks:
+                    if task["status"] == self.task_manager.STATUS_PENDING:
+                        # Reset the CodeActAgent's step counter for this task
+                        self.codeact_agent.steps_counter = 0
+                        
+                        # Execute the task
+                        success, reason = await self._execute_task_with_codeact(task)
+                        
+                        # Update task info with detailed result for the planner
+                        task_idx = tasks.index(task)
+                        result_info = {
+                            "execution_details": reason,
+                            "step_executed": step_counter,
+                            "codeact_steps": self.codeact_agent.steps_counter
+                        }
+                        
+                        # Only update if not already updated in _execute_task_with_codeact
+                        if success:
+                            self.task_manager.update_status(
+                                task_idx, 
+                                self.task_manager.STATUS_COMPLETED, 
+                                result_info
+                            )
+                            logger.info(f"✅ Task completed: {task['description']}")
+                        
+                        if not success:
+                            # Store detailed failure information if not already set
+                            if "failure_reason" not in task:
+                                self.task_manager.update_status(
+                                    task_idx,
+                                    self.task_manager.STATUS_FAILED,
+                                    {"failure_reason": reason, **result_info}
+                                )
+                            
+                            # Handle retries
+                            if retry_counter < self.max_retries:
+                                retry_counter += 1
+                                logger.info(f"Retrying... ({retry_counter}/{self.max_retries})")
+                                # Next iteration will generate a new plan based on current state
+                                break
+                            else:
+                                logger.error(f"Max retries exceeded for task")
+                                final_message = f"Failed after {self.max_retries} retries. Reason: {reason}"
+                                return {"success": False, "reason": final_message}
+                
+                # Reset retry counter for new task sequence
+                retry_counter = 0
+                
+                # Check if all tasks are completed
+                all_completed = all(task["status"] == self.task_manager.STATUS_COMPLETED for task in tasks)
+                if all_completed:
+                    # Get a new plan (the planner might decide we're done)
+                    continue
+            
+            # Check if we exited due to max steps
+            if step_counter >= self.max_steps and not overall_success:
+                final_message = f"Reached maximum number of steps ({self.max_steps})"
+                overall_success = False
+                
+            return {
+                "success": overall_success,
+                "reason": final_message,
+                "steps": step_counter,
+                "task_history": self.task_manager.get_task_history()
+            }
+                
+        except Exception as e:
+            logger.error(f"❌ Error during DroidAgent execution: {e}")
+            if self.debug:
+                import traceback
+                logger.error(traceback.format_exc())
+            return {
+                "success": False, 
+                "reason": str(e),
+                "task_history": self.task_manager.get_task_history()
+            } 

droidrun/agent/planner/__init__.py

@@ -0,0 +1,15 @@
+from .workflow import PlannerAgent
+from .task_manager import TaskManager
+from .prompts import (
+    DEFAULT_PLANNER_SYSTEM_PROMPT,
+    DEFAULT_PLANNER_USER_PROMPT,
+    DEFAULT_PLANNER_TASK_FAILED_PROMPT
+)
+
+__all__ = [
+    "PlannerAgent", 
+    "TaskManager",
+    "DEFAULT_PLANNER_SYSTEM_PROMPT",
+    "DEFAULT_PLANNER_USER_PROMPT",
+    "DEFAULT_PLANNER_TASK_FAILED_PROMPT"
+]

droidrun/agent/planner/events.py

@@ -0,0 +1,20 @@
+from typing import List
+from llama_index.core.llms import ChatMessage
+from llama_index.core.workflow import Event
+from llama_index.core.tools import FunctionTool
+
+
+class InputEvent(Event):
+    input: list[ChatMessage]
+
+class ModelResponseEvent(Event):
+    response: str
+
+
+class ExecutePlan(Event):
+    pass
+
+class TaskFailedEvent(Event):
+    task_description: str
+    reason: str
+

droidrun/agent/planner/prompts.py

@@ -0,0 +1,144 @@
+"""
+Prompt templates for the PlannerAgent.
+
+This module contains all the prompts used by the PlannerAgent,
+separated from the workflow logic for better maintainability.
+"""
+
+# System prompt for the PlannerAgent that explains its role and capabilities
+DEFAULT_PLANNER_SYSTEM_PROMPT = """You are an Android Task Planner. Your job is to create short, functional plans (1-5 steps) to achieve a user's goal on an Android device.
+
+**Inputs You Receive:**
+1.  **User's Overall Goal.**
+2.  **Current Device State:**
+    *   A **screenshot** of the current screen.
+    *   **JSON data** of visible UI elements.
+    *   The current visible Android activity
+3.  **Complete Task History:**
+    * A record of ALL tasks that have been completed or failed throughout the session.
+    * For completed tasks, the results and any discovered information.
+    * For failed tasks, the detailed reasons for failure.
+    * This history persists across all planning cycles and is never lost, even when creating new tasks.
+
+**Your Task:**
+Given the goal, current state, and task history, devise the **next 1-5 functional steps**. Focus on what to achieve, not how. Planning fewer steps at a time improves accuracy, as the state can change.
+
+**Step Format:**
+Each step must be a functional goal. A **precondition** describing the expected starting screen/state for that step is highly recommended for clarity, especially for steps after the first in your 1-5 step plan. Each task string can start with "Precondition: ... Goal: ...". If a specific precondition isn't critical for the first step in your current plan segment, you can use "Precondition: None. Goal: ..." or simply state the goal if the context is implicitly clear from the first step of a new sequence.
+
+**Executor Agent Capabilities:**
+The plan you create will be executed by another agent. This executor can:
+*   `swipe(direction: str, distance_percentage: int)`
+*   `input_text(text: str, element_hint: Optional[str] = None)`
+*   `press_key(keycode: int)` (Common: 3=HOME, 4=BACK)
+*   `tap_by_coordinates(x: int, y: int)` (This is a fallback; prefer functional goals)
+*   `start_app(package_name: str)`
+*   `remember(info: str)
+`
+*   (The executor will use the UI JSON to find elements for your functional goals like "Tap 'Settings button'" or "Enter text into 'Username field'").
+
+**Your Output:**
+*   Use the `set_tasks` tool to provide your 1-5 step plan as a list of strings.
+*   **After your planned steps are executed, you will be invoked again with the new device state.** You will then:
+    1.  Assess if the **overall user goal** is complete.
+    2.  If complete, call the `complete_goal(message: str)` tool.
+    3.  If not complete, generate the next 1-5 steps using `set_tasks`.
+
+**Memory Persistence:**
+*   You maintain a COMPLETE memory of ALL tasks across the entire session:
+    * Every task that was completed or failed is preserved in your context.
+    * Previously completed steps are never lost when calling `set_tasks()` for new steps.
+    * You will see all historical tasks each time you're called.
+    * Use this accumulated knowledge to build progressively on successful steps.
+    * When you see discovered information (e.g., dates, locations), use it explicitly in future tasks.
+
+**Key Rules:**
+*   **Functional Goals ONLY:** (e.g., "Navigate to Wi-Fi settings", "Enter 'MyPassword' into the password field").
+*   **NO Low-Level Actions:** Do NOT specify swipes, taps on coordinates, or element IDs in your plan.
+*   **Short Plans (1-5 steps):** Plan only the immediate next actions.
+*   **Learn From History:** If a task failed previously, try a different approach.
+*   **Use Tools:** Your response *must* be a Python code block calling `set_tasks` or `complete_goal`.
+
+**Available Planning Tools:**
+*   `set_tasks(tasks: List[str])`: Defines the sequence of tasks. Each element in the list is a string representing a single task.
+*   `complete_goal(message: str)`: Call this when the overall user goal has been achieved. The message can summarize the completion.
+
+---
+
+**Example Interaction Flow:**
+
+**User Goal:** Turn on Wi-Fi.
+
+**(Round 1) Planner Input:**
+*   Goal: "Turn on Wi-Fi"
+*   Current State: Screenshot of Home screen, UI JSON.
+*   Task History: None (first planning cycle)
+
+**Planner Thought Process (Round 1):**
+Need to open settings first, then go to Network settings. This is the first plan.
+1. Task 1: "Precondition: None. Goal: Open the Settings app."
+2. Task 2: "Precondition: Settings main screen is open. Goal: Navigate to 'Network & internet' settings."
+
+**Planner Output (Round 1):**
+```python
+set_tasks(tasks=[
+    "Precondition: None. Goal: Open the Settings app.",
+    "Precondition: Settings main screen is open. Goal: Navigate to 'Network & internet' settings."
+])
+```
+
+**(After Executor performs these steps...)**
+
+**(Round 2) Planner Input:**
+*   Goal: "Turn on Wi-Fi"
+*   Current State: Screenshot of "Network & internet" screen, UI JSON showing "Wi-Fi" option.
+*   Task History: Shows ALL previously completed tasks, including "Open the Settings app" and "Navigate to 'Network & internet' settings"
+
+**Planner Thought Process (Round 2):**
+Now on "Network & internet". Need to tap Wi-Fi, then enable it. I can see from history that we've already opened Settings and navigated to Network & internet.
+1. Task 1: "Precondition: 'Network & internet' screen is open. Goal: Tap the 'Wi-Fi' option."
+2. Task 2: "Precondition: Wi-Fi settings screen is open. Goal: Enable the Wi-Fi toggle if it's off."
+
+**Planner Output (Round 2):**
+```python
+set_tasks(tasks=[
+    "Precondition: 'Network & internet' screen is open. Goal: Tap the 'Wi-Fi' option.",
+    "Precondition: Wi-Fi settings screen is open. Goal: Enable the Wi-Fi toggle if it's off."
+])
+```
+
+**(After Executor performs these steps...)**
+
+**(Round 3) Planner Input:**
+*   Goal: "Turn on Wi-Fi"
+*   Current State: Screenshot of Wi-Fi screen, UI JSON showing Wi-Fi is now ON.
+*   Task History: Shows ALL previous tasks completed successfully (all 4 tasks from previous rounds)
+
+**Planner Output (Round 3):**
+```python
+complete_goal(message="Wi-Fi has been successfully enabled.")
+```"""
+
+# User prompt template that simply states the goal
+DEFAULT_PLANNER_USER_PROMPT = """Goal: {goal}"""
+
+# Prompt template for when a task fails, to help recover and plan new steps
+DEFAULT_PLANNER_TASK_FAILED_PROMPT = """
+PLANNING UPDATE: The execution of a task failed.
+
+Failed Task Description: "{task_description}"
+Reported Reason: {reason}
+
+The previous plan has been stopped. I have attached a screenshot representing the device's **current state** immediately after the failure. Please analyze this visual information.
+
+Original Goal: {goal}
+
+Instruction: Based **only** on the provided screenshot showing the current state and the reason for the previous failure ('{reason}'), generate a NEW plan starting from this observed state to achieve the original goal: '{goal}'.
+"""
+
+# Export all prompts
+__all__ = [
+    "DEFAULT_PLANNER_SYSTEM_PROMPT",
+    "DEFAULT_PLANNER_USER_PROMPT", 
+    "DEFAULT_PLANNER_TASK_FAILED_PROMPT"
+]