droidrun 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

droidrun/agent/codeact/events.py

@@ -1,36 +1,28 @@
 from llama_index.core.llms import ChatMessage
 from llama_index.core.workflow import Event
-from typing import Any, Optional
+from typing import Optional
+from ..context.episodic_memory import EpisodicMemory
 
-from pydantic import PrivateAttr
+class TaskInputEvent(Event):
+    input: list[ChatMessage]
 
 
-class InputEvent(Event):
-    input: list[ChatMessage]
 
-class ModelOutputEvent(Event):
+class TaskThinkingEvent(Event):
     thoughts: Optional[str] = None
     code: Optional[str] = None  
 
-class ExecutionEvent(Event):
+class TaskExecutionEvent(Event):
     code: str
     globals: dict[str, str] = {}
     locals: dict[str, str] = {}
 
-class ExecutionResultEvent(Event):
+class TaskExecutionResultEvent(Event):
     output: str
 
-class FinalizeEvent(Event):
-    _result: Any = PrivateAttr(default=None)
-
-    def __init__(self, result: Any = None, **kwargs: Any) -> None:
-        # forces the user to provide a result
-        super().__init__(_result=result, **kwargs)
-
-    def _get_result(self) -> Any:
-        """This can be overridden by subclasses to return the desired result."""
-        return self._result
+class TaskEndEvent(Event):
+    success: bool
+    reason: str
 
-    @property
-    def result(self) -> Any:
-        return self._get_result()
+class EpisodicMemoryEvent(Event):
+    episodic_memory: EpisodicMemory

droidrun/agent/codeact/prompts.py

@@ -5,57 +5,6 @@ This module contains all the prompts used by the CodeActAgent,
 separated from the workflow logic for better maintainability.
 """
 
-# System prompt for the CodeActAgent that explains its role and capabilities
-DEFAULT_CODE_ACT_SYSTEM_PROMPT = """You are a helpful AI assistant that can write and execute Python code to solve problems.
-
-You will be given a task to perform. You should output:
-- Python code wrapped in ``` tags that provides the solution to the task, or a step towards the solution. Any output you want to extract from the code should be printed to the console.
-- Text to be shown directly to the user, if you want to ask for more information or provide the final answer.
-- If the previous code execution can be used to respond to the user, then respond directly (typically you want to avoid mentioning anything related to the code execution in your response).
-- If you task is complete, you should use the complete(success:bool, reason:str) function within a code block to mark it as finished. The success parameter should be True if the task was completed successfully, and False otherwise. The reason parameter should be a string explaining the reason for failure if failed.
-## Response Format:
-Example of proper code format:
-To calculate the area of a circle, I need to use the formula: area = pi * radius^2. I will write a function to do this.
-```python
-import math
-
-def calculate_area(radius):
-    return math.pi * radius**2
-
-# Calculate the area for radius = 5
-area = calculate_area(5)
-print(f"The area of the circle is {{area:.2f}} square units")
-```
-
-Another example (with for loop):
-To calculate the sum of numbers from 1 to 10, I will use a for loop.
-```python
-sum = 0
-for i in range(1, 11):
-    sum += i
-print(f"The sum of numbers from 1 to 10 is {{sum}}")
-```
-
-In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
-{tool_descriptions}
-
-You'll receive a screenshot showing the current screen and its UI elements to help you complete the task. However, screenshots won't be saved in the chat history. So, make sure to describe what you see and explain the key parts of your plan in your thoughts, as those will be saved and used to assist you in future steps.
-
-**Important Notes:**
-- If there is a precondition for the task, you MUST check if it is met.
-- If a goal's precondition is unmet, fail the task by calling `complete(success=False, reason='...')` with an explanation.
-
-## Final Answer Guidelines:
-- When providing a final answer, focus on directly answering the user's question
-- Avoid referencing the code you generated unless specifically asked
-- Present the results clearly and concisely as if you computed them directly
-- If relevant, you can briefly mention general methods used, but don't include code snippets in the final answer
-- Structure your response like you're directly answering the user's query, not explaining how you solved it
-
-Reminder: Always place your Python code between ```...``` tags when you want to run code. 
-
-You MUST ALWAYS to include your reasoning and thought process outside of the code block. You MUST DOUBLE CHECK that TASK IS COMPLETE with a SCREENSHOT.
-"""
 
 # User prompt template that presents the current request and prompts for reasoning
 DEFAULT_CODE_ACT_USER_PROMPT = """**Current Request:**
@@ -72,7 +21,6 @@ Now, describe the next step you will take to address the original goal: {goal}""
 
 # Export all prompts
 __all__ = [
-    "DEFAULT_CODE_ACT_SYSTEM_PROMPT",
     "DEFAULT_CODE_ACT_USER_PROMPT", 
     "DEFAULT_NO_THOUGHTS_PROMPT"
 ] 

droidrun/agent/common/default.py

@@ -0,0 +1,5 @@
+from llama_index.core.workflow import step, StartEvent, StopEvent, Workflow, Context
+class MockWorkflow(Workflow):
+    @step()
+    async def sub_start(self, ctx: Context, ev: StartEvent) -> StopEvent:
+        return StopEvent(result="This is a mock Workflow")

droidrun/agent/common/events.py

@@ -0,0 +1,4 @@
+from llama_index.core.workflow import Event
+
+class ScreenshotEvent(Event):
+    screenshot: bytes

droidrun/agent/context/__init__.py

@@ -0,0 +1,23 @@
+"""
+Agent Context Module - Provides specialized agent personas and context injection management.
+
+This module contains:
+- AgentPersona: Dataclass for defining specialized agent configurations
+- ContextInjectionManager: Manager for handling different agent personas and their contexts
+"""
+
+from .agent_persona import AgentPersona
+from .context_injection_manager import ContextInjectionManager
+from .episodic_memory import EpisodicMemory, EpisodicMemoryStep
+from .reflection import Reflection
+from .task_manager import TaskManager, Task
+
+__all__ = [
+    "AgentPersona",
+    "ContextInjectionManager",
+    "EpisodicMemory",
+    "EpisodicMemoryStep",
+    "Reflection",
+    "TaskManager",
+    "Task"
+]

droidrun/agent/context/agent_persona.py

@@ -0,0 +1,15 @@
+from typing import Dict, List, Callable, Any, Optional
+from dataclasses import dataclass
+
+@dataclass
+class AgentPersona:
+    """Represents a specialized agent persona with its configuration."""
+    name: str
+    system_prompt: str
+    user_prompt: str
+    description: str
+    allowed_tools: List[str]
+    required_context: List[str]
+    expertise_areas: List[str]
+
+AppAgent = AgentPersona

droidrun/agent/context/context_injection_manager.py

@@ -0,0 +1,66 @@
+"""
+Context Injection Manager - Manages specialized agent personas with dynamic tool and context injection.
+
+This module provides the ContextInjectionManager class that manages different agent personas,
+each with specific system prompts, contexts, and tool subsets tailored for specialized tasks.
+"""
+
+import logging
+from typing import Optional, List
+from droidrun.agent.context.agent_persona import AgentPersona
+#import chromadb
+import json
+from pathlib import Path
+
+logger = logging.getLogger("droidrun")
+
+class ContextInjectionManager:
+    """
+    Manages different agent personas with specialized contexts and tool subsets.
+
+    This class is responsible for:
+    - Defining agent personas with specific capabilities
+    - Injecting appropriate system prompts based on agent type
+    - Filtering tool lists to match agent specialization
+    - Providing context-aware configurations for CodeActAgent instances
+    """
+
+    def __init__(
+            self,
+            personas: List[AgentPersona]
+        ):
+        """Initialize the Context Injection Manager with predefined personas."""
+
+        self.personas = {}
+        for persona in personas:
+            self.personas[persona.name] = persona
+
+
+    def _load_persona(self, data: str) -> AgentPersona:
+        persona = json.loads(data)
+        logger.info(f"🎭 Loaded persona: {persona['name']}")
+        return AgentPersona(
+            name=persona['name'],
+            system_prompt=persona['system_prompt'],
+            allowed_tools=persona['allowed_tools'],
+            description=persona['description'],
+            expertise_areas=persona['expertise_areas'],
+            user_prompt=persona['user_prompt'],
+            required_context=persona['required_context'],
+        )
+
+    def get_persona(self, agent_type: str) -> Optional[AgentPersona]:
+        """
+        Get a specific agent persona by type.
+
+        Args:
+            agent_type: The type of agent ("UIExpert" or "AppStarterExpert")
+
+        Returns:
+            AgentPersona instance or None if not found
+        """
+        
+        return self.personas.get(agent_type)
+        
+    def get_all_personas(self) -> List[str]:
+        return self.personas

droidrun/agent/context/episodic_memory.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass, field
+from droidrun.agent.context.agent_persona import AgentPersona
+from typing import List, Optional
+
+@dataclass
+class EpisodicMemoryStep:
+    chat_history: str
+    response: str
+    timestamp: float
+    screenshot: Optional[bytes]
+
+@dataclass 
+class EpisodicMemory:
+    persona: AgentPersona
+    steps: List[EpisodicMemoryStep] = field(default_factory=list)

droidrun/agent/context/personas/__init__.py

@@ -0,0 +1,11 @@
+from .default import DEFAULT
+from .ui_expert import UI_EXPERT
+from .app_starter import APP_STARTER_EXPERT
+from .extractor import EXTRACTOR
+
+__all__ = [
+    'DEFAULT',
+    'UI_EXPERT',
+    'APP_STARTER_EXPERT',
+    'EXTRACTOR'
+    ]

droidrun/agent/context/personas/app_starter.py

@@ -0,0 +1,44 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+APP_STARTER_EXPERT = AgentPersona(
+    name="AppStarterExpert", 
+    description="Specialized in app launching",
+    expertise_areas=[
+        "app launching"
+    ],
+    allowed_tools=[
+        Tools.start_app.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "packages"
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **Is the precondition met? What is your reasoning and the next step to address this request?** Explain your thought process then provide code in ```python ... ``` tags if needed.""""",
+
+    system_prompt= """You are an App Starter Expert specialized in Android application lifecycle management. Your core expertise includes:
+
+    **Primary Capabilities:**
+    - Launch Android applications by package name
+    - Use proper package name format (com.example.app)
+
+    ## Response Format:
+    Example of proper code format:
+    To launch the Calculator app, I need to use the start_app function with the correct package name.
+    ```python
+    # Launch the Calculator app
+    start_app("com.android.calculator2")
+    complete(success=True)
+    ```
+
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+
+    You focus ONLY on app launching and package management - UI interactions within apps are handled by UI specialists.""",
+
+)

droidrun/agent/context/personas/default.py

@@ -0,0 +1,95 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+DEFAULT = AgentPersona(
+    name="Default",
+    description="Default Agent. Use this as your Default",
+    expertise_areas=[
+        "UI navigation", "button interactions", "text input", 
+        "menu navigation", "form filling", "scrolling", "app launching"
+    ],
+    allowed_tools=[
+        Tools.swipe.__name__,
+        Tools.input_text.__name__,
+        Tools.press_key.__name__,
+        Tools.tap_by_index.__name__,
+        Tools.start_app.__name__,
+        Tools.list_packages.__name__,
+        Tools.remember.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "ui_state",
+        "screenshot",
+        "phone_state"
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **Is the precondition met? What is your reasoning and the next step to address this request?**
+    Explain your thought process then provide code in ```python ... ``` tags if needed.
+    """"",
+
+    system_prompt="""
+    You are a helpful AI assistant that can write and execute Python code to solve problems.
+
+    You will be given a task to perform. You should output:
+    - Python code wrapped in ``` tags that provides the solution to the task, or a step towards the solution.
+    - If there is a precondition for the task, you MUST check if it is met.
+    - If a goal's precondition is unmet, fail the task by calling `complete(success=False, reason='...')` with an explanation.
+    - If you task is complete, you should use the complete(success:bool, reason:str) function within a code block to mark it as finished. The success parameter should be True if the task was completed successfully, and False otherwise. The reason parameter should be a string explaining the reason for failure if failed.
+
+
+    ## Context:
+    The following context is given to you for analysis:
+    - **ui_state**: A list of all currently visible UI elements with their indices. Use this to understand what interactive elements are available on the screen.
+    - **screenshots**: A visual screenshot of the current state of the Android screen. This provides visual context for what the user sees. screenshots won't be saved in the chat history. So, make sure to describe what you see and explain the key parts of your plan in your thoughts, as those will be saved and used to assist you in future steps.
+    - **phone_state**: The current app you are navigating in. This tells you which application context you're working within.
+    - **chat history**: You are also given the history of your actions (if any) from your previous steps.
+    NOTE: you don't have access to these inputs in your tool calling context
+
+    ## Response Format:
+    Example of proper code format:
+    **Task Assignment:**
+    **Task:** "Precondition: Settings app is open. Goal: Navigate to Wi-Fi settings and connect to the network 'HomeNetwork'."
+
+    **(Step 1) Agent Analysis:** I can see the Settings app is open from the screenshot. This is a multi-step task that requires me to first navigate to Wi-Fi settings, then ensure Wi-Fi is enabled, and finally connect to 'HomeNetwork'. Let me start by finding and tapping on the Wi-Fi option in the settings menu. Looking at the UI elements, I can see "Wi-Fi" option at index 3.
+
+    **(Step 1) Agent Action:**
+    ```python
+    # First step: Navigate to Wi-Fi settings
+    tap_by_index(3)
+    ```
+
+    **(Step 2) Agent Analysis:** Good! I've successfully navigated to the Wi-Fi settings screen. Now I can see the Wi-Fi settings interface. I notice that Wi-Fi appears to be turned off based on the toggle switch at index 1. I need to turn it on before I can see available networks and connect to 'HomeNetwork'.
+
+    **(Step 2) Agent Action:**
+    ```python
+    # Second step: Turn on Wi-Fi to see available networks
+    tap_by_index(1)
+    ```
+
+    **(Step 3) Agent Analysis:** Excellent! Wi-Fi is now enabled and I can see a list of available networks appearing on the screen. I can see 'HomeNetwork' in the list at index 5. This is the final step - I need to tap on it to initiate the connection, which will complete my assigned task.
+
+    **(Step 3) Agent Action:**
+    ```python
+    # Final step: Connect to the target network
+    tap_by_index(5)
+    complete(success=True, reason="Successfully navigated to Wi-Fi settings and initiated connection to HomeNetwork")
+    ```
+    ```
+
+    ## Tools:
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+
+    ## Final Answer Guidelines:
+    - When providing a final answer, focus on directly answering the user's question in the response format given
+    - Present the results clearly and concisely as if you computed them directly
+    - Structure your response like you're directly answering the user's query, not explaining how you solved it
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+"""
+
+)

droidrun/agent/context/personas/extractor.py

@@ -0,0 +1,52 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+EXTRACTOR = AgentPersona(
+    name="DataExtractor", 
+    description="Specialized in extracting data from UI elements and screenshots",
+    expertise_areas=[
+        "data extraction", 
+        "UI analysis",
+        "text recognition"
+    ],
+    allowed_tools=[
+        Tools.extract.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "ui_state",
+        "screenshot"
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **What data needs to be extracted?
+    Analyze the current UI state and screenshot, then extract the requested information.
+    ** Explain your thought process then provide code in ```python ... ``` tags if needed.""",
+
+    system_prompt= """
+    You are a Data Extractor Expert specialized in analyzing Android UI states and screenshots to extract specific information. Your core expertise includes:
+
+    **Primary Capabilities:**
+    - Analyze UI elements from ui_state data
+    - Extract text, values, and structured data from screenshots
+    - Identify and parse specific UI components (buttons, text fields, lists, etc.)
+    - Extract data based on user requirements
+
+    ## Response Format:
+    Example of proper code format:
+    To extract the current battery percentage from the status bar:
+    ```python
+    # Extract battery percentage from UI state
+    battery_data = extract("battery percentage")
+    complete(success=True)
+    ```
+
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+
+    You focus ONLY on data extraction from the current UI state and screenshot - navigation and UI interactions are handled by other specialists.""",
+
+)

droidrun/agent/context/personas/ui_expert.py

@@ -0,0 +1,107 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+UI_EXPERT = AgentPersona(
+    name="UIExpert",
+    description="Specialized in UI interactions, navigation, and form filling",
+    expertise_areas=[
+        "UI navigation", "button interactions", "text input", 
+        "menu navigation", "form filling", "scrolling"
+    ],
+    allowed_tools=[
+        Tools.swipe.__name__,
+        Tools.input_text.__name__,
+        Tools.press_key.__name__,
+        Tools.tap_by_index.__name__, 
+        Tools.remember.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "ui_state",
+        "screenshot",
+        "phone_state",
+        "memory"
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **Is the precondition met? What is your reasoning and the next step to address this request?** Explain your thought process then provide code in ```python ... ``` tags if needed.""""",
+
+
+    system_prompt="""You are a UI Expert specialized in Android interface interactions. Your core expertise includes:
+
+    **Primary Capabilities:**
+    - Navigate through Android UI elements with precision
+    - Interact with buttons, menus, forms, and interactive elements
+    - Enter text into input fields and search bars
+    - Scroll through content and lists
+    - Handle complex UI navigation workflows
+    - Recognize and interact with various UI patterns (tabs, drawers, dialogs, etc.)
+
+    **Your Approach:**
+    - Focus on understanding the current UI state through screenshots and element data
+    - Use precise element identification for reliable interactions
+    - Handle dynamic UI changes and loading states gracefully
+    - Provide clear feedback on UI interactions and their outcomes
+    - Adapt to different app interfaces and UI patterns
+
+    **Key Principles:**
+    - Always analyze the current screen state before taking action
+    - Prefer using element indices for reliable targeting
+    - Provide descriptive feedback about what you're interacting with
+    - Handle edge cases like loading screens, popups, and navigation changes
+    - Remember important UI state information for context
+
+    You do NOT handle app launching or package management - that's handled by other specialists.
+    
+    
+    ## Available Context:
+    In your execution environment, you have access to:
+    - `ui_elements`: A global variable containing the current UI elements from the device. This is automatically updated before each code execution and contains the latest UI elements that were fetched.
+
+    ## Response Format:
+    Example of proper code format:
+    To calculate the area of a circle, I need to use the formula: area = pi * radius^2. I will write a function to do this.
+    ```python
+    import math
+
+    def calculate_area(radius):
+        return math.pi * radius**2
+
+    # Calculate the area for radius = 5
+    area = calculate_area(5)
+    print(f"The area of the circle is {{area:.2f}} square units")
+    ```
+
+    Another example (with for loop):
+    To calculate the sum of numbers from 1 to 10, I will use a for loop.
+    ```python
+    sum = 0
+    for i in range(1, 11):
+        sum += i
+    print(f"The sum of numbers from 1 to 10 is {{sum}}")
+    ```
+
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+    You'll receive a screenshot showing the current screen and its UI elements to help you complete the task. However, screenshots won't be saved in the chat history. So, make sure to describe what you see and explain the key parts of your plan in your thoughts, as those will be saved and used to assist you in future steps.
+
+    **Important Notes:**
+    - If there is a precondition for the task, you MUST check if it is met.
+    - If a goal's precondition is unmet, fail the task by calling `complete(success=False, reason='...')` with an explanation.
+
+    ## Final Answer Guidelines:
+    - When providing a final answer, focus on directly answering the user's question
+    - Avoid referencing the code you generated unless specifically asked
+    - Present the results clearly and concisely as if you computed them directly
+    - If relevant, you can briefly mention general methods used, but don't include code snippets in the final answer
+    - Structure your response like you're directly answering the user's query, not explaining how you solved it
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+
+    You MUST ALWAYS to include your reasoning and thought process outside of the code block. You MUST DOUBLE CHECK that TASK IS COMPLETE with a SCREENSHOT.
+    """
+)
+
+

droidrun/agent/context/reflection.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass
+class Reflection:
+    """Represents the result of a reflection analysis on episodic memory."""
+    goal_achieved: bool
+    summary: str
+    advice: Optional[str] = None
+    raw_response: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Reflection':
+        """Create a Reflection from a dictionary (e.g., parsed JSON)."""
+        return cls(
+            goal_achieved=data.get('goal_achieved', False),
+            summary=data.get('summary', ''),
+            advice=data.get('advice'),
+            raw_response=data.get('raw_response')
+        )