droidrun 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

droidrun/__init__.py

@@ -6,17 +6,21 @@ __version__ = "0.3.0"
 
 # Import main classes for easier access
 from droidrun.agent.utils.llm_picker import load_llm
-from droidrun.adb.manager import DeviceManager
 from droidrun.tools import Tools, AdbTools, IOSTools
 from droidrun.agent.droid import DroidAgent
 
+# Import macro functionality
+from droidrun.macro import MacroPlayer, replay_macro_file, replay_macro_folder
+
 
 # Make main components available at package level
 __all__ = [
     "DroidAgent",
-    "DeviceManager",
     "load_llm",
     "Tools",
     "AdbTools",
     "IOSTools",
+    "MacroPlayer",
+    "replay_macro_file",
+    "replay_macro_folder",
 ]

droidrun/agent/codeact/codeact_agent.py

@@ -97,6 +97,7 @@ class CodeActAgent(Workflow):
             loop=asyncio.get_event_loop(),
             locals={},
             tools=self.tool_list,
+            tools_instance=tools_instance,
             globals={"__builtins__": __builtins__},
         )
 
@@ -164,20 +165,22 @@ class CodeActAgent(Workflow):
             chat_history = await chat_utils.add_memory_block(self.remembered_info, chat_history)
 
         for context in self.required_context:
-            if model == "DeepSeek":
-                logger.warning(
-                    "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
-                )
-            elif self.vision == True and context == "screenshot":
-                screenshot = (await self.tools.take_screenshot())[1]
+            if context == "screenshot":
+                # if vision is disabled, screenshot should save to trajectory
+                screenshot = (self.tools.take_screenshot())[1]
                 ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
 
                 await ctx.set("screenshot", screenshot)
-                chat_history = await chat_utils.add_screenshot_image_block(screenshot, chat_history)
+                if model == "DeepSeek":
+                    logger.warning(
+                        "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
+                    )
+                elif self.vision == True: # if vision is enabled, add screenshot to chat history
+                    chat_history = await chat_utils.add_screenshot_image_block(screenshot, chat_history)
 
             if context == "ui_state":
                 try:
-                    state = await self.tools.get_state()
+                    state = self.tools.get_state()
                     await ctx.set("ui_state", state["a11y_tree"])
                     chat_history = await chat_utils.add_ui_text_block(
                         state["a11y_tree"], chat_history
@@ -189,7 +192,7 @@ class CodeActAgent(Workflow):
 
             if context == "packages":
                 chat_history = await chat_utils.add_packages_block(
-                    await self.tools.list_packages(include_system_apps=True),
+                    self.tools.list_packages(include_system_apps=True),
                     chat_history,
                 )
 
@@ -242,12 +245,15 @@ class CodeActAgent(Workflow):
         code = ev.code
         assert code, "Code cannot be empty."
         logger.info(f"⚡ Executing action...")
-        logger.debug(f"Code to execute:\n```python\n{code}\n```")
+        logger.info(f"Code to execute:\n```python\n{code}\n```")
 
         try:
             self.code_exec_counter += 1
             result = await self.executor.execute(ctx, code)
-            logger.info(f"💡 Code execution successful. Result: {result}")
+            logger.info(f"💡 Code execution successful. Result: {result['output']}")
+            screenshots = result['screenshots']
+            for screenshot in screenshots[:-1]: # the last screenshot will be captured by next step
+                ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
 
             if self.tools.finished == True:
                 logger.debug("  - Task completed.")
@@ -259,7 +265,7 @@ class CodeActAgent(Workflow):
             
             self.remembered_info = self.tools.memory
             
-            event = TaskExecutionResultEvent(output=str(result))
+            event = TaskExecutionResultEvent(output=str(result['output']))
             ctx.write_event_to_stream(event)
             return event
 
@@ -398,13 +404,13 @@ class CodeActAgent(Workflow):
             ui_state = None
             
             try:
-                _, screenshot_bytes = await self.tools.take_screenshot()
+                _, screenshot_bytes = self.tools.take_screenshot()
                 screenshot = screenshot_bytes
             except Exception as e:
                 logger.warning(f"Failed to capture final screenshot: {e}")
             
             try:
-                (a11y_tree, phone_state) = await self.tools.get_state()
+                (a11y_tree, phone_state) = self.tools.get_state()
             except Exception as e:
                 logger.warning(f"Failed to capture final UI state: {e}")
             

droidrun/agent/common/events.py

@@ -1,4 +1,47 @@
 from llama_index.core.workflow import Event
 
 class ScreenshotEvent(Event):
-    screenshot: bytes
+    screenshot: bytes
+
+class MacroEvent(Event):
+    """Base class for coordinate-based action events"""
+    action_type: str
+    description: str
+
+class TapActionEvent(MacroEvent):
+    """Event for tap actions with coordinates"""
+    x: int
+    y: int
+    element_index: int = None
+    element_text: str = ""
+    element_bounds: str = ""
+    
+class SwipeActionEvent(MacroEvent):
+    """Event for swipe actions with coordinates"""
+    start_x: int
+    start_y: int
+    end_x: int
+    end_y: int
+    duration_ms: int
+
+class DragActionEvent(MacroEvent):
+    """Event for drag actions with coordinates"""
+    start_x: int
+    start_y: int
+    end_x: int
+    end_y: int
+    duration_ms: int
+
+class InputTextActionEvent(MacroEvent):
+    """Event for text input actions"""
+    text: str
+
+class KeyPressActionEvent(MacroEvent):
+    """Event for key press actions"""
+    keycode: int
+    key_name: str = ""
+
+class StartAppEvent(MacroEvent):
+    """"Event for starting an app"""
+    package: str
+    activity: str = None

droidrun/agent/context/personas/__init__.py

@@ -1,9 +1,11 @@
 from .default import DEFAULT
 from .ui_expert import UI_EXPERT
 from .app_starter import APP_STARTER_EXPERT
+from .big_agent import BIG_AGENT
 
 __all__ = [
     'DEFAULT',
     'UI_EXPERT',
     'APP_STARTER_EXPERT',
+    'BIG_AGENT',
     ]

droidrun/agent/context/personas/big_agent.py

@@ -0,0 +1,96 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+BIG_AGENT = AgentPersona(
+    name="Big Agent",
+    description="Big Agent. Use this as your Big Agent",
+    expertise_areas=[
+        "UI navigation", "button interactions", "text input", 
+        "menu navigation", "form filling", "scrolling", "app launching"
+    ],
+    allowed_tools=[
+        Tools.swipe.__name__,
+        Tools.input_text.__name__,
+        Tools.press_key.__name__,
+        Tools.drag.__name__,
+        Tools.tap_by_index.__name__,
+        Tools.start_app.__name__,
+        Tools.list_packages.__name__,
+        Tools.remember.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "ui_state",
+        "screenshot",
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **Is the precondition met? What is your reasoning and the next step to address this request?**
+    Explain your thought process then provide code in ```python ... ``` tags if needed.
+    """"",
+
+    system_prompt="""
+    You are a helpful AI assistant that can write and execute Python code to solve problems.
+
+    You will be given a task to perform. You should output:
+    - Python code wrapped in ``` tags that provides the solution to the task, or a step towards the solution.
+    - If there is a precondition for the task, you MUST check if it is met.
+    - If a goal's precondition is unmet, fail the task by calling `complete(success=False, reason='...')` with an explanation.
+    - If you task is complete, you should use the complete(success:bool, reason:str) function within a code block to mark it as finished. The success parameter should be True if the task was completed successfully, and False otherwise. The reason parameter should be a string explaining the reason for failure if failed.
+
+
+    ## Context:
+    The following context is given to you for analysis:
+    - **ui_state**: A list of all currently visible UI elements with their indices. Use this to understand what interactive elements are available on the screen.
+    - **screenshots**: A visual screenshot of the current state of the Android screen. This provides visual context for what the user sees. screenshots won't be saved in the chat history. So, make sure to describe what you see and explain the key parts of your plan in your thoughts, as those will be saved and used to assist you in future steps.
+    - **phone_state**: The current app you are navigating in. This tells you which application context you're working within.
+    - **chat history**: You are also given the history of your actions (if any) from your previous steps.
+    - **execution result**: The result of your last Action
+    NOTE: you don't have access to these inputs in your tool calling context
+
+    ## Response Format:
+    Example of proper code format:
+    **Task Assignment:**
+    **Task:** "Precondition: Settings app is open. Goal: Navigate to Wi-Fi settings and connect to the network 'HomeNetwork'."
+
+    **(Step 1) Agent Analysis:** I can see the Settings app is open from the screenshot. This is a multi-step task that requires me to first navigate to Wi-Fi settings, then ensure Wi-Fi is enabled, and finally connect to 'HomeNetwork'. Let me start by finding and tapping on the Wi-Fi option in the settings menu. Looking at the UI elements, I can see "Wi-Fi" option at index 3.
+
+    **(Step 1) Agent Action:**
+    ```python
+    # First step: Navigate to Wi-Fi settings
+    tap_by_index(3)
+    ```
+
+    **(Step 2) Agent Analysis:** Good! I've successfully navigated to the Wi-Fi settings screen. Now I can see the Wi-Fi settings interface. I notice that Wi-Fi appears to be turned off based on the toggle switch at index 1. I need to turn it on before I can see available networks and connect to 'HomeNetwork'.
+
+    **(Step 2) Agent Action:**
+    ```python
+    # Second step: Turn on Wi-Fi to see available networks
+    tap_by_index(1)
+    ```
+
+    **(Step 3) Agent Analysis:** Excellent! Wi-Fi is now enabled and I can see a list of available networks appearing on the screen. I can see 'HomeNetwork' in the list at index 5. This is the final step - I need to tap on it to initiate the connection, which will complete my assigned task.
+
+    **(Step 3) Agent Action:**
+    ```python
+    # Final step: Connect to the target network
+    tap_by_index(5)
+    complete(success=True, reason="Successfully navigated to Wi-Fi settings and initiated connection to HomeNetwork")
+    ```
+    ```
+
+    ## Tools:
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+
+    ## Final Answer Guidelines:
+    - When providing a final answer, focus on directly answering the user's question in the response format given
+    - Present the results clearly and concisely as if you computed them directly
+    - Structure your response like you're directly answering the user's query, not explaining how you solved it
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+"""
+
+)

droidrun/agent/context/personas/ui_expert.py

@@ -13,6 +13,7 @@ UI_EXPERT = AgentPersona(
         Tools.input_text.__name__,
         Tools.press_key.__name__,
         Tools.tap_by_index.__name__, 
+        Tools.drag.__name__,
         Tools.remember.__name__,
         Tools.complete.__name__
     ],

droidrun/agent/context/task_manager.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Dict
+from typing import List, Dict, Optional
 from dataclasses import dataclass
 import copy
 
@@ -11,6 +11,9 @@ class Task:
     description: str
     status: str
     agent_type: str
+    # Optional fields to carry success/failure context back to the planner
+    message: Optional[str] = None
+    failure_reason: Optional[str] = None
     
 
 class TaskManager:
@@ -40,14 +43,16 @@ class TaskManager:
     def get_task_history(self):
         return self.task_history
 
-    def complete_task(self, task: Task):
+    def complete_task(self, task: Task, message: Optional[str] = None):
         task = copy.deepcopy(task)
         task.status = self.STATUS_COMPLETED
+        task.message = message
         self.task_history.append(task)
 
-    def fail_task(self, task: Task):
+    def fail_task(self, task: Task, failure_reason: Optional[str] = None):
         task = copy.deepcopy(task)
         task.status = self.STATUS_FAILED
+        task.failure_reason = failure_reason
         self.task_history.append(task)
 
     def get_completed_tasks(self) -> list[dict]:

droidrun/agent/droid/droid_agent.py

@@ -16,7 +16,7 @@ from droidrun.agent.planner import PlannerAgent
 from droidrun.agent.context.task_manager import TaskManager
 from droidrun.agent.utils.trajectory import Trajectory
 from droidrun.tools import Tools, describe_tools
-from droidrun.agent.common.events import ScreenshotEvent
+from droidrun.agent.common.events import ScreenshotEvent, MacroEvent
 from droidrun.agent.common.default import MockWorkflow
 from droidrun.agent.context import ContextInjectionManager
 from droidrun.agent.context.agent_persona import AgentPersona
@@ -68,7 +68,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         reflection: bool = False,
         enable_tracing: bool = False,
         debug: bool = False,
-        save_trajectories: bool = False,
+        save_trajectories: str = "none",
+        excluded_tools: List[str] = None,
         *args,
         **kwargs
     ):
@@ -85,8 +86,13 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             reflection: Whether to reflect on steps the CodeActAgent did to give the PlannerAgent advice
             enable_tracing: Whether to enable Arize Phoenix tracing
             debug: Whether to enable verbose debug logging
+            save_trajectories: Trajectory saving level. Can be:
+                - "none" (no saving)
+                - "step" (save per step)
+                - "action" (save per action)
             **kwargs: Additional keyword arguments to pass to the agents
         """
+        self.user_id = kwargs.pop("user_id", None)
         super().__init__(timeout=timeout ,*args,**kwargs)
         # Configure default logging if not already configured
         self._configure_default_logging(debug=debug)
@@ -112,18 +118,33 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         self.debug = debug
 
         self.event_counter = 0
-        self.save_trajectories = save_trajectories
+        # Handle backward compatibility: bool -> str mapping
+        if isinstance(save_trajectories, bool):
+            self.save_trajectories = "step" if save_trajectories else "none"
+        else:
+            # Validate string values
+            valid_values = ["none", "step", "action"]
+            if save_trajectories not in valid_values:
+                logger.warning(f"Invalid save_trajectories value: {save_trajectories}. Using 'none' instead.")
+                self.save_trajectories = "none"
+            else:
+                self.save_trajectories = save_trajectories
         
-        self.trajectory = Trajectory()
+        self.trajectory = Trajectory(goal=goal)
         self.task_manager = TaskManager()
         self.task_iter = None
+
+        
         self.cim = ContextInjectionManager(personas=personas)
         self.current_episodic_memory = None
 
         logger.info("🤖 Initializing DroidAgent...")
+        logger.info(f"💾 Trajectory saving level: {self.save_trajectories}")
         
-        self.tool_list = describe_tools(tools)
+        self.tool_list = describe_tools(tools, excluded_tools)
         self.tools_instance = tools
+        
+        self.tools_instance.save_trajectories = self.save_trajectories
 
 
         if self.reasoning:
@@ -162,17 +183,18 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
                 enable_tracing=enable_tracing,
                 debug=debug,
                 save_trajectories=save_trajectories,
-            )
+            ),
+            self.user_id
         )
 
         
         logger.info("✅ DroidAgent initialized successfully.")
 
-    def run(self) -> WorkflowHandler:
+    def run(self, *args, **kwargs) -> WorkflowHandler:
         """
         Run the DroidAgent workflow.
         """
-        return super().run()
+        return super().run(*args, **kwargs)
     
     @step
     async def execute_task(
@@ -232,16 +254,24 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             return CodeActResultEvent(success=False, reason=f"Error: {str(e)}", task=task, steps=[])
     
     @step
-    async def handle_codeact_execute(self, ctx: Context, ev: CodeActResultEvent) -> FinalizeEvent | ReflectionEvent:
+    async def handle_codeact_execute(self, ctx: Context, ev: CodeActResultEvent) -> FinalizeEvent | ReflectionEvent | ReasoningLogicEvent:
         try:
             task = ev.task
             if not self.reasoning:
                 return FinalizeEvent(success=ev.success, reason=ev.reason, output=ev.reason, task=[task], tasks=[task], steps=ev.steps)
             
-            if self.reflection:
+            if self.reflection and ev.success:
                 return ReflectionEvent(task=task)
-            
-            return ReasoningLogicEvent()
+
+            # Reasoning is enabled but reflection is disabled.
+            # Success: mark complete and proceed to next step in reasoning loop.
+            # Failure: mark failed and trigger planner immediately without advancing to the next queued task.
+            if ev.success:
+                self.task_manager.complete_task(task, message=ev.reason)
+                return ReasoningLogicEvent()
+            else:
+                self.task_manager.fail_task(task, failure_reason=ev.reason)
+                return ReasoningLogicEvent(force_planning=True)
 
         except Exception as e:
             logger.error(f"❌ Error during DroidAgent execution: {e}")
@@ -293,7 +323,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if ev.reflection:
                 handler = planner_agent.run(remembered_info=self.tools_instance.memory, reflection=ev.reflection)
             else:
-                if self.task_iter:
+                if not ev.force_planning and self.task_iter:
                     try:
                         task = next(self.task_iter)
                         return CodeActExecuteEvent(task=task, reflection=None)
@@ -369,7 +399,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
                 success=ev.success,
                 output=ev.output,
                 steps=ev.steps,
-            )
+            ),
+            self.user_id
         )
         flush()
 
@@ -381,7 +412,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             "steps": ev.steps,
         }
 
-        if self.trajectory and self.save_trajectories:
+        if self.trajectory and self.save_trajectories != "none":
             self.trajectory.save_trajectory()
 
         return StopEvent(result)
@@ -391,13 +422,16 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         if isinstance(ev, EpisodicMemoryEvent):
             self.current_episodic_memory = ev.episodic_memory
             return
+        
+  
 
         if not isinstance(ev, StopEvent):
             ctx.write_event_to_stream(ev)
             
             if isinstance(ev, ScreenshotEvent):
                 self.trajectory.screenshots.append(ev.screenshot)
-
+            elif isinstance(ev, MacroEvent):
+                self.trajectory.macro.append(ev)
             else:
                 self.trajectory.events.append(ev)
 

droidrun/agent/droid/events.py

@@ -13,6 +13,7 @@ class CodeActResultEvent(Event):
 
 class ReasoningLogicEvent(Event):
     reflection: Optional[Reflection] = None
+    force_planning: bool = False
 
 class FinalizeEvent(Event):
     success: bool

droidrun/agent/planner/planner_agent.py

@@ -130,13 +130,14 @@ class PlannerAgent(Workflow):
         self.steps_counter += 1
         logger.info(f"🧠 Thinking about how to plan the goal...")
 
+        # if vision is disabled, screenshot should save to trajectory
+        screenshot = (self.tools_instance.take_screenshot())[1]
+        ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
         if self.vision:
-            screenshot = (await self.tools_instance.take_screenshot())[1]
-            ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
             await ctx.set("screenshot", screenshot)
 
         try:
-            state = await self.tools_instance.get_state()
+            state = self.tools_instance.get_state()
             await ctx.set("ui_state", state["a11y_tree"])
             await ctx.set("phone_state", state["phone_state"])
         except Exception as e:
@@ -168,11 +169,15 @@ class PlannerAgent(Workflow):
             try:
                 result = await self.executer.execute(ctx, code)
                 logger.info(f"📝 Planning complete")
-                logger.debug(f"  - Planning code executed. Result: {result}")
+                logger.debug(f"  - Planning code executed. Result: {result['output']}")
+
+                screenshots = result['screenshots']
+                for screenshot in screenshots[:-1]: # the last screenshot will be captured by next step
+                    ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
 
                 await self.chat_memory.aput(
                     ChatMessage(
-                        role="user", content=f"Execution Result:\n```\n{result}\n```"
+                        role="user", content=f"Execution Result:\n```\n{result['output']}\n```"
                     )
                 )
 
@@ -241,15 +246,15 @@ wrap your code inside this:
             logger.debug(f"  - Sending {len(chat_history)} messages to LLM.")
 
             model = self.llm.class_name()
-            if model == "DeepSeek":
-                logger.warning(
-                    "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
-                )
-
-            elif self.vision == True:
-                chat_history = await chat_utils.add_screenshot_image_block(
-                    await ctx.get("screenshot"), chat_history
-                )                   
+            if self.vision == True:
+                if model == "DeepSeek":
+                    logger.warning(
+                        "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
+                    )
+                else:
+                    chat_history = await chat_utils.add_screenshot_image_block(
+                        await ctx.get("screenshot"), chat_history
+                    )                   
 
 
 

droidrun/agent/utils/chat_utils.py

@@ -208,7 +208,7 @@ async def add_task_history_block(completed_tasks: list[dict], failed_tasks: list
     all_tasks = completed_tasks + failed_tasks
     
     if all_tasks:
-        task_history += "Task History (chronological order):\n"
+        task_history += "### Task Execution History (chronological):\n"
         for i, task in enumerate(all_tasks, 1):
             if hasattr(task, 'description'):
                 status_indicator = "[success]" if hasattr(task, 'status') and task.status == "completed" else "[failed]"

droidrun/agent/utils/executer.py

@@ -9,6 +9,7 @@ from llama_index.core.workflow import Context
 import asyncio
 from asyncio import AbstractEventLoop
 import threading
+from droidrun.tools.adb import AdbTools
 
 logger = logging.getLogger("droidrun")
 
@@ -29,6 +30,7 @@ class SimpleCodeExecutor:
         locals: Dict[str, Any] = {},
         globals: Dict[str, Any] = {},
         tools={},
+        tools_instance=None,
         use_same_scope: bool = True,
     ):
         """
@@ -38,8 +40,11 @@ class SimpleCodeExecutor:
             locals: Local variables to use in the execution context
             globals: Global variables to use in the execution context
             tools: List of tools available for execution
+            tools_instance: Original tools instance (e.g., AdbTools instance)
         """
 
+        self.tools_instance = tools_instance
+
         # loop throught tools and add them to globals, but before that check if tool value is async, if so convert it to sync. tools is a dictionary of tool name: function
         # e.g. tools = {'tool_name': tool_function}
 
@@ -74,6 +79,7 @@ class SimpleCodeExecutor:
         self.locals = locals
         self.loop = loop
         self.use_same_scope = use_same_scope
+        self.tools = tools
         if self.use_same_scope:
             # If using the same scope, set the globals and locals to the same dictionary
             self.globals = self.locals = {
@@ -93,7 +99,12 @@ class SimpleCodeExecutor:
         """
         # Update UI elements before execution
         self.globals['ui_state'] = await ctx.get("ui_state", None)
+        self.globals['step_screenshots'] = []
+        self.globals['step_ui_states'] = []
         
+        if self.tools_instance and isinstance(self.tools_instance, AdbTools):
+            self.tools_instance._set_context(ctx)
+
         # Capture stdout and stderr
         stdout = io.StringIO()
         stderr = io.StringIO()
@@ -129,4 +140,9 @@ class SimpleCodeExecutor:
             output = f"Error: {type(e).__name__}: {str(e)}\n"
             output += traceback.format_exc()
 
-        return output
+        result = {
+            'output': output,
+            'screenshots': self.globals['step_screenshots'],
+            'ui_states': self.globals['step_ui_states']
+        }
+        return result