droidrun 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

droidrun/__init__.py

@@ -5,27 +5,22 @@ DroidRun - A framework for controlling Android devices through LLM agents.
 __version__ = "0.3.0"
 
 # Import main classes for easier access
-from droidrun.agent.codeact.codeact_agent import CodeActAgent
-from droidrun.agent.planner.planner_agent import PlannerAgent
-from droidrun.agent.utils.executer import SimpleCodeExecutor
 from droidrun.agent.utils.llm_picker import load_llm
-from droidrun.adb.manager import DeviceManager
-from droidrun.tools.tools import Tools
-from droidrun.tools.adb import AdbTools
-from droidrun.tools.ios import IOSTools
+from droidrun.tools import Tools, AdbTools, IOSTools
 from droidrun.agent.droid import DroidAgent
 
+# Import macro functionality
+from droidrun.macro import MacroPlayer, replay_macro_file, replay_macro_folder
+
 
 # Make main components available at package level
 __all__ = [
     "DroidAgent",
-    "CodeActAgent",
-    "PlannerAgent",
-    "DeviceManager",
-    "Tools",
     "load_llm",
-    "SimpleCodeExecutor",
     "Tools",
     "AdbTools",
     "IOSTools",
+    "MacroPlayer",
+    "replay_macro_file",
+    "replay_macro_folder",
 ]

droidrun/agent/codeact/codeact_agent.py

@@ -97,6 +97,7 @@ class CodeActAgent(Workflow):
             loop=asyncio.get_event_loop(),
             locals={},
             tools=self.tool_list,
+            tools_instance=tools_instance,
             globals={"__builtins__": __builtins__},
         )
 
@@ -169,7 +170,7 @@ class CodeActAgent(Workflow):
                     "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
                 )
             elif self.vision == True and context == "screenshot":
-                screenshot = (await self.tools.take_screenshot())[1]
+                screenshot = (self.tools.take_screenshot())[1]
                 ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
 
                 await ctx.set("screenshot", screenshot)
@@ -177,7 +178,7 @@ class CodeActAgent(Workflow):
 
             if context == "ui_state":
                 try:
-                    state = await self.tools.get_state()
+                    state = self.tools.get_state()
                     await ctx.set("ui_state", state["a11y_tree"])
                     chat_history = await chat_utils.add_ui_text_block(
                         state["a11y_tree"], chat_history
@@ -189,7 +190,7 @@ class CodeActAgent(Workflow):
 
             if context == "packages":
                 chat_history = await chat_utils.add_packages_block(
-                    await self.tools.list_packages(include_system_apps=True),
+                    self.tools.list_packages(include_system_apps=True),
                     chat_history,
                 )
 
@@ -242,7 +243,7 @@ class CodeActAgent(Workflow):
         code = ev.code
         assert code, "Code cannot be empty."
         logger.info(f"⚡ Executing action...")
-        logger.debug(f"Code to execute:\n```python\n{code}\n```")
+        logger.info(f"Code to execute:\n```python\n{code}\n```")
 
         try:
             self.code_exec_counter += 1
@@ -312,6 +313,7 @@ class CodeActAgent(Workflow):
             {
                 "success": ev.success,
                 "reason": ev.reason,
+                "output": ev.reason,
                 "codeact_steps": self.steps_counter,
                 "code_executions": self.code_exec_counter,
             }
@@ -321,7 +323,7 @@ class CodeActAgent(Workflow):
             EpisodicMemoryEvent(episodic_memory=self.episodic_memory)
         )
 
-        return StopEvent(result=result)
+        return StopEvent(result)
 
     async def _get_llm_response(
         self, ctx: Context, chat_history: List[ChatMessage]
@@ -397,13 +399,13 @@ class CodeActAgent(Workflow):
             ui_state = None
             
             try:
-                _, screenshot_bytes = await self.tools.take_screenshot()
+                _, screenshot_bytes = self.tools.take_screenshot()
                 screenshot = screenshot_bytes
             except Exception as e:
                 logger.warning(f"Failed to capture final screenshot: {e}")
             
             try:
-                (a11y_tree, phone_state) = await self.tools.get_state()
+                (a11y_tree, phone_state) = self.tools.get_state()
             except Exception as e:
                 logger.warning(f"Failed to capture final UI state: {e}")
             

droidrun/agent/common/events.py

@@ -1,4 +1,47 @@
 from llama_index.core.workflow import Event
 
 class ScreenshotEvent(Event):
-    screenshot: bytes
+    screenshot: bytes
+
+class MacroEvent(Event):
+    """Base class for coordinate-based action events"""
+    action_type: str
+    description: str
+
+class TapActionEvent(MacroEvent):
+    """Event for tap actions with coordinates"""
+    x: int
+    y: int
+    element_index: int = None
+    element_text: str = ""
+    element_bounds: str = ""
+    
+class SwipeActionEvent(MacroEvent):
+    """Event for swipe actions with coordinates"""
+    start_x: int
+    start_y: int
+    end_x: int
+    end_y: int
+    duration_ms: int
+
+class DragActionEvent(MacroEvent):
+    """Event for drag actions with coordinates"""
+    start_x: int
+    start_y: int
+    end_x: int
+    end_y: int
+    duration_ms: int
+
+class InputTextActionEvent(MacroEvent):
+    """Event for text input actions"""
+    text: str
+
+class KeyPressActionEvent(MacroEvent):
+    """Event for key press actions"""
+    keycode: int
+    key_name: str = ""
+
+class StartAppEvent(MacroEvent):
+    """"Event for starting an app"""
+    package: str
+    activity: str = None

droidrun/agent/context/personas/__init__.py

@@ -1,11 +1,11 @@
 from .default import DEFAULT
 from .ui_expert import UI_EXPERT
 from .app_starter import APP_STARTER_EXPERT
-from .extractor import EXTRACTOR
+from .big_agent import BIG_AGENT
 
 __all__ = [
     'DEFAULT',
     'UI_EXPERT',
     'APP_STARTER_EXPERT',
-    'EXTRACTOR'
+    'BIG_AGENT',
     ]

droidrun/agent/context/personas/big_agent.py

@@ -0,0 +1,96 @@
+from droidrun.agent.context.agent_persona import AgentPersona
+from droidrun.tools import Tools
+
+BIG_AGENT = AgentPersona(
+    name="Big Agent",
+    description="Big Agent. Use this as your Big Agent",
+    expertise_areas=[
+        "UI navigation", "button interactions", "text input", 
+        "menu navigation", "form filling", "scrolling", "app launching"
+    ],
+    allowed_tools=[
+        Tools.swipe.__name__,
+        Tools.input_text.__name__,
+        Tools.press_key.__name__,
+        Tools.drag.__name__,
+        Tools.tap_by_index.__name__,
+        Tools.start_app.__name__,
+        Tools.list_packages.__name__,
+        Tools.remember.__name__,
+        Tools.complete.__name__
+    ],
+    required_context=[
+        "ui_state",
+        "screenshot",
+    ],
+    user_prompt="""
+    **Current Request:**
+    {goal}
+    **Is the precondition met? What is your reasoning and the next step to address this request?**
+    Explain your thought process then provide code in ```python ... ``` tags if needed.
+    """"",
+
+    system_prompt="""
+    You are a helpful AI assistant that can write and execute Python code to solve problems.
+
+    You will be given a task to perform. You should output:
+    - Python code wrapped in ``` tags that provides the solution to the task, or a step towards the solution.
+    - If there is a precondition for the task, you MUST check if it is met.
+    - If a goal's precondition is unmet, fail the task by calling `complete(success=False, reason='...')` with an explanation.
+    - If you task is complete, you should use the complete(success:bool, reason:str) function within a code block to mark it as finished. The success parameter should be True if the task was completed successfully, and False otherwise. The reason parameter should be a string explaining the reason for failure if failed.
+
+
+    ## Context:
+    The following context is given to you for analysis:
+    - **ui_state**: A list of all currently visible UI elements with their indices. Use this to understand what interactive elements are available on the screen.
+    - **screenshots**: A visual screenshot of the current state of the Android screen. This provides visual context for what the user sees. screenshots won't be saved in the chat history. So, make sure to describe what you see and explain the key parts of your plan in your thoughts, as those will be saved and used to assist you in future steps.
+    - **phone_state**: The current app you are navigating in. This tells you which application context you're working within.
+    - **chat history**: You are also given the history of your actions (if any) from your previous steps.
+    - **execution result**: The result of your last Action
+    NOTE: you don't have access to these inputs in your tool calling context
+
+    ## Response Format:
+    Example of proper code format:
+    **Task Assignment:**
+    **Task:** "Precondition: Settings app is open. Goal: Navigate to Wi-Fi settings and connect to the network 'HomeNetwork'."
+
+    **(Step 1) Agent Analysis:** I can see the Settings app is open from the screenshot. This is a multi-step task that requires me to first navigate to Wi-Fi settings, then ensure Wi-Fi is enabled, and finally connect to 'HomeNetwork'. Let me start by finding and tapping on the Wi-Fi option in the settings menu. Looking at the UI elements, I can see "Wi-Fi" option at index 3.
+
+    **(Step 1) Agent Action:**
+    ```python
+    # First step: Navigate to Wi-Fi settings
+    tap_by_index(3)
+    ```
+
+    **(Step 2) Agent Analysis:** Good! I've successfully navigated to the Wi-Fi settings screen. Now I can see the Wi-Fi settings interface. I notice that Wi-Fi appears to be turned off based on the toggle switch at index 1. I need to turn it on before I can see available networks and connect to 'HomeNetwork'.
+
+    **(Step 2) Agent Action:**
+    ```python
+    # Second step: Turn on Wi-Fi to see available networks
+    tap_by_index(1)
+    ```
+
+    **(Step 3) Agent Analysis:** Excellent! Wi-Fi is now enabled and I can see a list of available networks appearing on the screen. I can see 'HomeNetwork' in the list at index 5. This is the final step - I need to tap on it to initiate the connection, which will complete my assigned task.
+
+    **(Step 3) Agent Action:**
+    ```python
+    # Final step: Connect to the target network
+    tap_by_index(5)
+    complete(success=True, reason="Successfully navigated to Wi-Fi settings and initiated connection to HomeNetwork")
+    ```
+    ```
+
+    ## Tools:
+    In addition to the Python Standard Library and any functions you have already written, you can use the following functions:
+    {tool_descriptions}
+
+
+    ## Final Answer Guidelines:
+    - When providing a final answer, focus on directly answering the user's question in the response format given
+    - Present the results clearly and concisely as if you computed them directly
+    - Structure your response like you're directly answering the user's query, not explaining how you solved it
+
+    Reminder: Always place your Python code between ```...``` tags when you want to run code. 
+"""
+
+)

droidrun/agent/context/personas/ui_expert.py

@@ -13,6 +13,7 @@ UI_EXPERT = AgentPersona(
         Tools.input_text.__name__,
         Tools.press_key.__name__,
         Tools.tap_by_index.__name__, 
+        Tools.drag.__name__,
         Tools.remember.__name__,
         Tools.complete.__name__
     ],

droidrun/agent/droid/droid_agent.py

@@ -8,6 +8,7 @@ from typing import List
 
 from llama_index.core.llms.llm import LLM
 from llama_index.core.workflow import step, StartEvent, StopEvent, Workflow, Context
+from llama_index.core.workflow.handler import WorkflowHandler
 from droidrun.agent.droid.events import *
 from droidrun.agent.codeact import CodeActAgent
 from droidrun.agent.codeact.events import EpisodicMemoryEvent
@@ -15,12 +16,13 @@ from droidrun.agent.planner import PlannerAgent
 from droidrun.agent.context.task_manager import TaskManager
 from droidrun.agent.utils.trajectory import Trajectory
 from droidrun.tools import Tools, describe_tools
-from droidrun.agent.common.events import ScreenshotEvent
+from droidrun.agent.common.events import ScreenshotEvent, MacroEvent
 from droidrun.agent.common.default import MockWorkflow
 from droidrun.agent.context import ContextInjectionManager
 from droidrun.agent.context.agent_persona import AgentPersona
 from droidrun.agent.context.personas import DEFAULT
 from droidrun.agent.oneflows.reflector import Reflector
+from droidrun.telemetry import capture, flush, DroidAgentInitEvent, DroidAgentFinalizeEvent
 
 
 logger = logging.getLogger("droidrun")
@@ -67,6 +69,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         enable_tracing: bool = False,
         debug: bool = False,
         save_trajectories: bool = False,
+        excluded_tools: List[str] = None,
         *args,
         **kwargs
     ):
@@ -85,8 +88,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             debug: Whether to enable verbose debug logging
             **kwargs: Additional keyword arguments to pass to the agents
         """
+        self.user_id = kwargs.pop("user_id", None)
         super().__init__(timeout=timeout ,*args,**kwargs)
-        
         # Configure default logging if not already configured
         self._configure_default_logging(debug=debug)
         
@@ -113,15 +116,17 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         self.event_counter = 0
         self.save_trajectories = save_trajectories
         
-        self.trajectory = Trajectory()
+        self.trajectory = Trajectory(goal=goal)
         self.task_manager = TaskManager()
         self.task_iter = None
+
+        
         self.cim = ContextInjectionManager(personas=personas)
         self.current_episodic_memory = None
 
         logger.info("🤖 Initializing DroidAgent...")
         
-        self.tool_list = describe_tools(tools)
+        self.tool_list = describe_tools(tools, excluded_tools)
         self.tools_instance = tools
 
 
@@ -146,8 +151,33 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         else:
             logger.debug("🚫 Planning disabled - will execute tasks directly with CodeActAgent")
             self.planner_agent = None
+
+        capture(
+            DroidAgentInitEvent(
+                goal=goal,
+                llm=llm.class_name(),
+                tools=",".join(self.tool_list),
+                personas=",".join([p.name for p in personas]),
+                max_steps=max_steps,
+                timeout=timeout,
+                vision=vision,
+                reasoning=reasoning,
+                reflection=reflection,
+                enable_tracing=enable_tracing,
+                debug=debug,
+                save_trajectories=save_trajectories,
+            ),
+            self.user_id
+        )
+
         
         logger.info("✅ DroidAgent initialized successfully.")
+
+    def run(self) -> WorkflowHandler:
+        """
+        Run the DroidAgent workflow.
+        """
+        return super().run()
     
     @step
     async def execute_task(
@@ -211,7 +241,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         try:
             task = ev.task
             if not self.reasoning:
-                return FinalizeEvent(success=ev.success, reason=ev.reason, task=[task], steps=ev.steps)
+                return FinalizeEvent(success=ev.success, reason=ev.reason, output=ev.reason, task=[task], tasks=[task], steps=ev.steps)
             
             if self.reflection:
                 return ReflectionEvent(task=task)
@@ -223,7 +253,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if self.debug:
                 import traceback
                 logger.error(traceback.format_exc())
-            return FinalizeEvent(success=False, reason=str(e), task=self.task_manager.get_task_history(), steps=self.step_counter)
+            tasks = self.task_manager.get_task_history()
+            return FinalizeEvent(success=False, reason=str(e), output=str(e), task=tasks, tasks=tasks, steps=self.step_counter)
         
 
     @step
@@ -259,7 +290,9 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         ) -> FinalizeEvent | CodeActExecuteEvent:
         try:
             if self.step_counter >= self.max_steps:
-                return FinalizeEvent(success=False, reason=f"Reached maximum number of steps ({self.max_steps})", task=self.task_manager.get_task_history(), steps=self.step_counter)
+                output = f"Reached maximum number of steps ({self.max_steps})"
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=False, reason=output, output=output, task=tasks, tasks=tasks, steps=self.step_counter)
             self.step_counter += 1
 
             if ev.reflection:
@@ -286,10 +319,13 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
 
             if self.task_manager.goal_completed:
                 logger.info(f"✅ Goal completed: {self.task_manager.message}")
-                return FinalizeEvent(success=True, reason=self.task_manager.message, task=self.task_manager.get_task_history(), steps=self.step_counter)
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=True, reason=self.task_manager.message, output=self.task_manager.message, task=tasks, tasks=tasks, steps=self.step_counter)
             if not self.tasks:
                 logger.warning("No tasks generated by planner")
-                return FinalizeEvent(success=False, reason="Planner did not generate any tasks", task=self.task_manager.get_task_history(), steps=self.step_counter)
+                output = "Planner did not generate any tasks"
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=False, reason=output, output=output, task=tasks, tasks=tasks, steps=self.step_counter)
             
             return CodeActExecuteEvent(task=next(self.task_iter), reflection=None)
         
@@ -298,7 +334,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if self.debug:
                 import traceback
                 logger.error(traceback.format_exc())
-            return FinalizeEvent(success=False, reason=str(e), task=self.task_manager.get_task_history(), steps=self.step_counter)
+            tasks = self.task_manager.get_task_history()
+            return FinalizeEvent(success=False, reason=str(e), output=str(e), task=tasks, tasks=tasks, steps=self.step_counter)
     
 
     @step
@@ -331,10 +368,22 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
     @step
     async def finalize(self, ctx: Context, ev: FinalizeEvent) -> StopEvent:
         ctx.write_event_to_stream(ev)
+        capture(
+            DroidAgentFinalizeEvent(
+                tasks=",".join([f"{t.agent_type}:{t.description}" for t in ev.task]),
+                success=ev.success,
+                output=ev.output,
+                steps=ev.steps,
+            ),
+            self.user_id
+        )
+        flush()
 
         result = {
             "success": ev.success,
+            # deprecated. use output instead.
             "reason": ev.reason,
+            "output": ev.output,
             "steps": ev.steps,
         }
 
@@ -348,13 +397,16 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         if isinstance(ev, EpisodicMemoryEvent):
             self.current_episodic_memory = ev.episodic_memory
             return
+        
+  
 
         if not isinstance(ev, StopEvent):
             ctx.write_event_to_stream(ev)
             
             if isinstance(ev, ScreenshotEvent):
                 self.trajectory.screenshots.append(ev.screenshot)
-
+            elif isinstance(ev, MacroEvent):
+                self.trajectory.macro.append(ev)
             else:
                 self.trajectory.events.append(ev)
 

droidrun/agent/droid/events.py

@@ -16,8 +16,12 @@ class ReasoningLogicEvent(Event):
 
 class FinalizeEvent(Event):
     success: bool
+    # deprecated. use output instead.
     reason: str
+    output: str
+    # deprecated. use tasks instead.
     task: List[Task]
+    tasks: List[Task]
     steps: int = 1
 
 class TaskRunnerEvent(Event):

droidrun/agent/planner/planner_agent.py

@@ -131,12 +131,12 @@ class PlannerAgent(Workflow):
         logger.info(f"🧠 Thinking about how to plan the goal...")
 
         if self.vision:
-            screenshot = (await self.tools_instance.take_screenshot())[1]
+            screenshot = (self.tools_instance.take_screenshot())[1]
             ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
             await ctx.set("screenshot", screenshot)
 
         try:
-            state = await self.tools_instance.get_state()
+            state = self.tools_instance.get_state()
             await ctx.set("ui_state", state["a11y_tree"])
             await ctx.set("phone_state", state["phone_state"])
         except Exception as e:

droidrun/agent/utils/executer.py

@@ -9,6 +9,7 @@ from llama_index.core.workflow import Context
 import asyncio
 from asyncio import AbstractEventLoop
 import threading
+from droidrun.tools.adb import AdbTools
 
 logger = logging.getLogger("droidrun")
 
@@ -29,6 +30,7 @@ class SimpleCodeExecutor:
         locals: Dict[str, Any] = {},
         globals: Dict[str, Any] = {},
         tools={},
+        tools_instance=None,
         use_same_scope: bool = True,
     ):
         """
@@ -38,8 +40,11 @@ class SimpleCodeExecutor:
             locals: Local variables to use in the execution context
             globals: Global variables to use in the execution context
             tools: List of tools available for execution
+            tools_instance: Original tools instance (e.g., AdbTools instance)
         """
 
+        self.tools_instance = tools_instance
+
         # loop throught tools and add them to globals, but before that check if tool value is async, if so convert it to sync. tools is a dictionary of tool name: function
         # e.g. tools = {'tool_name': tool_function}
 
@@ -74,6 +79,7 @@ class SimpleCodeExecutor:
         self.locals = locals
         self.loop = loop
         self.use_same_scope = use_same_scope
+        self.tools = tools
         if self.use_same_scope:
             # If using the same scope, set the globals and locals to the same dictionary
             self.globals = self.locals = {
@@ -93,8 +99,10 @@ class SimpleCodeExecutor:
         """
         # Update UI elements before execution
         self.globals['ui_state'] = await ctx.get("ui_state", None)
-        
-        # Capture stdout and stderr
+
+        if self.tools_instance and isinstance(self.tools_instance, AdbTools):
+            self.tools_instance._set_context(ctx)
+
         stdout = io.StringIO()
         stderr = io.StringIO()
 

droidrun/agent/utils/llm_picker.py

@@ -31,6 +31,7 @@ def load_llm(provider_name: str, **kwargs: Any) -> LLM:
         raise ValueError("provider_name cannot be empty.")
     if provider_name == "OpenAILike":
         module_provider_part = "openai_like"
+        kwargs.setdefault("is_chat_model", True)
     elif provider_name == "GoogleGenAI":
         module_provider_part = "google_genai"
     else: