droidrun 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

droidrun/agent/droid/droid_agent.py

@@ -8,6 +8,7 @@ from typing import List
 
 from llama_index.core.llms.llm import LLM
 from llama_index.core.workflow import step, StartEvent, StopEvent, Workflow, Context
+from llama_index.core.workflow.handler import WorkflowHandler
 from droidrun.agent.droid.events import *
 from droidrun.agent.codeact import CodeActAgent
 from droidrun.agent.codeact.events import EpisodicMemoryEvent
@@ -21,6 +22,7 @@ from droidrun.agent.context import ContextInjectionManager
 from droidrun.agent.context.agent_persona import AgentPersona
 from droidrun.agent.context.personas import DEFAULT
 from droidrun.agent.oneflows.reflector import Reflector
+from droidrun.telemetry import capture, flush, DroidAgentInitEvent, DroidAgentFinalizeEvent
 
 
 logger = logging.getLogger("droidrun")
@@ -61,6 +63,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         personas: List[AgentPersona] = [DEFAULT],
         max_steps: int = 15,
         timeout: int = 1000,
+        vision: bool = False,
         reasoning: bool = False,
         reflection: bool = False,
         enable_tracing: bool = False,
@@ -85,7 +88,6 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             **kwargs: Additional keyword arguments to pass to the agents
         """
         super().__init__(timeout=timeout ,*args,**kwargs)
-        
         # Configure default logging if not already configured
         self._configure_default_logging(debug=debug)
         
@@ -101,6 +103,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
 
         self.goal = goal
         self.llm = llm
+        self.vision = vision
         self.max_steps = max_steps
         self.max_codeact_steps = max_steps
         self.timeout = timeout
@@ -128,6 +131,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             self.planner_agent = PlannerAgent(
                 goal=goal,
                 llm=llm,
+                vision=vision,
                 personas=personas,
                 task_manager=self.task_manager,
                 tools_instance=tools,
@@ -143,8 +147,32 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         else:
             logger.debug("🚫 Planning disabled - will execute tasks directly with CodeActAgent")
             self.planner_agent = None
+
+        capture(
+            DroidAgentInitEvent(
+                goal=goal,
+                llm=llm.class_name(),
+                tools=",".join(self.tool_list),
+                personas=",".join([p.name for p in personas]),
+                max_steps=max_steps,
+                timeout=timeout,
+                vision=vision,
+                reasoning=reasoning,
+                reflection=reflection,
+                enable_tracing=enable_tracing,
+                debug=debug,
+                save_trajectories=save_trajectories,
+            )
+        )
+
         
         logger.info("✅ DroidAgent initialized successfully.")
+
+    def run(self) -> WorkflowHandler:
+        """
+        Run the DroidAgent workflow.
+        """
+        return super().run()
     
     @step
     async def execute_task(
@@ -171,6 +199,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             codeact_agent = CodeActAgent(
                 llm=self.llm,
                 persona=persona,
+                vision=self.vision,
                 max_steps=self.max_codeact_steps,
                 all_tools_list=self.tool_list,
                 tools_instance=self.tools_instance,
@@ -200,14 +229,14 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if self.debug:
                 import traceback
                 logger.error(traceback.format_exc())
-            return CodeActResultEvent(success=False, reason=f"Error: {str(e)}", task=task, steps=result["codeact_steps"])
+            return CodeActResultEvent(success=False, reason=f"Error: {str(e)}", task=task, steps=[])
     
     @step
     async def handle_codeact_execute(self, ctx: Context, ev: CodeActResultEvent) -> FinalizeEvent | ReflectionEvent:
         try:
             task = ev.task
             if not self.reasoning:
-                return FinalizeEvent(success=ev.success, reason=ev.reason, task=[task], steps=ev.steps)
+                return FinalizeEvent(success=ev.success, reason=ev.reason, output=ev.reason, task=[task], tasks=[task], steps=ev.steps)
             
             if self.reflection:
                 return ReflectionEvent(task=task)
@@ -219,7 +248,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if self.debug:
                 import traceback
                 logger.error(traceback.format_exc())
-            return FinalizeEvent(success=False, reason=str(e), task=self.task_manager.get_task_history(), steps=self.step_counter)
+            tasks = self.task_manager.get_task_history()
+            return FinalizeEvent(success=False, reason=str(e), output=str(e), task=tasks, tasks=tasks, steps=self.step_counter)
         
 
     @step
@@ -255,7 +285,9 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
         ) -> FinalizeEvent | CodeActExecuteEvent:
         try:
             if self.step_counter >= self.max_steps:
-                return FinalizeEvent(success=False, reason=f"Reached maximum number of steps ({self.max_steps})", task=self.task_manager.get_task_history(), steps=self.step_counter)
+                output = f"Reached maximum number of steps ({self.max_steps})"
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=False, reason=output, output=output, task=tasks, tasks=tasks, steps=self.step_counter)
             self.step_counter += 1
 
             if ev.reflection:
@@ -282,10 +314,13 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
 
             if self.task_manager.goal_completed:
                 logger.info(f"✅ Goal completed: {self.task_manager.message}")
-                return FinalizeEvent(success=True, reason=self.task_manager.message, task=self.task_manager.get_task_history(), steps=self.step_counter)
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=True, reason=self.task_manager.message, output=self.task_manager.message, task=tasks, tasks=tasks, steps=self.step_counter)
             if not self.tasks:
                 logger.warning("No tasks generated by planner")
-                return FinalizeEvent(success=False, reason="Planner did not generate any tasks", task=self.task_manager.get_task_history(), steps=self.step_counter)
+                output = "Planner did not generate any tasks"
+                tasks = self.task_manager.get_task_history()
+                return FinalizeEvent(success=False, reason=output, output=output, task=tasks, tasks=tasks, steps=self.step_counter)
             
             return CodeActExecuteEvent(task=next(self.task_iter), reflection=None)
         
@@ -294,7 +329,8 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             if self.debug:
                 import traceback
                 logger.error(traceback.format_exc())
-            return FinalizeEvent(success=False, reason=str(e), task=self.task_manager.get_task_history(), steps=self.step_counter)
+            tasks = self.task_manager.get_task_history()
+            return FinalizeEvent(success=False, reason=str(e), output=str(e), task=tasks, tasks=tasks, steps=self.step_counter)
     
 
     @step
@@ -306,6 +342,7 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
             Dict containing the execution result
         """
         logger.info(f"🚀 Running DroidAgent to achieve goal: {self.goal}")
+        ctx.write_event_to_stream(ev)
         
         self.step_counter = 0
         self.retry_counter = 0
@@ -326,10 +363,21 @@ A wrapper class that coordinates between PlannerAgent (creates plans) and
     @step
     async def finalize(self, ctx: Context, ev: FinalizeEvent) -> StopEvent:
         ctx.write_event_to_stream(ev)
+        capture(
+            DroidAgentFinalizeEvent(
+                tasks=",".join([f"{t.agent_type}:{t.description}" for t in ev.task]),
+                success=ev.success,
+                output=ev.output,
+                steps=ev.steps,
+            )
+        )
+        flush()
 
         result = {
             "success": ev.success,
+            # deprecated. use output instead.
             "reason": ev.reason,
+            "output": ev.output,
             "steps": ev.steps,
         }
 

droidrun/agent/droid/events.py

@@ -16,8 +16,12 @@ class ReasoningLogicEvent(Event):
 
 class FinalizeEvent(Event):
     success: bool
+    # deprecated. use output instead.
     reason: str
+    output: str
+    # deprecated. use tasks instead.
     task: List[Task]
+    tasks: List[Task]
     steps: int = 1
 
 class TaskRunnerEvent(Event):

droidrun/agent/planner/planner_agent.py

@@ -42,6 +42,7 @@ class PlannerAgent(Workflow):
         self,
         goal: str,
         llm: LLM,
+        vision: bool,
         personas: List[AgentPersona],
         task_manager: TaskManager,
         tools_instance: Tools,
@@ -57,6 +58,7 @@ class PlannerAgent(Workflow):
         self.goal = goal
         self.task_manager = task_manager
         self.debug = debug
+        self.vision = vision
 
         self.chat_memory = None
         self.remembered_info = None
@@ -128,12 +130,19 @@ class PlannerAgent(Workflow):
         self.steps_counter += 1
         logger.info(f"🧠 Thinking about how to plan the goal...")
 
-        screenshot = (await self.tools_instance.take_screenshot())[1]
-        ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
-        await ctx.set("screenshot", screenshot)
+        if self.vision:
+            screenshot = (await self.tools_instance.take_screenshot())[1]
+            ctx.write_event_to_stream(ScreenshotEvent(screenshot=screenshot))
+            await ctx.set("screenshot", screenshot)
+
+        try:
+            state = await self.tools_instance.get_state()
+            await ctx.set("ui_state", state["a11y_tree"])
+            await ctx.set("phone_state", state["phone_state"])
+        except Exception as e:
+            logger.warning(f"⚠️ Error retrieving state from the connected device. Is the Accessibility Service enabled?")
+
 
-        await ctx.set("ui_state", await self.tools_instance.get_clickables())
-        await ctx.set("phone_state", await self.tools_instance.get_phone_state())
         await ctx.set("remembered_info", self.remembered_info)
         await ctx.set("reflection", self.reflection)
 
@@ -187,7 +196,11 @@ class PlannerAgent(Workflow):
                 await self.chat_memory.aput(
                     ChatMessage(
                         role="user",
-                        content=f"Please either set new tasks using set_tasks_with_agents() or mark the goal as complete using complete_goal() if done.",
+                        content="""Please either set new tasks using set_tasks_with_agents() or mark the goal as complete using complete_goal() if done.
+wrap your code inside this:
+```python
+<YOUR CODE HERE>
+```""",
                     )
                 )
                 logger.debug("🔄 Waiting for next plan or completion.")
@@ -196,7 +209,11 @@ class PlannerAgent(Workflow):
             await self.chat_memory.aput(
                 ChatMessage(
                     role="user",
-                    content=f"Please either set new tasks using set_tasks_with_agents() or mark the goal as complete using complete_goal() if done.",
+                    content="""Please either set new tasks using set_tasks_with_agents() or mark the goal as complete using complete_goal() if done.
+wrap your code inside this:
+```python
+<YOUR CODE HERE>
+```""",
                 )
             )
             logger.debug("🔄 Waiting for next plan or completion.")
@@ -224,15 +241,18 @@ class PlannerAgent(Workflow):
             logger.debug(f"  - Sending {len(chat_history)} messages to LLM.")
 
             model = self.llm.class_name()
-            if model != "DeepSeek":
-                chat_history = await chat_utils.add_screenshot_image_block(
-                    await ctx.get("screenshot"), chat_history
-                )
-            else:
+            if model == "DeepSeek":
                 logger.warning(
                     "[yellow]DeepSeek doesnt support images. Disabling screenshots[/]"
                 )
 
+            elif self.vision == True:
+                chat_history = await chat_utils.add_screenshot_image_block(
+                    await ctx.get("screenshot"), chat_history
+                )                   
+
+
+
             chat_history = await chat_utils.add_task_history_block(
                 self.task_manager.get_completed_tasks(),
                 self.task_manager.get_failed_tasks(),

droidrun/agent/utils/chat_utils.py

@@ -132,24 +132,21 @@ async def add_phone_state_block(phone_state, chat_history: List[ChatMessage]) ->
     
     # Format the phone state data nicely
     if isinstance(phone_state, dict) and 'error' not in phone_state:
-        current_app = phone_state.get('currentApp', 'Unknown')
+        current_app = phone_state.get('currentApp', '')
         package_name = phone_state.get('packageName', 'Unknown')
         keyboard_visible = phone_state.get('keyboardVisible', False)
         focused_element = phone_state.get('focusedElement')
         
         # Format the focused element
         if focused_element:
-            element_text = focused_element.get('text', 'No text')
-            element_class = focused_element.get('className', 'Unknown')
-            element_bounds = focused_element.get('bounds', 'Unknown')
-            element_type = focused_element.get('type', 'unknown')
+            element_text = focused_element.get('text', '')
+            element_class = focused_element.get('className', '')
             element_resource_id = focused_element.get('resourceId', '')
             
             # Build focused element description
-            focused_desc = f"'{element_text}' ({element_class})"
+            focused_desc = f"'{element_text}' {element_class}"
             if element_resource_id:
                 focused_desc += f" | ID: {element_resource_id}"
-            focused_desc += f" | Bounds: {element_bounds} | Type: {element_type}"
         else:
             focused_desc = "None"
         

droidrun/agent/utils/llm_picker.py

@@ -31,6 +31,7 @@ def load_llm(provider_name: str, **kwargs: Any) -> LLM:
         raise ValueError("provider_name cannot be empty.")
     if provider_name == "OpenAILike":
         module_provider_part = "openai_like"
+        kwargs.setdefault("is_chat_model", True)
     elif provider_name == "GoogleGenAI":
         module_provider_part = "google_genai"
     else: