droidrun 0.3.10.dev3__py3-none-any.whl → 0.3.10.dev4__py3-none-any.whl

droidrun/agent/droid/droid_agent.py

@@ -16,14 +16,9 @@ from llama_index.core.workflow import Context, StartEvent, StopEvent, Workflow,
 from llama_index.core.workflow.handler import WorkflowHandler
 from workflows.events import Event
 
-from droidrun.config_manager.config_manager import VisionConfig
-
 from droidrun.agent.codeact import CodeActAgent
 from droidrun.agent.codeact.events import EpisodicMemoryEvent
 from droidrun.agent.common.events import MacroEvent, RecordUIStateEvent, ScreenshotEvent
-from droidrun.agent.context import ContextInjectionManager
-from droidrun.agent.context.agent_persona import AgentPersona
-from droidrun.agent.context.personas import DEFAULT
 from droidrun.agent.context.task_manager import Task, TaskManager
 from droidrun.agent.droid.events import (
     CodeActExecuteEvent,
@@ -39,6 +34,17 @@ from droidrun.agent.executor import ExecutorAgent
 from droidrun.agent.manager import ManagerAgent
 from droidrun.agent.utils.tools import ATOMIC_ACTION_SIGNATURES
 from droidrun.agent.utils.trajectory import Trajectory
+from droidrun.config_manager.config_manager import (
+    AgentConfig,
+    DeviceConfig,
+    DroidRunConfig,
+    LoggingConfig,
+    TelemetryConfig,
+    ToolsConfig,
+    TracingConfig,
+)
+
+from droidrun.agent.utils.tools import open_app
 from droidrun.telemetry import (
     DroidAgentFinalizeEvent,
     DroidAgentInitEvent,
@@ -87,16 +93,16 @@ class DroidAgent(Workflow):
         goal: str,
         llms: dict[str, LLM] | LLM,
         tools: Tools,
-        personas: List[AgentPersona] = [DEFAULT],  # noqa: B006
-        max_steps: int = 15,
-        timeout: int = 1000,
-        vision: "VisionConfig | dict | bool" = False,
-        reasoning: bool = False,
-        enable_tracing: bool = False,
-        debug: bool = False,
-        save_trajectories: str = "none",
+        config: DroidRunConfig | None = None,
+        agent_config: AgentConfig | None = None,
+        device_config: DeviceConfig | None = None,
+        tools_config: ToolsConfig | None = None,
+        logging_config: LoggingConfig | None = None,
+        tracing_config: TracingConfig | None = None,
+        telemetry_config: TelemetryConfig | None = None,
         excluded_tools: List[str] = None,
         custom_tools: dict = None,
+        timeout: int = 1000,
         *args,
         **kwargs,
     ):
@@ -104,35 +110,40 @@ class DroidAgent(Workflow):
         Initialize the DroidAgent wrapper.
 
         Args:
-            goal: The user's goal or command to execute
-            llm: The language model to use for both agents
-            max_steps: Maximum number of steps for both agents
-            timeout: Timeout for agent execution in seconds
-            reasoning: Whether to use Manager+Executor for complex reasoning (True)
-                      or send tasks directly to CodeActAgent (False)
-            enable_tracing: Whether to enable Arize Phoenix tracing
-            debug: Whether to enable verbose debug logging
-            save_trajectories: Trajectory saving level. Can be:
-                - "none" (no saving)
-                - "step" (save per step)
-                - "action" (save per action)
-            custom_tools: Dictionary of custom tools in ATOMIC_ACTION_SIGNATURES format:
-                {
-                    "tool_name": {
-                        "arguments": ["arg1", "arg2"],
-                        "description": "Tool description with usage example",
-                        "function": callable
-                    }
-                }
-            **kwargs: Additional keyword arguments to pass to the agents
+            goal: User's goal or command
+            llms: Dict of agent-specific LLMs or single LLM for all
+            tools: Tools instance (AdbTools or IOSTools)
+            config: Full config override (optional)
+            agent_config: Agent config override (optional)
+            device_config: Device config override (optional)
+            tools_config: Tools config override (optional)
+            logging_config: Logging config override (optional)
+            tracing_config: Tracing config override (optional)
+            telemetry_config: Telemetry config override (optional)
+            excluded_tools: Tools to exclude
+            custom_tools: Custom tool definitions
+            timeout: Workflow timeout in seconds
         """
+
         self.user_id = kwargs.pop("user_id", None)
-        super().__init__(timeout=timeout, *args, **kwargs)  # noqa: B026
-        # Configure default logging if not already configured
-        self._configure_default_logging(debug=debug)
 
-        # Setup global tracing first if enabled
-        if enable_tracing:
+        base_config = config
+
+        self.config = DroidRunConfig(
+            agent=agent_config or base_config.agent,
+            device=device_config or base_config.device,
+            tools=tools_config or base_config.tools,
+            logging=logging_config or base_config.logging,
+            tracing=tracing_config or base_config.tracing,
+            telemetry=telemetry_config or base_config.telemetry,
+            llm_profiles=base_config.llm_profiles,
+        )
+
+        super().__init__(*args, timeout=timeout, **kwargs)
+
+        self._configure_default_logging(debug=self.config.logging.debug)
+
+        if self.config.tracing.enabled:
             try:
                 handler = arize_phoenix_callback_handler()
                 llama_index.core.global_handler = handler
@@ -144,148 +155,110 @@ class DroidAgent(Workflow):
                     "    • If installed via tool: `uv tool install droidrun[phoenix]`"
                     "    • If installed via pip: `uv pip install droidrun[phoenix]`\n"
                 )
-                enable_tracing = False
 
         self.goal = goal
-        self.max_steps = max_steps
-        self.max_codeact_steps = max_steps
         self.timeout = timeout
-        self.reasoning = reasoning
-        self.debug = debug
         self.custom_tools = custom_tools or {}
 
-        # ====================================================================
-        # Handle LLM parameter - support both dict and single LLM
-        # ====================================================================
         if isinstance(llms, dict):
             self.manager_llm = llms.get('manager')
             self.executor_llm = llms.get('executor')
             self.codeact_llm = llms.get('codeact')
             self.text_manipulator_llm = llms.get('text_manipulator')
             self.app_opener_llm = llms.get('app_opener')
-            
-            # Validate required LLMs are present
-            if reasoning and (not self.manager_llm or not self.executor_llm):
+
+            if self.config.agent.reasoning and (not self.manager_llm or not self.executor_llm):
                 raise ValueError("When reasoning=True, 'manager' and 'executor' LLMs must be provided in llms dict")
             if not self.codeact_llm:
                 raise ValueError("'codeact' LLM must be provided in llms dict")
-            
+
             logger.info("📚 Using agent-specific LLMs from dictionary")
         else:
-            # single LLM for all agents
-            logger.info("📚 Using single LLM for all agents (backward compatibility mode)")
+            logger.info("📚 Using single LLM for all agents")
             self.manager_llm = llms
             self.executor_llm = llms
             self.codeact_llm = llms
             self.text_manipulator_llm = llms
             self.app_opener_llm = llms
 
-        # ====================================================================
-        # Handle vision parameter - support VisionConfig, dict, or bool
-        # ====================================================================
-        if isinstance(vision, VisionConfig):
-            self.vision_config = vision
-        elif isinstance(vision, dict):
-            self.vision_config = VisionConfig.from_dict(vision)
-        elif isinstance(vision, bool):
-            # Backward compatibility: single bool for all agents
-            logger.info(f"👁️  Using vision={vision} for all agents (backward compatibility mode)")
-            self.vision_config = VisionConfig(manager=vision, executor=vision, codeact=vision)
-        else:
-            raise TypeError(f"vision must be VisionConfig, dict, or bool, got {type(vision)}")
-
-        # Store individual vision flags for easy access
-        self.manager_vision = self.vision_config.manager
-        self.executor_vision = self.vision_config.executor
-        self.codeact_vision = self.vision_config.codeact
-
 
         self.event_counter = 0
-        # Handle backward compatibility: bool -> str mapping
-        if isinstance(save_trajectories, bool):
-            self.save_trajectories = "step" if save_trajectories else "none"
-        else:
-            # Validate string values
-            valid_values = ["none", "step", "action"]
-            if save_trajectories not in valid_values:
-                logger.warning(
-                    f"Invalid save_trajectories value: {save_trajectories}. Using 'none' instead."
-                )
-                self.save_trajectories = "none"
-            else:
-                self.save_trajectories = save_trajectories
-
         self.trajectory = Trajectory(goal=goal)
         self.task_manager = TaskManager()
         self.task_iter = None
-
-        self.cim = ContextInjectionManager(personas=personas)
         self.current_episodic_memory = None
 
+        open_app_tool = {
+            "arguments": ["text"],
+            "description": "Open an app by name. Usage example: {\"action\": \"open_app\", \"text\": \"the name of app\"}",
+            "function": open_app,
+        }
+        # Merge with user-provided custom tools
+        self.custom_tools = {**self.custom_tools, "open_app": open_app_tool}
+
         logger.info("🤖 Initializing DroidAgent...")
-        logger.info(f"💾 Trajectory saving level: {self.save_trajectories}")
+        logger.info(f"💾 Trajectory saving: {self.config.logging.save_trajectory}")
 
         self.tools_instance = tools
+        self.tools_instance.save_trajectories = self.config.logging.save_trajectory
+        # Set app_opener_llm on tools instance for open_app custom tool
+        self.tools_instance.app_opener_llm = self.app_opener_llm
 
-        self.tools_instance.save_trajectories = self.save_trajectories
-
-        # Create shared state instance for Manager/Executor workflows
         self.shared_state = DroidAgentState(
             instruction=goal,
             err_to_manager_thresh=2
         )
 
-        if self.reasoning:
+        if self.config.agent.reasoning:
             logger.info("📝 Initializing Manager and Executor Agents...")
             self.manager_agent = ManagerAgent(
                 llm=self.manager_llm,
-                vision=self.manager_vision,
-                personas=personas,
                 tools_instance=tools,
                 shared_state=self.shared_state,
+                agent_config=self.config.agent,
                 custom_tools=self.custom_tools,
                 timeout=timeout,
-                debug=debug,
             )
             self.executor_agent = ExecutorAgent(
                 llm=self.executor_llm,
-                vision=self.executor_vision,
                 tools_instance=tools,
                 shared_state=self.shared_state,
-                persona=None,  # Need to figure this out
+                agent_config=self.config.agent,
                 custom_tools=self.custom_tools,
                 timeout=timeout,
-                debug=debug,
             )
-            self.max_codeact_steps = 5
-
-
-            # Keep planner_agent for backward compatibility (can be removed later)
             self.planner_agent = None
-
         else:
-            logger.debug("🚫 Reasoning disabled - will execute tasks directly with CodeActAgent")
+            logger.debug("🚫 Reasoning disabled - executing directly with CodeActAgent")
             self.manager_agent = None
             self.executor_agent = None
             self.planner_agent = None
 
-        # Get tool names from ATOMIC_ACTION_SIGNATURES for telemetry
         atomic_tools = list(ATOMIC_ACTION_SIGNATURES.keys())
 
         capture(
-            # TODO: do proper telemetry instead of this ductaped crap
             DroidAgentInitEvent(
                 goal=goal,
-                llm=self.llm.class_name(),
+                llms={
+                    "manager": self.manager_llm.class_name() if self.manager_llm else "None",
+                    "executor": self.executor_llm.class_name() if self.executor_llm else "None",
+                    "codeact": self.codeact_llm.class_name() if self.codeact_llm else "None",
+                    "text_manipulator": self.text_manipulator_llm.class_name() if
+                self.text_manipulator_llm else "None",
+                    "app_opener": self.app_opener_llm.class_name() if self.app_opener_llm else "None",
+                },
                 tools=",".join(atomic_tools + ["remember", "complete"]),
-                personas=",".join([p.name for p in personas]),
-                max_steps=max_steps,
+                max_steps=self.config.agent.max_steps,
                 timeout=timeout,
-                vision=self.vision,
-                reasoning=reasoning,
-                enable_tracing=enable_tracing,
-                debug=debug,
-                save_trajectories=save_trajectories,
+                vision={
+                    "manager": self.config.agent.manager.vision,
+                    "executor": self.config.agent.executor.vision,
+                    "codeact": self.config.agent.codeact.vision
+                },
+                reasoning=self.config.agent.reasoning,
+                enable_tracing=self.config.tracing.enabled,
+                debug=self.config.logging.debug,
+                save_trajectories=self.config.logging.save_trajectory,
             ),
             self.user_id,
         )
@@ -339,19 +312,17 @@ class DroidAgent(Workflow):
             Tuple of (success, reason)
         """
         task: Task = ev.task
-        persona = self.cim.get_persona(task.agent_type)
 
         logger.info(f"🔧 Executing task: {task.description}")
 
         try:
             codeact_agent = CodeActAgent(
                 llm=self.codeact_llm,
-                persona=persona,
-                vision=self.codeact_vision,
-                max_steps=self.max_codeact_steps,
+                agent_config=self.config.agent,
                 tools_instance=self.tools_instance,
                 custom_tools=self.custom_tools,
-                debug=self.debug,
+                debug=self.config.logging.debug,
+                shared_state=self.shared_state,
                 timeout=self.timeout,
             )
 
@@ -382,11 +353,10 @@ class DroidAgent(Workflow):
 
         except Exception as e:
             logger.error(f"Error during task execution: {e}")
-            if self.debug:
+            if self.config.logging.debug:
                 import traceback
-
                 logger.error(traceback.format_exc())
-            return CodeActResultEvent(success=False, reason=f"Error: {str(e)}", task=task, steps=[])
+            return CodeActResultEvent(success=False, reason=f"Error: {str(e)}", task=task, steps=0)
 
     @step
     async def handle_codeact_execute(
@@ -404,9 +374,8 @@ class DroidAgent(Workflow):
             )
         except Exception as e:
             logger.error(f"❌ Error during DroidAgent execution: {e}")
-            if self.debug:
+            if self.config.logging.debug:
                 import traceback
-
                 logger.error(traceback.format_exc())
             tasks = self.task_manager.get_task_history()
             return FinalizeEvent(
@@ -434,17 +403,15 @@ class DroidAgent(Workflow):
         self.step_counter = 0
         self.retry_counter = 0
 
-        if not self.reasoning:
+        if not self.config.agent.reasoning:
             logger.info(f"🔄 Direct execution mode - executing goal: {self.goal}")
             task = Task(
                 description=self.goal,
                 status=self.task_manager.STATUS_PENDING,
                 agent_type="Default",
             )
-
             return CodeActExecuteEvent(task=task)
 
-        # Reasoning mode - state already initialized in __init__, start with Manager
         logger.info("🧠 Reasoning mode - initializing Manager/Executor workflow")
         return ManagerInputEvent()
 
@@ -464,17 +431,15 @@ class DroidAgent(Workflow):
         Pre-flight checks for termination before running manager.
         The Manager analyzes current state and creates a plan with subgoals.
         """
-        # Check if we've reached the maximum number of steps
-        if self.step_counter >= self.max_steps:
-            logger.warning(f"⚠️ Reached maximum steps ({self.max_steps})")
+        if self.step_counter >= self.config.agent.max_steps:
+            logger.warning(f"⚠️ Reached maximum steps ({self.config.agent.max_steps})")
             return self._create_finalize_event(
                 success=False,
-                reason=f"Reached maximum steps ({self.max_steps})",
-                output=f"Reached maximum steps ({self.max_steps})"
+                reason=f"Reached maximum steps ({self.config.agent.max_steps})",
+                output=f"Reached maximum steps ({self.config.agent.max_steps})"
             )
 
-        # Continue with Manager execution
-        logger.info(f"📋 Running Manager for planning... (step {self.step_counter}/{self.max_steps})")
+        logger.info(f"📋 Running Manager for planning... (step {self.step_counter}/{self.config.agent.max_steps})")
 
         # Run Manager workflow
         handler = self.manager_agent.run()
@@ -489,7 +454,6 @@ class DroidAgent(Workflow):
         return ManagerPlanEvent(
             plan=result["plan"],
             current_subgoal=result["current_subgoal"],
-            completed_plan=result["completed_plan"],
             thought=result["thought"],
             manager_answer=result.get("manager_answer", "")
         )
@@ -551,7 +515,6 @@ class DroidAgent(Workflow):
         self.shared_state.last_summary = result["summary"]
         self.shared_state.last_action_thought = result.get("thought", "")
         self.shared_state.action_pool.append(result["action_json"])
-        self.shared_state.progress_status = self.shared_state.completed_plan
 
         return ExecutorResultEvent(
             action=result["action"],
@@ -583,9 +546,8 @@ class DroidAgent(Workflow):
                 self.shared_state.error_flag_plan = True
 
         self.step_counter += 1
-        logger.info(f"🔄 Step {self.step_counter}/{self.max_steps} complete, looping to Manager")
+        logger.info(f"🔄 Step {self.step_counter}/{self.config.agent.max_steps} complete, looping to Manager")
 
-        # Always loop back to Manager (it will check max steps in pre-flight)
         return ManagerInputEvent()
 
     # ========================================================================
@@ -608,13 +570,12 @@ class DroidAgent(Workflow):
 
         result = {
             "success": ev.success,
-            # deprecated. use output instead.
             "reason": ev.reason,
             "output": ev.output,
             "steps": ev.steps,
         }
 
-        if self.trajectory and self.save_trajectories != "none":
+        if self.trajectory and self.config.logging.save_trajectory != "none":
             self.trajectory.save_trajectory()
 
         return StopEvent(result)

droidrun/agent/droid/events.py

@@ -1,3 +1,15 @@
+"""
+DroidAgent coordination events.
+
+These events are used for WORKFLOW COORDINATION between DroidAgent and its child agents.
+They carry minimal data needed for routing workflow steps.
+
+For internal events with full debugging metadata, see:
+- manager/events.py (ManagerInternalPlanEvent)
+- executor/events.py (ExecutorInternalActionEvent, ExecutorInternalResultEvent)
+- codeact/events.py (Task*, EpisodicMemoryEvent)
+"""
+
 from typing import Dict, List
 
 from llama_index.core.workflow import Event
@@ -12,6 +24,7 @@ class CodeActExecuteEvent(Event):
 class CodeActResultEvent(Event):
     success: bool
     reason: str
+    task: Task
     steps: int
 
 
@@ -29,11 +42,6 @@ class TaskRunnerEvent(Event):
     pass
 
 
-
-# ============================================================================
-# DroidAgentState - State model for llama-index Context
-# ============================================================================
-
 class DroidAgentState(BaseModel):
     """
     State model for DroidAgent workflow - shared across parent and child workflows.
@@ -42,21 +50,36 @@ class DroidAgentState(BaseModel):
     # Task context
     instruction: str = ""
 
-    # UI State
-    ui_elements_list_before: str = ""
-    ui_elements_list_after: str = ""
+    # Formatted device state for prompts (complete text)
+    formatted_device_state: str = ""
+
+    # Focused element text
     focused_text: str = ""
-    device_state_text: str = ""
+
+    # Raw device state components (for access to raw data)
+    a11y_tree: List[Dict] = Field(default_factory=list)
+    phone_state: Dict = Field(default_factory=dict)
+
+    # Derived fields (extracted from phone_state)
+    current_package_name: str = ""
+    current_app_name: str = ""
+
+    # Previous device state (for before/after comparison in Manager)
+    previous_formatted_device_state: str = ""
+
+    # Screen dimensions and screenshot
     width: int = 0
     height: int = 0
     screenshot: str | bytes | None = None
+
+    # Text manipulation flag
     has_text_to_modify: bool = False
 
     # Action tracking
     action_pool: List[Dict] = Field(default_factory=list)
     action_history: List[Dict] = Field(default_factory=list)
     summary_history: List[str] = Field(default_factory=list)
-    action_outcomes: List[str] = Field(default_factory=list)  # "A", "B", "C"
+    action_outcomes: List[bool] = Field(default_factory=list)  # "A", "B", "C"
     error_descriptions: List[str] = Field(default_factory=list)
 
     # Last action info
@@ -70,7 +93,6 @@ class DroidAgentState(BaseModel):
 
     # Planning
     plan: str = ""
-    completed_plan: str = ""
     current_subgoal: str = ""
     finish_thought: str = ""
     progress_status: str = ""
@@ -94,10 +116,14 @@ class ManagerInputEvent(Event):
 
 
 class ManagerPlanEvent(Event):
-    """Manager has created a plan"""
+    """
+    Coordination event from ManagerAgent to DroidAgent.
+
+    Used for workflow step routing only (NOT streamed to frontend).
+    For internal events with memory_update metadata, see ManagerInternalPlanEvent.
+    """
     plan: str
     current_subgoal: str
-    completed_plan: str
     thought: str
     manager_answer: str = ""
 
@@ -108,7 +134,12 @@ class ExecutorInputEvent(Event):
 
 
 class ExecutorResultEvent(Event):
-    """Executor action result"""
+    """
+    Coordination event from ExecutorAgent to DroidAgent.
+
+    Used for workflow step routing only (NOT streamed to frontend).
+    For internal events with thought/action_json metadata, see ExecutorInternalResultEvent.
+    """
     action: Dict
     outcome: bool
     error: str

droidrun/agent/executor/__init__.py

@@ -2,12 +2,14 @@
 Executor Agent - Action execution workflow.
 """
 
-from droidrun.agent.executor.events import ExecutorActionEvent, ExecutorResultEvent
+from droidrun.agent.droid.events import ExecutorInputEvent, ExecutorResultEvent
+from droidrun.agent.executor.events import ExecutorInternalActionEvent, ExecutorInternalResultEvent
 from droidrun.agent.executor.executor_agent import ExecutorAgent
 
 __all__ = [
     "ExecutorAgent",
-    "ExecutorThinkingEvent",
-    "ExecutorActionEvent",
-    "ExecutorResultEvent"
+    "ExecutorInputEvent",
+    "ExecutorResultEvent",
+    "ExecutorInternalActionEvent",
+    "ExecutorInternalResultEvent"
 ]

droidrun/agent/executor/events.py

@@ -1,5 +1,11 @@
 """
 Events for the ExecutorAgent workflow.
+
+These are INTERNAL events used within ExecutorAgent for:
+- Streaming to frontend/logging
+- Carrying full debug metadata (thought process, raw action JSON)
+
+For workflow coordination with DroidAgent, see droid/events.py
 """
 
 from typing import Dict
@@ -7,18 +13,32 @@ from typing import Dict
 from llama_index.core.workflow.events import Event
 
 
-class ExecutorActionEvent(Event):
-    """Executor has selected an action to execute"""
-    action_json: str
-    thought: str
-    description: str
+class ExecutorInternalActionEvent(Event):
+    """
+    Internal Executor action selection event with thought process.
+
+    This event is streamed to frontend/logging but NOT used for
+    workflow coordination between ExecutorAgent and DroidAgent.
+
+    For workflow coordination, see ExecutorInputEvent in droid/events.py
+    """
+    action_json: str  # Raw JSON string of the action
+    thought: str  # Debugging metadata: LLM's reasoning process
+    description: str  # Human-readable action description
+
+
+class ExecutorInternalResultEvent(Event):
+    """
+    Internal Executor result event with full debug information.
 
+    This event is streamed to frontend/logging but NOT used for
+    workflow coordination between ExecutorAgent and DroidAgent.
 
-class ExecutorResultEvent(Event):
-    """Executor action result"""
+    For workflow coordination, see ExecutorResultEvent in droid/events.py
+    """
     action: Dict
     outcome: bool
     error: str
     summary: str
-    thought: str = ""
-    action_json: str = ""
+    thought: str = ""  # Debugging metadata: LLM's thought process
+    action_json: str = ""  # Debugging metadata: Raw action JSON