droidrun 0.3.9__py3-none-any.whl → 0.3.10.dev3__py3-none-any.whl

droidrun/tools/adb.py

@@ -2,25 +2,27 @@
 UI Actions - Core UI interaction tools for Android device control.
 """
 
-import os
+import base64
 import io
 import json
-import time
 import logging
+import os
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+from adbutils import adb
 from llama_index.core.workflow import Context
-from typing import Optional, Dict, Tuple, List, Any
+
 from droidrun.agent.common.events import (
+    DragActionEvent,
     InputTextActionEvent,
     KeyPressActionEvent,
     StartAppEvent,
     SwipeActionEvent,
     TapActionEvent,
-    DragActionEvent,
 )
 from droidrun.tools.tools import Tools
-from adbutils import adb
-import requests
-import base64
 
 logger = logging.getLogger("droidrun-tools")
 PORTAL_DEFAULT_TCP_PORT = 8080
@@ -34,6 +36,8 @@ class AdbTools(Tools):
         serial: str | None = None,
         use_tcp: bool = False,
         remote_tcp_port: int = PORTAL_DEFAULT_TCP_PORT,
+        app_opener_llm = None,
+        text_manipulator_llm = None,
     ) -> None:
         """Initialize the AdbTools instance.
 
@@ -41,6 +45,8 @@ class AdbTools(Tools):
             serial: Device serial number
             use_tcp: Whether to use TCP communication (default: False)
             tcp_port: TCP port for communication (default: 8080)
+            app_opener_llm: LLM instance for app opening workflow (optional)
+            text_manipulator_llm: LLM instance for text manipulation (optional)
         """
         self.device = adb.device(serial=serial)
         self.use_tcp = use_tcp
@@ -60,6 +66,10 @@ class AdbTools(Tools):
         self.screenshots: List[Dict[str, Any]] = []
         # Trajectory saving level
         self.save_trajectories = "none"
+        
+        # LLM instances for specialized workflows
+        self.app_opener_llm = app_opener_llm
+        self.text_manipulator_llm = text_manipulator_llm
 
         self.setup_keyboard()
 
@@ -68,6 +78,13 @@ class AdbTools(Tools):
             self.setup_tcp_forward()
 
 
+    def get_date(self) -> str:
+        """
+        Get the current date and time on device.
+        """
+        return self.device.shell("date").strip()
+
+
     def setup_tcp_forward(self) -> bool:
         """
         Set up ADB TCP port forwarding for communication with the portal app.
@@ -126,7 +143,7 @@ class AdbTools(Tools):
                 c.close()
 
                 self.tcp_forwarded = False
-                logger.debug(f"TCP port forwarding removed")
+                logger.debug("TCP port forwarding removed")
                 return True
             return True
         except Exception as e:
@@ -137,7 +154,7 @@ class AdbTools(Tools):
         """
         Set up the DroidRun keyboard as the default input method.
         Simple setup that just switches to DroidRun keyboard without saving/restoring.
-        
+
         Returns:
             bool: True if setup was successful, False otherwise
         """
@@ -207,20 +224,19 @@ class AdbTools(Tools):
             return None
 
     @Tools.ui_action
-    def tap_by_index(self, index: int) -> str:
+    def _extract_element_coordinates_by_index(self, index: int) -> Tuple[int, int]:
         """
-        Tap on a UI element by its index.
-
-        This function uses the cached clickable elements
-        to find the element with the given index and tap on its center coordinates.
+        Extract center coordinates from an element by its index.
 
         Args:
-            index: Index of the element to tap
+            index: Index of the element to find and extract coordinates from
 
         Returns:
-            Result message
-        """
+            Tuple of (x, y) center coordinates
 
+        Raises:
+            ValueError: If element not found, bounds format is invalid, or missing bounds
+        """
         def collect_all_indices(elements):
             """Recursively collect all indices from elements and their children."""
             indices = []
@@ -244,40 +260,57 @@ class AdbTools(Tools):
                     return result
             return None
 
+        # Check if we have cached elements
+        if not self.clickable_elements_cache:
+            raise ValueError("No UI elements cached. Call get_state first.")
+
+        # Find the element with the given index (including in children)
+        element = find_element_by_index(self.clickable_elements_cache, index)
+
+        if not element:
+            # List available indices to help the user
+            indices = sorted(collect_all_indices(self.clickable_elements_cache))
+            indices_str = ", ".join(str(idx) for idx in indices[:20])
+            if len(indices) > 20:
+                indices_str += f"... and {len(indices) - 20} more"
+            raise ValueError(f"No element found with index {index}. Available indices: {indices_str}")
+
+        # Get the bounds of the element
+        bounds_str = element.get("bounds")
+        if not bounds_str:
+            element_text = element.get("text", "No text")
+            element_type = element.get("type", "unknown")
+            element_class = element.get("className", "Unknown class")
+            raise ValueError(f"Element with index {index} ('{element_text}', {element_class}, type: {element_type}) has no bounds and cannot be tapped")
+
+        # Parse the bounds (format: "left,top,right,bottom")
         try:
-            # Check if we have cached elements
-            if not self.clickable_elements_cache:
-                return "Error: No UI elements cached. Call get_state first."
+            left, top, right, bottom = map(int, bounds_str.split(","))
+        except ValueError:
+            raise ValueError(f"Invalid bounds format for element with index {index}: {bounds_str}") from ValueError
 
-            # Find the element with the given index (including in children)
-            element = find_element_by_index(self.clickable_elements_cache, index)
+        # Calculate the center of the element
+        x = (left + right) // 2
+        y = (top + bottom) // 2
 
-            if not element:
-                # List available indices to help the user
-                indices = sorted(collect_all_indices(self.clickable_elements_cache))
-                indices_str = ", ".join(str(idx) for idx in indices[:20])
-                if len(indices) > 20:
-                    indices_str += f"... and {len(indices) - 20} more"
+        return x, y
 
-                return f"Error: No element found with index {index}. Available indices: {indices_str}"
+    def tap_by_index(self, index: int) -> str:
+        """
+        Tap on a UI element by its index.
 
-            # Get the bounds of the element
-            bounds_str = element.get("bounds")
-            if not bounds_str:
-                element_text = element.get("text", "No text")
-                element_type = element.get("type", "unknown")
-                element_class = element.get("className", "Unknown class")
-                return f"Error: Element with index {index} ('{element_text}', {element_class}, type: {element_type}) has no bounds and cannot be tapped"
+        This function uses the cached clickable elements
+        to find the element with the given index and tap on its center coordinates.
 
-            # Parse the bounds (format: "left,top,right,bottom")
-            try:
-                left, top, right, bottom = map(int, bounds_str.split(","))
-            except ValueError:
-                return f"Error: Invalid bounds format for element with index {index}: {bounds_str}"
+        Args:
+            index: Index of the element to tap
 
-            # Calculate the center of the element
-            x = (left + right) // 2
-            y = (top + bottom) // 2
+        Returns:
+            Result message
+        """
+        try:
+            # Extract coordinates using the helper function
+            x, y = self._extract_element_coordinates_by_index(index)
 
             logger.debug(
                 f"Tapping element with index {index} at coordinates ({x}, {y})"
@@ -287,10 +320,24 @@ class AdbTools(Tools):
             logger.debug(f"Tapped element with index {index} at coordinates ({x}, {y})")
 
             # Emit coordinate action event for trajectory recording
-
             if self._ctx:
-                element_text = element.get("text", "No text")
-                element_class = element.get("className", "Unknown class")
+                # Find element again for event details
+                def find_element_by_index(elements, target_index):
+                    """Recursively find an element with the given index."""
+                    for item in elements:
+                        if item.get("index") == target_index:
+                            return item
+                        # Check children if present
+                        children = item.get("children", [])
+                        result = find_element_by_index(children, target_index)
+                        if result:
+                            return result
+                    return None
+
+                element = find_element_by_index(self.clickable_elements_cache, index)
+                element_text = element.get("text", "No text") if element else "No text"
+                element_class = element.get("className", "Unknown class") if element else "Unknown class"
+                bounds_str = element.get("bounds", "") if element else ""
 
                 tap_event = TapActionEvent(
                     action_type="tap",
@@ -307,20 +354,34 @@ class AdbTools(Tools):
             time.sleep(0.5)
 
             # Create a descriptive response
+            def find_element_by_index(elements, target_index):
+                """Recursively find an element with the given index."""
+                for item in elements:
+                    if item.get("index") == target_index:
+                        return item
+                    # Check children if present
+                    children = item.get("children", [])
+                    result = find_element_by_index(children, target_index)
+                    if result:
+                        return result
+                return None
+
+            element = find_element_by_index(self.clickable_elements_cache, index)
             response_parts = []
             response_parts.append(f"Tapped element with index {index}")
-            response_parts.append(f"Text: '{element.get('text', 'No text')}'")
-            response_parts.append(f"Class: {element.get('className', 'Unknown class')}")
-            response_parts.append(f"Type: {element.get('type', 'unknown')}")
+            response_parts.append(f"Text: '{element.get('text', 'No text') if element else 'No text'}'")
+            response_parts.append(f"Class: {element.get('className', 'Unknown class') if element else 'Unknown class'}")
+            response_parts.append(f"Type: {element.get('type', 'unknown') if element else 'unknown'}")
 
             # Add information about children if present
-            children = element.get("children", [])
-            if children:
-                child_texts = [
-                    child.get("text") for child in children if child.get("text")
-                ]
-                if child_texts:
-                    response_parts.append(f"Contains text: {' | '.join(child_texts)}")
+            if element:
+                children = element.get("children", [])
+                if children:
+                    child_texts = [
+                        child.get("text") for child in children if child.get("text")
+                    ]
+                    if child_texts:
+                        response_parts.append(f"Contains text: {' | '.join(child_texts)}")
 
             response_parts.append(f"Coordinates: ({x}, {y})")
 
@@ -453,24 +514,31 @@ class AdbTools(Tools):
             return False
 
     @Tools.ui_action
-    def input_text(self, text: str) -> str:
+    def input_text(self, text: str, index: int = -1, clear: bool = False) -> str:
         """
         Input text on the device.
         Always make sure that the Focused Element is not None before inputting text.
 
         Args:
             text: Text to input. Can contain spaces, newlines, and special characters including non-ASCII.
+            index: Index of the element to input text into. If -1, the focused element will be used.
+            clear: Whether to clear the text before inputting.
 
         Returns:
             Result message
         """
         try:
+            if index != -1:
+                self.tap_by_index(index)
+            # Encode the text to Base64 (needed for both TCP and content provider)
+            encoded_text = base64.b64encode(text.encode()).decode()
 
             if self.use_tcp and self.tcp_forwarded:
                 # Use TCP communication
-                encoded_text = base64.b64encode(text.encode()).decode()
-
-                payload = {"base64_text": encoded_text}
+                payload = {
+                    "base64_text": encoded_text,
+                    "clear": clear  # Include clear parameter for TCP
+                }
                 response = requests.post(
                     f"{self.tcp_base_url}/keyboard/input",
                     json=payload,
@@ -479,32 +547,48 @@ class AdbTools(Tools):
                 )
 
                 logger.debug(
-                     f"Keyboard input TCP response: {response.status_code}, {response.text}"
+                    f"Keyboard input TCP response: {response.status_code}, {response.text}"
                 )
 
                 if response.status_code != 200:
                     return f"Error: HTTP request failed with status {response.status_code}: {response.text}"
 
+                # For TCP, you might want to parse the response for success/error details
+                try:
+                    result_data = response.json()
+                    if result_data.get("status") == "success":
+                        return f"Text input completed (clear={clear}): {text[:50]}{'...' if len(text) > 50 else ''}"
+                    else:
+                        return f"Error: {result_data.get('error', 'Unknown error')}"
+                except:  # noqa: E722
+                    return f"Text input completed (clear={clear}): {text[:50]}{'...' if len(text) > 50 else ''}"
+
             else:
                 # Fallback to content provider method
-                # Encode the text to Base64
-                encoded_text = base64.b64encode(text.encode()).decode()
+                # Build the content insert command with clear parameter
+                clear_str = "true" if clear else "false"
+                cmd = (
+                    f'content insert --uri "content://com.droidrun.portal/keyboard/input" '
+                    f'--bind base64_text:s:"{encoded_text}" '
+                    f'--bind clear:b:{clear_str}'
+                )
 
-                cmd = f'content insert --uri "content://com.droidrun.portal/keyboard/input" --bind base64_text:s:"{encoded_text}"'
-                self.device.shell(cmd)
+                # Execute the command and capture output for better error handling
+                result = self.device.shell(cmd)
+                logger.debug(f"Content provider result: {result}")
 
             if self._ctx:
                 input_event = InputTextActionEvent(
                     action_type="input_text",
-                    description=f"Input text: '{text[:50]}{'...' if len(text) > 50 else ''}'",
+                    description=f"Input text: '{text[:50]}{'...' if len(text) > 50 else ''}' (clear={clear})",
                     text=text,
                 )
                 self._ctx.write_event_to_stream(input_event)
 
             logger.debug(
-                f"Text input completed: {text[:50]}{'...' if len(text) > 50 else ''}"
+                f"Text input completed (clear={clear}): {text[:50]}{'...' if len(text) > 50 else ''}"
             )
-            return f"Text input completed: {text[:50]}{'...' if len(text) > 50 else ''}"
+            return f"Text input completed (clear={clear}): {text[:50]}{'...' if len(text) > 50 else ''}"
 
         except requests.exceptions.RequestException as e:
             return f"Error: TCP request failed: {str(e)}"
@@ -526,13 +610,13 @@ class AdbTools(Tools):
             if self._ctx:
                 key_event = KeyPressActionEvent(
                     action_type="key_press",
-                    description=f"Pressed key BACK",
+                    description="Pressed key BACK",
                     keycode=4,
                     key_name="BACK",
                 )
                 self._ctx.write_event_to_stream(key_event)
 
-            return f"Pressed key BACK"
+            return "Pressed key BACK"
         except ValueError as e:
             return f"Error: {str(e)}"
 
@@ -645,7 +729,7 @@ class AdbTools(Tools):
         Take a screenshot of the device.
         This function captures the current screen and adds the screenshot to context in the next message.
         Also stores the screenshot in the screenshots list with timestamp for later GIF creation.
-        
+
         Args:
             hide_overlay: Whether to hide the overlay elements during screenshot (default: True)
         """
@@ -659,11 +743,11 @@ class AdbTools(Tools):
                 url = f"{self.tcp_base_url}/screenshot"
                 if not hide_overlay:
                     url += "?hideOverlay=false"
-                
+
                 response = requests.get(url, timeout=10)
                 if response.status_code == 200:
                     tcp_response = response.json()
-                    
+
                     # Check if response has the expected format with data field
                     if tcp_response.get("status") == "success" and "data" in tcp_response:
                         # Decode base64 string to bytes
@@ -696,11 +780,12 @@ class AdbTools(Tools):
             return img_format, image_bytes
 
         except requests.exceptions.RequestException as e:
-            raise ValueError(f"Error taking screenshot via TCP: {str(e)}")
+            raise ValueError(f"Error taking screenshot via TCP: {str(e)}") from e
         except ValueError as e:
-            raise ValueError(f"Error taking screenshot: {str(e)}")
+            raise ValueError(f"Error taking screenshot: {str(e)}") from e
         except Exception as e:
-            raise ValueError(f"Unexpected error taking screenshot: {str(e)}")
+            raise ValueError(f"Unexpected error taking screenshot: {str(e)}") from e
+
 
     def list_packages(self, include_system_apps: bool = False) -> List[str]:
         """
@@ -716,7 +801,50 @@ class AdbTools(Tools):
             logger.debug("Listing packages")
             return self.device.list_packages(["-3"] if not include_system_apps else [])
         except ValueError as e:
-            raise ValueError(f"Error listing packages: {str(e)}")
+            raise ValueError(f"Error listing packages: {str(e)}") from e
+
+    def get_apps(self, include_system: bool = True) -> List[Dict[str, str]]:
+        """
+        Get installed apps with package name and label in human readable format.
+
+        Args:
+            include_system: Whether to include system apps (default: True)
+
+        Returns:
+            List of dictionaries containing 'package' and 'label' keys
+        """
+        try:
+            logger.debug("Getting apps via content provider")
+
+            # Query the content provider for packages
+            adb_output = self.device.shell(
+                "content query --uri content://com.droidrun.portal/packages"
+            )
+
+            # Parse the content provider output
+            packages_data = self._parse_content_provider_output(adb_output)
+
+            if not packages_data or "packages" not in packages_data:
+                logger.warning("No packages data found in content provider response")
+                return []
+
+            apps = []
+            for package_info in packages_data["packages"]:
+                # Filter system apps if requested
+                if not include_system and package_info.get("isSystemApp", False):
+                    continue
+
+                apps.append({
+                    "package": package_info.get("packageName", ""),
+                    "label": package_info.get("label", "")
+                })
+
+            logger.debug(f"Found {len(apps)} apps")
+            return apps
+
+        except Exception as e:
+            logger.error(f"Error getting apps: {str(e)}")
+            raise ValueError(f"Error getting apps: {str(e)}") from e
 
     @Tools.ui_action
     def complete(self, success: bool, reason: str = ""):
@@ -831,7 +959,7 @@ class AdbTools(Tools):
                     data_str = None
                     if "data" in state_data:
                         data_str = state_data["data"]
-                    
+
                     if data_str:
                         try:
                             combined_data = json.loads(data_str)
@@ -842,7 +970,7 @@ class AdbTools(Tools):
                             }
                     else:
                         return {
-                            "error": "Format Error", 
+                            "error": "Format Error",
                             "message": "Neither 'data' nor 'message' field found in ContentProvider response",
                         }
                 else:
@@ -956,8 +1084,8 @@ def _shell_test_cli(serial: str, command: str) -> tuple[str, float]:
     Returns:
         Tuple of (output, elapsed_time)
     """
-    import time
     import subprocess
+    import time
 
     adb_cmd = ["adb", "-s", serial, "shell", command]
     start = time.perf_counter()
@@ -1009,8 +1137,8 @@ def _shell_test_cli(serial: str, command: str) -> tuple[str, float]:
     Returns:
         Tuple of (output, elapsed_time)
     """
-    import time
     import subprocess
+    import time
 
     adb_cmd = ["adb", "-s", serial, "shell", command]
     start = time.perf_counter()

droidrun/tools/ios.py

@@ -2,11 +2,13 @@
 UI Actions - Core UI interaction tools for iOS device control.
 """
 
+import logging
 import re
 import time
-from typing import Optional, Dict, Tuple, List, Any
-import logging
+from typing import Any, Dict, List, Optional, Tuple
+
 import requests
+
 from droidrun.tools.tools import Tools
 
 logger = logging.getLogger("IOS")
@@ -37,7 +39,7 @@ SYSTEM_BUNDLE_IDENTIFIERS = [
 class IOSTools(Tools):
     """Core UI interaction tools for iOS device control."""
 
-    def __init__(self, url: str, bundle_identifiers: List[str] = []) -> None:
+    def __init__(self, url: str, bundle_identifiers: List[str] = []) -> None:  # noqa: B006
         """Initialize the IOSTools instance.
 
         Args:
@@ -387,7 +389,7 @@ class IOSTools(Tools):
             Bool indicating success or failure
         """
         # TODO: implement this
-        logger.info(f"Drag action FAILED! Not implemented for iOS")
+        logger.info("Drag action FAILED! Not implemented for iOS")
         return False
 
     def input_text(self, text: str) -> str:
@@ -506,7 +508,7 @@ class IOSTools(Tools):
 
         except Exception as e:
             logger.error(f"Error capturing screenshot: {e}")
-            raise ValueError(f"Error taking screenshot: {str(e)}")
+            raise ValueError(f"Error taking screenshot: {str(e)}") from e
 
     def _get_phone_state(self) -> Dict[str, Any]:
         """

droidrun/tools/tools.py

@@ -1,9 +1,8 @@
-from abc import ABC, abstractmethod
-from typing import List, Optional, Dict, Any
 import logging
-from typing import Tuple, Dict, Callable, Any, Optional
-from functools import wraps
 import sys
+from abc import ABC, abstractmethod
+from functools import wraps
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 # Get a logger for this module
 logger = logging.getLogger(__name__)
@@ -24,15 +23,15 @@ class Tools(ABC):
         def wrapper(*args, **kwargs):
             self = args[0]
             result = func(*args, **kwargs)
-            
+
             # Check if save_trajectories attribute exists and is set to "action"
             if hasattr(self, 'save_trajectories') and self.save_trajectories == "action":
                 frame = sys._getframe(1)
                 caller_globals = frame.f_globals
-                
+
                 step_screenshots = caller_globals.get('step_screenshots')
                 step_ui_states = caller_globals.get('step_ui_states')
-                
+
                 if step_screenshots is not None:
                     step_screenshots.append(self.take_screenshot()[1])
                 if step_ui_states is not None:
@@ -46,6 +45,12 @@ class Tools(ABC):
         Get the current state of the tool.
         """
         pass
+    @abstractmethod
+    def get_date(self) -> str:
+        """
+        Get the current date on device.
+        """
+        pass
 
     @abstractmethod
     def tap_by_index(self, index: int) -> str:
@@ -77,7 +82,7 @@ class Tools(ABC):
         pass
 
     @abstractmethod
-    def input_text(self, text: str) -> str:
+    def input_text(self, text: str, index: int = -1, clear: bool = False) -> str:
         """
         Input the given text into a focused input field.
         """
@@ -117,6 +122,12 @@ class Tools(ABC):
         List all packages on the device.
         """
         pass
+    @abstractmethod
+    def get_apps(self, include_system_apps: bool = True) -> List[Dict[str, Any]]:
+        """
+        List all apps on the device.
+        """
+        pass
 
     @abstractmethod
     def remember(self, information: str) -> str:
@@ -138,6 +149,12 @@ class Tools(ABC):
         Complete the tool. This is used to indicate that the tool has completed its task.
         """
         pass
+    @abstractmethod
+    def _extract_element_coordinates_by_index(self, index: int) -> Tuple[int, int]:
+        """
+        Extract the coordinates of the element with the given index.
+        """
+        pass
 
 
 def describe_tools(tools: Tools, exclude_tools: Optional[List[str]] = None) -> Dict[str, Callable[..., Any]]:

{droidrun-0.3.9.dist-info → droidrun-0.3.10.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: droidrun
-Version: 0.3.9
+Version: 0.3.10.dev3
 Summary: A framework for controlling Android devices through LLM agents
 Project-URL: Homepage, https://github.com/droidrun/droidrun
 Project-URL: Bug Tracker, https://github.com/droidrun/droidrun/issues
@@ -27,10 +27,10 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Software Development :: Testing :: Acceptance
 Classifier: Topic :: System :: Emulators
 Classifier: Topic :: Utilities
-Requires-Python: >=3.11
+Requires-Python: >=3.13
 Requires-Dist: adbutils>=2.10.2
 Requires-Dist: apkutils==2.0.0
-Requires-Dist: llama-index-llms-google-genai>=0.6.2
+Requires-Dist: arize-phoenix>=12.3.0
 Requires-Dist: llama-index==0.14.4
 Requires-Dist: posthog>=6.7.6
 Requires-Dist: pydantic>=2.11.10
@@ -54,6 +54,8 @@ Provides-Extra: openai
 Requires-Dist: llama-index-llms-openai-like>=0.5.1; extra == 'openai'
 Requires-Dist: llama-index-llms-openai>=0.5.6; extra == 'openai'
 Requires-Dist: openai>=1.99.1; extra == 'openai'
+Provides-Extra: openrouter
+Requires-Dist: llama-index-llms-openrouter>=0.4.2; extra == 'openrouter'
 Description-Content-Type: text/markdown
 
 <picture>