android-emu-agent 0.1.3__py3-none-any.whl

android_emu_agent/ui/snapshotter.py

@@ -0,0 +1,236 @@
+"""UI Snapshotter - Multi-source extraction, filtering, scoring."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import structlog
+from lxml import etree
+
+if TYPE_CHECKING:
+    pass
+
+logger = structlog.get_logger()
+
+# Warning threshold for snapshot size (20KB)
+SNAPSHOT_SIZE_WARNING_BYTES = 20480
+
+# XPath for interactive elements only
+INTERACTIVE_XPATH = (
+    "//*["
+    "@clickable='true' or "
+    "@focusable='true' or "
+    "@scrollable='true' or "
+    "@checkable='true' or "
+    "@editable='true' or "
+    "string-length(@text)>0"
+    "]"
+)
+
+
+@dataclass
+class ElementNode:
+    """Represents an interactive UI element."""
+
+    ref: str
+    role: str
+    label: str | None
+    resource_id: str | None
+    class_name: str
+    bounds: list[int]
+    state: dict[str, bool]
+    content_desc: str | None = None
+    text: str | None = None
+
+
+@dataclass
+class Snapshot:
+    """Complete UI snapshot with context and elements."""
+
+    schema_version: int
+    session_id: str
+    generation: int
+    timestamp_ms: int
+    device: dict[str, Any]
+    context: dict[str, Any]
+    elements: list[ElementNode]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to JSON-serializable dict."""
+        result: dict[str, Any] = {
+            "schema_version": self.schema_version,
+            "session_id": self.session_id,
+            "generation": self.generation,
+            "timestamp_ms": self.timestamp_ms,
+            "device": self.device,
+            "context": self.context,
+            "elements": [
+                {
+                    "ref": e.ref,
+                    "role": e.role,
+                    "label": e.label,
+                    "resource_id": e.resource_id,
+                    "class": e.class_name,
+                    "bounds": e.bounds,
+                    "state": e.state,
+                    "content_desc": e.content_desc,
+                    "text": e.text,
+                }
+                for e in self.elements
+            ],
+        }
+
+        # Check size and add warning if needed
+        size_bytes = len(json.dumps(result).encode())
+        if size_bytes > SNAPSHOT_SIZE_WARNING_BYTES:
+            result["warnings"] = [
+                f"Snapshot size {size_bytes // 1024}KB exceeds 20KB target. "
+                "Consider filtering or using a simpler screen."
+            ]
+
+        return result
+
+
+class UISnapshotter:
+    """Generates compact UI snapshots from device hierarchy."""
+
+    def __init__(self) -> None:
+        self._ref_counter = 0
+
+    def parse_hierarchy(
+        self,
+        xml_content: bytes,
+        session_id: str,
+        generation: int,
+        device_info: dict[str, Any],
+        context_info: dict[str, Any],
+        interactive_only: bool = True,
+    ) -> Snapshot:
+        """Parse XML hierarchy and extract elements."""
+        import time
+
+        start = time.time()
+        tree = etree.fromstring(xml_content)
+        elements = self._extract_elements(tree, interactive_only=interactive_only)
+        elapsed = (time.time() - start) * 1000
+
+        logger.info(
+            "hierarchy_parsed",
+            element_count=len(elements),
+            elapsed_ms=round(elapsed, 2),
+        )
+
+        return Snapshot(
+            schema_version=1,
+            session_id=session_id,
+            generation=generation,
+            timestamp_ms=int(time.time() * 1000),
+            device=device_info,
+            context=context_info,
+            elements=elements,
+        )
+
+    def _extract_elements(
+        self,
+        tree: etree._Element,
+        *,
+        interactive_only: bool,
+    ) -> list[ElementNode]:
+        """Extract elements using XPath."""
+        elements: list[ElementNode] = []
+        self._ref_counter = 0
+
+        xpath = INTERACTIVE_XPATH if interactive_only else "//*"
+        nodes = tree.xpath(xpath)
+        if not isinstance(nodes, list):
+            return elements
+        for node in nodes:
+            if not isinstance(node, etree._Element):
+                continue
+            element = self._node_to_element(node)
+            if element:
+                elements.append(element)
+
+        return elements
+
+    def _node_to_element(self, node: etree._Element) -> ElementNode | None:
+        """Convert XML node to ElementNode."""
+        self._ref_counter += 1
+        ref = f"@a{self._ref_counter}"
+
+        # Parse bounds "[left,top][right,bottom]"
+        bounds_str = node.get("bounds", "[0,0][0,0]")
+        bounds = self._parse_bounds(bounds_str)
+
+        # Determine role from class name
+        class_name = node.get("class", "")
+        role = self._infer_role(class_name, node)
+
+        # Get label (prefer content-desc, then text)
+        content_desc = node.get("content-desc")
+        text = node.get("text")
+        label = content_desc or text or None
+
+        return ElementNode(
+            ref=ref,
+            role=role,
+            label=label,
+            resource_id=node.get("resource-id"),
+            class_name=class_name,
+            bounds=bounds,
+            content_desc=content_desc,
+            text=text,
+            state={
+                "clickable": node.get("clickable") == "true",
+                "focusable": node.get("focusable") == "true",
+                "focused": node.get("focused") == "true",
+                "enabled": node.get("enabled") == "true",
+                "checked": node.get("checked") == "true",
+                "selected": node.get("selected") == "true",
+                "scrollable": node.get("scrollable") == "true",
+                "editable": node.get("editable") == "true",
+            },
+        )
+
+    def _parse_bounds(self, bounds_str: str) -> list[int]:
+        """Parse bounds string '[left,top][right,bottom]' to list."""
+        try:
+            # Remove brackets and split
+            clean = bounds_str.replace("][", ",").strip("[]")
+            parts = clean.split(",")
+            return [int(p) for p in parts[:4]]
+        except (ValueError, IndexError):
+            return [0, 0, 0, 0]
+
+    def _infer_role(self, class_name: str, node: etree._Element) -> str:
+        """Infer semantic role from class name."""
+        class_lower = class_name.lower()
+
+        if "button" in class_lower:
+            return "button"
+        if "edittext" in class_lower:
+            return "textfield"
+        if "textview" in class_lower:
+            return "text"
+        if "imageview" in class_lower:
+            return "image"
+        if "checkbox" in class_lower:
+            return "checkbox"
+        if "switch" in class_lower:
+            return "switch"
+        if "radiobutton" in class_lower:
+            return "radio"
+        if "recyclerview" in class_lower or "listview" in class_lower:
+            return "list"
+        if "scrollview" in class_lower:
+            return "scrollable"
+
+        # Fallback based on state
+        if node.get("clickable") == "true":
+            return "clickable"
+        if node.get("editable") == "true":
+            return "textfield"
+
+        return "element"

android_emu_agent/validation.py

@@ -0,0 +1,59 @@
+"""Validation helpers for user input."""
+
+from __future__ import annotations
+
+import re
+
+from android_emu_agent.errors import invalid_package_error, invalid_uri_error, not_emulator_error
+
+# Package name: starts with letter, segments separated by dots, each segment alphanumeric/underscore
+PACKAGE_PATTERN = re.compile(r"^[a-zA-Z][a-zA-Z0-9_]*(\.[a-zA-Z][a-zA-Z0-9_]*)+$")
+
+
+def validate_package(package: str) -> None:
+    """Validate Android package name format.
+
+    Args:
+        package: Package name to validate
+
+    Raises:
+        AgentError: If package name is invalid
+    """
+    if not PACKAGE_PATTERN.match(package):
+        raise invalid_package_error(package)
+
+
+def validate_uri(uri: str) -> None:
+    """Validate URI has a scheme.
+
+    Args:
+        uri: URI to validate
+
+    Raises:
+        AgentError: If URI is invalid
+    """
+    if not uri or "://" not in uri:
+        raise invalid_uri_error(uri)
+
+
+def get_console_port(serial: str) -> int:
+    """Extract emulator console port from serial.
+
+    Args:
+        serial: Device serial (e.g., 'emulator-5554')
+
+    Returns:
+        Console port number
+
+    Raises:
+        AgentError: If serial is not an emulator
+    """
+    if not serial.startswith("emulator-"):
+        raise not_emulator_error(serial)
+
+    try:
+        port = int(serial.split("-")[1])
+    except (IndexError, ValueError) as err:
+        raise not_emulator_error(serial) from err
+
+    return port

android_emu_agent-0.1.3.dist-info/METADATA

@@ -0,0 +1,375 @@
+Metadata-Version: 2.4
+Name: android-emu-agent
+Version: 0.1.3
+Summary: CLI + daemon for LLM-driven Android UI control on emulators and rooted devices
+Project-URL: Homepage, https://github.com/alehkot/android-emu-agent
+Author-email: Oleg Kot <kot.oleg@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: android,automation,llm,ui-testing
+Requires-Python: >=3.11
+Requires-Dist: adbutils>=2.0.0
+Requires-Dist: aiosqlite>=0.19.0
+Requires-Dist: fastapi>=0.128.1
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: structlog>=24.0.0
+Requires-Dist: typer>=0.9.0
+Requires-Dist: uiautomator2>=3.0.0
+Requires-Dist: uvicorn[standard]>=0.27.0
+Provides-Extra: dev
+Requires-Dist: lxml-stubs>=0.5.0; extra == 'dev'
+Requires-Dist: mypy>=1.8.0; extra == 'dev'
+Requires-Dist: pyright>=1.1.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.2.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Android Emu Agent
+
+A CLI + daemon system for LLM-driven Android UI control on emulators and rooted devices.
+
+## Overview
+
+Android Emu Agent automates Android apps using a fast observe-act-verify loop:
+
+1. Observe: capture a compact UI snapshot (interactive elements only)
+2. Act: issue commands using ephemeral element refs like `@a1`
+3. Verify: re-snapshot when needed
+
+The CLI is a thin client. A long-running daemon handles all device I/O.
+
+## Inspiration
+
+Inspired by [agent-browser](https://github.com/vercel-labs/agent-browser), a fast headless browser
+automation CLI for AI agents.
+
+## Requirements
+
+- Python 3.11+
+- `uv` package manager
+- Android SDK with `adb` on PATH
+- Android emulator or rooted device (primary target)
+
+## Install
+
+```bash
+# Clone the repository
+git clone https://github.com/alehkot/android-emu-agent.git
+cd android-emu-agent
+
+# Install dependencies
+uv sync
+```
+
+Inside this repo, prefer `uv run android-emu-agent <command>`.
+
+## Quick Start (2 minutes)
+
+```bash
+# 1. Start the daemon (optional, CLI will auto-start by default)
+uv run android-emu-agent daemon start
+
+# 2. List connected devices
+uv run android-emu-agent device list
+
+# 3. Start a session (prints the session id)
+uv run android-emu-agent session start --device emulator-5554
+
+# 4. Take a compact snapshot and read refs
+uv run android-emu-agent ui snapshot s-abc123 --format text
+
+# 5. Tap an element using a ref
+uv run android-emu-agent action tap s-abc123 @a1
+
+# 6. Stop the session
+uv run android-emu-agent session stop s-abc123
+```
+
+Most commands accept `--json` for machine-readable output.
+
+Example `--json` output for `session start`:
+
+```json
+{
+  "status": "done",
+  "session_id": "s-abc123",
+  "device_serial": "emulator-5554",
+  "generation": 0
+}
+```
+
+## Core Concepts
+
+Sessions
+
+- Sessions tie actions to a specific device. Most commands accept a session id.
+- `session start` returns a session id. `session stop` releases it.
+
+Snapshots and refs
+
+- `ui snapshot` returns `@refs` that are stable only for that snapshot.
+- If you get `ERR_STALE_REF`, take a new snapshot and use fresh refs.
+
+Selectors
+
+- `action tap` accepts an `@ref` or a selector.
+- `long-tap`, `set-text`, and `clear` require an `@ref`.
+
+Selector examples:
+
+```text
+@a1
+text:"Sign in"
+id:com.example:id/login_btn
+desc:"Open navigation"
+coords:120,450
+```
+
+## Snapshot Format
+
+Compact snapshot output includes context and interactive elements only:
+
+```json
+{
+  "schema_version": 1,
+  "session_id": "s-abc123",
+  "generation": 42,
+  "context": {
+    "package": "com.example.app",
+    "activity": ".MainActivity",
+    "orientation": "PORTRAIT",
+    "ime_visible": false
+  },
+  "elements": [
+    {
+      "ref": "@a1",
+      "role": "button",
+      "label": "Sign in",
+      "resource_id": "com.example:id/login_btn",
+      "bounds": [100, 200, 300, 250],
+      "state": { "clickable": true, "enabled": true }
+    }
+  ]
+}
+```
+
+## Daemon and State
+
+- Socket: `/tmp/android-emu-agent.sock`
+- Logs: `~/.android-emu-agent/daemon.log`
+- PID file: `~/.android-emu-agent/daemon.pid`
+
+The CLI auto-starts the daemon on first request. Use these to debug:
+
+```bash
+uv run android-emu-agent daemon status --json
+uv run android-emu-agent daemon stop
+uv run android-emu-agent daemon start
+```
+
+## Common Workflows
+
+Login flow
+
+```bash
+uv run android-emu-agent app launch s-abc123 com.example.app
+uv run android-emu-agent wait idle s-abc123 --timeout-ms 5000
+uv run android-emu-agent ui snapshot s-abc123 --format text
+uv run android-emu-agent action set-text s-abc123 @a3 "user@example.com"
+uv run android-emu-agent action set-text s-abc123 @a4 "hunter2"
+uv run android-emu-agent action tap s-abc123 @a5
+```
+
+When elements are missing
+
+```bash
+uv run android-emu-agent ui snapshot s-abc123 --full
+uv run android-emu-agent wait idle s-abc123 --timeout-ms 3000
+uv run android-emu-agent ui snapshot s-abc123
+```
+
+Visual debug
+
+```bash
+uv run android-emu-agent ui screenshot --device emulator-5554 --pull --output ./screen.png
+uv run android-emu-agent artifact bundle s-abc123
+```
+
+Emulator snapshots
+
+```bash
+uv run android-emu-agent emulator snapshot save emulator-5554 clean
+uv run android-emu-agent emulator snapshot restore emulator-5554 clean
+```
+
+These commands require an emulator serial (`emulator-5554`). If you pass a non-emulator serial, you
+will see `ERR_NOT_EMULATOR`.
+
+## Real Devices (Non-Root)
+
+The project targets emulators and rooted devices, but many commands do not enforce root and can work
+on real devices as long as `adb` is connected and uiautomator2 can attach (ATX server on port 7912).
+In practice, these are usually safe on non-root devices:
+
+- UI snapshots and screenshots
+- Actions (tap, set-text, swipe, scroll, back/home/recents)
+- Wait commands
+- App list/launch/force-stop/reset/deeplink
+- File `push` and `pull` to shared storage
+
+Emulator-only commands are `emulator snapshot save|restore`. Root-required commands are listed
+below.
+
+## Root-Required Operations
+
+The following require a rooted device or emulator with root access:
+
+- `reliability oom-adj`
+- `reliability pull anr`
+- `reliability pull tombstones`
+- `reliability pull dropbox`
+- `file find`
+- `file list`
+- `file app push`
+- `file app pull`
+
+If root is missing, you will see `ERR_PERMISSION`.
+
+## Artifacts
+
+Artifacts are written to `~/.android-emu-agent/artifacts` by default.
+
+- Reliability outputs: `~/.android-emu-agent/artifacts/reliability`
+- File transfers: `~/.android-emu-agent/artifacts/files`
+
+## Troubleshooting
+
+Quick checks
+
+```bash
+uv run android-emu-agent device list
+adb devices
+uv run android-emu-agent daemon status --json
+```
+
+Common errors
+
+| Error Code            | Meaning                  | Fix                                             |
+| --------------------- | ------------------------ | ----------------------------------------------- |
+| `ERR_STALE_REF`       | Ref from an old snapshot | Take a new snapshot                             |
+| `ERR_NOT_FOUND`       | Element not found        | Verify screen, use `--full` or a selector       |
+| `ERR_BLOCKED_INPUT`   | Dialog/IME blocking      | `wait idle` or `back`                           |
+| `ERR_TIMEOUT`         | Wait condition not met   | Increase `--timeout-ms` or check condition      |
+| `ERR_DEVICE_OFFLINE`  | Device disconnected      | Reconnect and re-run `device list`              |
+| `ERR_SESSION_EXPIRED` | Session is gone          | Start a new session                             |
+| `ERR_PERMISSION`      | Root required            | Use a rooted device/emulator                    |
+| `ERR_ADB_NOT_FOUND`   | `adb` not on PATH        | Install Android SDK and ensure `adb` is on PATH |
+| `ERR_ADB_COMMAND`     | ADB command failed       | Check device connectivity and retry             |
+
+For deeper guidance, see `skills/android-emu-agent/references/troubleshooting.md`.
+
+## CLI Reference
+
+A concise command list lives at `skills/android-emu-agent/references/command-reference.md`.
+
+If you prefer an interactive guide:
+
+```bash
+uv run android-emu-agent --help
+uv run android-emu-agent <group> --help
+```
+
+## Architecture
+
+```text
+┌─────────────────┐     Unix Socket      ┌──────────────────────────────────┐
+│                 │  ◄────────────────►  │            Daemon                │
+│   CLI Client    │                      │  ┌────────────────────────────┐  │
+│                 │                      │  │     Device Manager         │  │
+└─────────────────┘                      │  │  (adbutils, uiautomator2)  │  │
+                                         │  └────────────────────────────┘  │
+                                         │  ┌────────────────────────────┐  │
+                                         │  │    Session Manager         │  │
+                                         │  │  (refs, state, SQLite)     │  │
+                                         │  └────────────────────────────┘  │
+                                         │  ┌────────────────────────────┐  │
+                                         │  │    UI Snapshotter          │  │
+                                         │  │  (lxml, filtering)         │  │
+                                         │  └────────────────────────────┘  │
+                                         └──────────────────────────────────┘
+                                                         │
+                                                         ▼
+                                         ┌──────────────────────────────────┐
+                                         │     Android Device/Emulator      │
+                                         │  (ATX Server on port 7912)       │
+                                         └──────────────────────────────────┘
+```
+
+## Development
+
+```bash
+# Install with dev dependencies
+uv sync --all-extras
+
+# Run tests
+uv run pytest tests/unit -v
+
+# Run linter
+uv run ruff check .
+
+# Format code
+uv run ruff format .
+
+# Type check
+uv run mypy src/
+
+# Run all checks
+./scripts/dev.sh check
+```
+
+## Agent Skills
+
+This repo ships an `android-emu-agent` skill in `skills/android-emu-agent/` for agent environments
+that support skills. Use the install steps below.
+
+### Install for Codex
+
+```bash
+export CODEX_HOME="$HOME/.codex"
+mkdir -p "$CODEX_HOME/skills"
+ln -sfn "$(pwd)/skills/android-emu-agent" "$CODEX_HOME/skills/android-emu-agent"
+```
+
+### Install for Claude Code
+
+```bash
+mkdir -p ~/.claude/skills
+ln -sfn "$(pwd)/skills/android-emu-agent" ~/.claude/skills/android-emu-agent
+```
+
+If symlinks are not an option, copy the directory instead.
+
+### Using the Skill
+
+After installation, the agent should be primed to start a session with your connected device before
+you ask for specific actions. Begin with a direct initialization request, for example:
+
+`I want to interact with my connected Android emulator.`
+
+Once the agent has initialized the session, you can proceed with normal requests (snapshots,
+tap/type actions, app launches, waits, etc.).
+
+Prerequisites you may need (if the agent reports it cannot connect):
+
+1. Start the daemon: `uv run android-emu-agent daemon start`
+2. Confirm a device is visible: `uv run android-emu-agent device list`
+3. If needed, start a session explicitly:
+   `uv run android-emu-agent session start --device emulator-5554`
+
+## License
+
+MIT. See `LICENSE`.