androidctl 0.1.0__py3-none-any.whl

androidctld/semantics/targets.py

@@ -0,0 +1,167 @@
+"""Semantic compiler target and promotion rules."""
+
+from __future__ import annotations
+
+from androidctld.semantics.labels import (
+    GENERIC_SEMANTIC_ROLES,
+    LabelInfo,
+    infer_role,
+    normalize_text,
+    parent_node_for,
+    synthesize_label_info,
+)
+from androidctld.snapshots.models import RawNode
+
+CLICK_ACTIONS = {"tap", "click"}
+LONG_CLICK_ACTIONS = {"longTap", "longClick"}
+TYPE_ACTIONS = {"setText"}
+SCROLL_ACTIONS = {
+    "scrollForward",
+    "scrollBackward",
+    "scrollUp",
+    "scrollDown",
+    "scrollLeft",
+    "scrollRight",
+}
+SUBMIT_ACTIONS = {"submit"}
+
+
+def semantic_actions_for(raw_node: RawNode) -> list[str]:
+    raw_actions = {normalize_action_name(action) for action in raw_node.actions}
+    semantic_actions: list[str] = []
+    if raw_actions.intersection(CLICK_ACTIONS):
+        semantic_actions.append("tap")
+    if raw_actions.intersection(LONG_CLICK_ACTIONS):
+        semantic_actions.append("longTap")
+    if raw_node.editable and raw_actions.intersection(TYPE_ACTIONS):
+        semantic_actions.append("type")
+    if raw_node.scrollable and raw_actions.intersection(SCROLL_ACTIONS):
+        semantic_actions.append("scroll")
+    return semantic_actions
+
+
+def public_primary_actions_for(
+    *,
+    anchor_node: RawNode,
+    role: str,
+    primary_actions: list[str],
+) -> list[str]:
+    if role != "input" or not anchor_node.editable:
+        return list(primary_actions)
+    if anchor_node.focused:
+        return list(primary_actions)
+    return []
+
+
+def secondary_public_actions_for(
+    *,
+    anchor_node: RawNode,
+    role: str,
+    primary_actions: list[str],
+) -> list[str]:
+    del primary_actions
+    anchor_actions = {normalize_action_name(action) for action in anchor_node.actions}
+    if role != "input" or not anchor_node.editable:
+        return []
+    secondary_actions: list[str] = []
+    if anchor_node.focused:
+        if anchor_actions.intersection(SUBMIT_ACTIONS):
+            secondary_actions.append("submit")
+        return secondary_actions
+    if "focus" not in anchor_actions:
+        return secondary_actions
+    secondary_actions.append("focus")
+    return secondary_actions
+
+
+def select_target_sources(
+    raw_nodes: tuple[RawNode, ...],
+    raw_nodes_by_rid: dict[str, RawNode],
+    label_infos: dict[str, LabelInfo],
+    actionability: dict[str, list[str]],
+) -> dict[str, str]:
+    target_sources: dict[str, str] = {}
+    for raw_node in raw_nodes:
+        anchor_node = actionable_anchor_for(raw_node, raw_nodes_by_rid, actionability)
+        if anchor_node is None:
+            continue
+        existing_source = target_sources.get(anchor_node.rid)
+        if existing_source is None or target_source_sort_key(
+            raw_nodes_by_rid[existing_source],
+            anchor_node,
+            raw_nodes_by_rid,
+            label_infos,
+        ) < target_source_sort_key(
+            raw_node,
+            anchor_node,
+            raw_nodes_by_rid,
+            label_infos,
+        ):
+            target_sources[anchor_node.rid] = raw_node.rid
+    return target_sources
+
+
+def actionable_anchor_for(
+    raw_node: RawNode,
+    raw_nodes_by_rid: dict[str, RawNode],
+    actionability: dict[str, list[str]],
+) -> RawNode | None:
+    if actionability[raw_node.rid]:
+        return raw_node
+    current = parent_node_for(raw_node, raw_nodes_by_rid)
+    while current is not None:
+        anchor_actions = actionability[current.rid]
+        if anchor_actions:
+            if can_promote_to_actionable_ancestor(raw_node, anchor_actions):
+                return current
+            return None
+        current = parent_node_for(current, raw_nodes_by_rid)
+    return None
+
+
+def can_promote_to_actionable_ancestor(
+    raw_node: RawNode,
+    anchor_actions: list[str],
+) -> bool:
+    if "scroll" in anchor_actions:
+        return False
+    label_quality = synthesize_label_info(raw_node, {raw_node.rid: raw_node}).quality
+    role = infer_role(raw_node)
+    return label_quality > 0 or role not in GENERIC_SEMANTIC_ROLES
+
+
+def target_source_sort_key(
+    raw_node: RawNode,
+    anchor_node: RawNode,
+    raw_nodes_by_rid: dict[str, RawNode],
+    label_infos: dict[str, LabelInfo],
+) -> tuple[int, int, int, int, int, str]:
+    label_info = label_infos[raw_node.rid]
+    role = infer_role(raw_node)
+    actionable_role_bonus = int(role not in GENERIC_SEMANTIC_ROLES)
+    return (
+        label_info.quality,
+        actionable_role_bonus,
+        int(raw_node.rid != anchor_node.rid),
+        -ancestor_distance(raw_node, anchor_node.rid, raw_nodes_by_rid),
+        len(normalize_text(label_info.label)),
+        raw_node.rid,
+    )
+
+
+def ancestor_distance(
+    raw_node: RawNode,
+    ancestor_rid: str,
+    raw_nodes_by_rid: dict[str, RawNode],
+) -> int:
+    distance = 0
+    current: RawNode | None = raw_node
+    while current is not None and current.rid != ancestor_rid:
+        current = parent_node_for(current, raw_nodes_by_rid)
+        distance += 1
+    return distance
+
+
+def normalize_action_name(value: str) -> str:
+    normalized = normalize_text(value)
+    return normalized[:1].lower() + normalized[1:]

androidctld/snapshots/__init__.py

@@ -0,0 +1 @@
+"""Raw snapshot fetching and caching."""

androidctld/snapshots/models.py

@@ -0,0 +1,219 @@
+"""Validated raw snapshot domain models and parsing helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from pydantic import ValidationError
+
+from androidctld.device.errors import DeviceBootstrapError, device_rpc_failed
+from androidctld.schema.core import SchemaDecodeError
+from androidctld.schema.validation_errors import validation_error_to_schema_decode_error
+from androidctld.snapshots.schema import (
+    RawDisplayPayload,
+    RawImePayload,
+    RawNodePayload,
+    RawSnapshotPayload,
+    RawWindowPayload,
+)
+
+
+@dataclass(frozen=True)
+class RawIme:
+    visible: bool
+    window_id: str | None
+
+
+@dataclass(frozen=True)
+class RawWindow:
+    window_id: str
+    type: str
+    layer: int
+    package_name: str | None
+    bounds: tuple[int, int, int, int]
+    root_rid: str
+
+
+@dataclass(frozen=True)
+class RawNode:
+    rid: str
+    window_id: str
+    parent_rid: str | None
+    child_rids: tuple[str, ...]
+    class_name: str
+    resource_id: str | None
+    text: str | None
+    content_desc: str | None
+    hint_text: str | None
+    state_description: str | None
+    pane_title: str | None
+    package_name: str | None
+    bounds: tuple[int, int, int, int]
+    visible_to_user: bool
+    important_for_accessibility: bool
+    clickable: bool
+    enabled: bool
+    editable: bool
+    focusable: bool
+    focused: bool
+    checkable: bool
+    checked: bool
+    selected: bool
+    scrollable: bool
+    password: bool
+    actions: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class RawSnapshot:
+    snapshot_id: int
+    captured_at: str
+    package_name: str | None
+    activity_name: str | None
+    ime: RawIme
+    windows: tuple[RawWindow, ...]
+    nodes: tuple[RawNode, ...]
+    display: dict[str, int]
+
+
+def parse_raw_snapshot(payload: object) -> RawSnapshot:
+    try:
+        boundary = RawSnapshotPayload.model_validate(payload)
+    except ValidationError as error:
+        raise invalid_snapshot_validation_error(error, field_name="result") from error
+    try:
+        return adapt_raw_snapshot(boundary, field_name="result")
+    except SchemaDecodeError as error:
+        raise invalid_snapshot_payload(error.field, error.problem) from error
+
+
+def adapt_raw_snapshot(
+    payload: RawSnapshotPayload,
+    *,
+    field_name: str = "result",
+) -> RawSnapshot:
+    return RawSnapshot(
+        snapshot_id=payload.snapshot_id,
+        captured_at=payload.captured_at,
+        package_name=payload.package_name,
+        activity_name=payload.activity_name,
+        ime=adapt_ime(payload.ime, field_name=f"{field_name}.ime"),
+        windows=tuple(
+            adapt_raw_window(window, field_name=f"{field_name}.windows[{index}]")
+            for index, window in enumerate(payload.windows)
+        ),
+        nodes=tuple(
+            adapt_raw_node(node, field_name=f"{field_name}.nodes[{index}]")
+            for index, node in enumerate(payload.nodes)
+        ),
+        display=adapt_display(payload.display, field_name=f"{field_name}.display"),
+    )
+
+
+def adapt_display(
+    payload: RawDisplayPayload,
+    *,
+    field_name: str = "result.display",
+) -> dict[str, int]:
+    del field_name
+    return {
+        "widthPx": payload.width_px,
+        "heightPx": payload.height_px,
+        "densityDpi": payload.density_dpi,
+        "rotation": payload.rotation,
+    }
+
+
+def adapt_ime(
+    payload: RawImePayload,
+    *,
+    field_name: str = "result.ime",
+) -> RawIme:
+    del field_name
+    return RawIme(
+        visible=payload.visible,
+        window_id=payload.window_id,
+    )
+
+
+def adapt_raw_window(
+    payload: RawWindowPayload,
+    *,
+    field_name: str,
+) -> RawWindow:
+    return RawWindow(
+        window_id=payload.window_id,
+        type=payload.type,
+        layer=payload.layer,
+        package_name=payload.package_name,
+        bounds=adapt_bounds(payload.bounds, field_name=f"{field_name}.bounds"),
+        root_rid=payload.root_rid,
+    )
+
+
+def adapt_raw_node(
+    payload: RawNodePayload,
+    *,
+    field_name: str,
+) -> RawNode:
+    return RawNode(
+        rid=payload.rid,
+        window_id=payload.window_id,
+        parent_rid=payload.parent_rid,
+        child_rids=tuple(payload.child_rids),
+        class_name=payload.class_name,
+        resource_id=payload.resource_id,
+        text=payload.text,
+        content_desc=payload.content_desc,
+        hint_text=payload.hint_text,
+        state_description=payload.state_description,
+        pane_title=payload.pane_title,
+        package_name=payload.package_name,
+        bounds=adapt_bounds(payload.bounds, field_name=f"{field_name}.bounds"),
+        visible_to_user=payload.visible_to_user,
+        important_for_accessibility=payload.important_for_accessibility,
+        clickable=payload.clickable,
+        enabled=payload.enabled,
+        editable=payload.editable,
+        focusable=payload.focusable,
+        focused=payload.focused,
+        checkable=payload.checkable,
+        checked=payload.checked,
+        selected=payload.selected,
+        scrollable=payload.scrollable,
+        password=payload.password,
+        actions=tuple(payload.actions),
+    )
+
+
+def adapt_bounds(
+    bounds: list[int],
+    *,
+    field_name: str,
+) -> tuple[int, int, int, int]:
+    if len(bounds) != 4:
+        raise SchemaDecodeError(field_name, "must contain exactly 4 integers")
+    return (bounds[0], bounds[1], bounds[2], bounds[3])
+
+
+def invalid_snapshot_validation_error(
+    error: ValidationError,
+    *,
+    field_name: str,
+) -> DeviceBootstrapError:
+    schema_error = validation_error_to_schema_decode_error(
+        error,
+        field_name=field_name,
+    )
+    return invalid_snapshot_payload(schema_error.field, schema_error.problem)
+
+
+def invalid_snapshot_payload(field_name: str, problem: str) -> DeviceBootstrapError:
+    return device_rpc_failed(
+        f"device RPC {field_name} {problem}",
+        {
+            "field": field_name,
+            "reason": "invalid_snapshot",
+        },
+        retryable=False,
+    )

androidctld/snapshots/refresh.py

@@ -0,0 +1,273 @@
+"""Shared screen refresh transaction and signature helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Any
+
+from androidctld.artifacts.models import ScreenArtifacts
+from androidctld.artifacts.writer import ArtifactWriter
+from androidctld.commands.models import CommandRecord
+from androidctld.errors import DaemonError, DaemonErrorCode
+from androidctld.protocol import CommandKind
+from androidctld.refs.models import RefRegistry
+from androidctld.refs.service import RefRegistryBuilder
+from androidctld.runtime import RuntimeKernel, RuntimeLifecycleLease
+from androidctld.runtime.kernel import ScreenRefreshUpdate
+from androidctld.runtime.models import WorkspaceRuntime
+from androidctld.semantics.compiler import (
+    CompiledScreen,
+    SemanticCompiler,
+    SemanticNode,
+)
+from androidctld.semantics.public_models import PublicScreen
+from androidctld.snapshots.models import RawSnapshot
+from androidctld.text_equivalence import canonical_text_key, searchable_raw_node_texts
+
+RefreshScreenResult = tuple[
+    RawSnapshot,
+    PublicScreen,
+    ScreenArtifacts,
+]
+RefreshCandidateValidator = Callable[[RawSnapshot, PublicScreen, CompiledScreen], None]
+
+
+@dataclass(frozen=True)
+class ScreenRefreshBasis:
+    sequence: int
+    previous_registry: RefRegistry
+    lifecycle_lease: RuntimeLifecycleLease | None
+    command_kind: CommandKind | None
+    wait_kind: object | None
+    record: CommandRecord | None
+
+
+def _normalize_wait_kind(wait_kind: object | None) -> str | None:
+    if wait_kind is None:
+        return None
+    return str(getattr(wait_kind, "value", wait_kind))
+
+
+class ScreenRefreshService:
+    def __init__(
+        self,
+        runtime_kernel: RuntimeKernel,
+        semantic_compiler: SemanticCompiler | None = None,
+        artifact_writer: ArtifactWriter | None = None,
+        ref_registry_builder: RefRegistryBuilder | None = None,
+    ) -> None:
+        self._runtime_kernel = runtime_kernel
+        self._semantic_compiler = semantic_compiler or SemanticCompiler()
+        self._artifact_writer = artifact_writer or ArtifactWriter()
+        self._ref_registry_builder = ref_registry_builder or RefRegistryBuilder()
+
+    @property
+    def runtime_kernel(self) -> RuntimeKernel:
+        return self._runtime_kernel
+
+    def refresh(
+        self,
+        session: WorkspaceRuntime,
+        snapshot: RawSnapshot,
+        *,
+        lifecycle_lease: RuntimeLifecycleLease | None = None,
+        command_kind: CommandKind | None = None,
+        wait_kind: object | None = None,
+        record: CommandRecord | None = None,
+        candidate_validator: RefreshCandidateValidator | None = None,
+    ) -> RefreshScreenResult:
+        basis = _capture_refresh_basis(
+            session,
+            lifecycle_lease=lifecycle_lease,
+            command_kind=command_kind,
+            wait_kind=wait_kind,
+            record=record,
+        )
+        compiled_screen = self._semantic_compiler.compile(basis.sequence, snapshot)
+        reconcile_result = self._ref_registry_builder.finalize_compiled_screen(
+            compiled_screen=compiled_screen,
+            snapshot_id=snapshot.snapshot_id,
+            previous_registry=basis.previous_registry,
+        )
+        ref_registry = reconcile_result.registry
+        finalized_compiled_screen = reconcile_result.compiled_screen
+        public_screen = finalized_compiled_screen.to_public_screen()
+        staged_artifacts = self._artifact_writer.stage_screen(
+            session,
+            public_screen,
+            sequence=finalized_compiled_screen.sequence,
+            source_snapshot_id=finalized_compiled_screen.source_snapshot_id,
+            captured_at=finalized_compiled_screen.captured_at,
+            ref_registry=ref_registry,
+        )
+        artifacts = staged_artifacts.artifacts
+
+        def raise_if_refresh_stale(active_session: WorkspaceRuntime) -> None:
+            raise_if_stale(
+                active_session,
+                basis.lifecycle_lease,
+                kind=basis.command_kind,
+                wait_kind=basis.wait_kind,
+                record=basis.record,
+            )
+            if candidate_validator is not None:
+                candidate_validator(snapshot, public_screen, finalized_compiled_screen)
+
+        self._runtime_kernel.commit_screen_refresh(
+            session,
+            update=ScreenRefreshUpdate(
+                sequence=basis.sequence,
+                snapshot=snapshot,
+                public_screen=public_screen,
+                compiled_screen=finalized_compiled_screen,
+                artifacts=artifacts,
+                ref_registry=ref_registry,
+                staged_artifacts=staged_artifacts,
+            ),
+            pre_commit=raise_if_refresh_stale,
+        )
+        return snapshot, public_screen, artifacts
+
+
+def _capture_refresh_basis(
+    session: WorkspaceRuntime,
+    *,
+    lifecycle_lease: RuntimeLifecycleLease | None,
+    command_kind: CommandKind | None,
+    wait_kind: object | None,
+    record: CommandRecord | None,
+) -> ScreenRefreshBasis:
+    with session.lock:
+        raise_if_stale(
+            session,
+            lifecycle_lease,
+            kind=command_kind,
+            wait_kind=wait_kind,
+            record=record,
+        )
+        return ScreenRefreshBasis(
+            sequence=session.screen_sequence + 1,
+            previous_registry=deepcopy(session.ref_registry),
+            lifecycle_lease=lifecycle_lease,
+            command_kind=command_kind,
+            wait_kind=deepcopy(wait_kind),
+            record=deepcopy(record),
+        )
+
+
+def raise_if_stale(
+    session: WorkspaceRuntime,
+    lease: RuntimeLifecycleLease | None,
+    *,
+    kind: CommandKind | None = None,
+    wait_kind: object | None = None,
+    record: CommandRecord | None = None,
+) -> None:
+    if lease is None or lease.is_current(session):
+        return
+    details = {"workspaceRoot": session.workspace_root.as_posix()}
+    if record is not None:
+        details["commandId"] = record.command_id
+    if kind is not None:
+        details["kind"] = kind.value
+    normalized_wait_kind = _normalize_wait_kind(wait_kind)
+    if kind is CommandKind.WAIT and normalized_wait_kind is not None:
+        details["waitKind"] = normalized_wait_kind
+    raise DaemonError(
+        code=DaemonErrorCode.COMMAND_CANCELLED,
+        message=(
+            "command was canceled"
+            if kind is None
+            else (
+                f"wait {normalized_wait_kind} was canceled"
+                if kind is CommandKind.WAIT and normalized_wait_kind is not None
+                else f"{kind.value} was canceled"
+            )
+        ),
+        retryable=False,
+        details=details,
+        http_status=200,
+    )
+
+
+def compiled_screen_nodes(compiled_screen: CompiledScreen) -> tuple[SemanticNode, ...]:
+    return (
+        *compiled_screen.targets,
+        *compiled_screen.context,
+        *compiled_screen.dialog,
+        *compiled_screen.keyboard,
+        *compiled_screen.system,
+    )
+
+
+def compiled_screen_signature(
+    compiled_screen: CompiledScreen | None,
+    snapshot: RawSnapshot,
+) -> tuple[Any, ...]:
+    if compiled_screen is None:
+        return (
+            snapshot.package_name,
+            snapshot.activity_name,
+            tuple(
+                tuple(
+                    canonical_text_key(value)
+                    for value in searchable_raw_node_texts(node)
+                )
+                for node in snapshot.nodes
+                if node.visible_to_user
+            ),
+        )
+    return (
+        compiled_screen.package_name,
+        compiled_screen.activity_name,
+        compiled_screen.keyboard_visible,
+        tuple(
+            (
+                node.group,
+                node.role,
+                canonical_text_key(node.label),
+                tuple(canonical_text_key(value) for value in node.state),
+                tuple(node.actions),
+                node.ref,
+                None if node.bounds is None else tuple(node.bounds),
+            )
+            for node in compiled_screen_nodes(compiled_screen)
+        ),
+    )
+
+
+def settle_screen_signature(
+    compiled_screen: CompiledScreen | None,
+    snapshot: RawSnapshot,
+) -> tuple[Any, ...]:
+    if compiled_screen is None:
+        return (
+            snapshot.package_name,
+            snapshot.activity_name,
+            tuple(
+                tuple(
+                    canonical_text_key(value)
+                    for value in searchable_raw_node_texts(node)
+                )
+                for node in snapshot.nodes
+                if node.visible_to_user
+            ),
+        )
+    return (
+        compiled_screen.package_name,
+        compiled_screen.activity_name,
+        compiled_screen.keyboard_visible,
+        tuple(
+            (
+                node.group,
+                node.role,
+                canonical_text_key(node.label),
+                tuple(canonical_text_key(value) for value in node.state),
+                tuple(node.actions),
+                None if node.bounds is None else tuple(node.bounds),
+            )
+            for node in compiled_screen_nodes(compiled_screen)
+        ),
+    )