screenforge 0.4.0__py3-none-any.whl

common/capabilities.py

@@ -0,0 +1,121 @@
+from common.runtime_modes import MODE_DOCTOR, MODE_DRY_RUN, MODE_PLAN_ONLY, MODE_RUN
+
+SUPPORTED_PLATFORMS = ("android", "ios", "web")
+SUPPORTED_ACTIONS = (
+    "goto",
+    "click",
+    "long_click",
+    "hover",
+    "input",
+    "swipe",
+    "press",
+    "scroll_into_view",
+    "select",
+    "upload",
+    "double_click",
+    "right_click",
+    "drag",
+    "wait_for",
+    "assert_exist",
+    "assert_not_exist",
+    "assert_text_equals",
+    "assert_text_contains",
+    "assert_value",
+    "assert_url",
+)
+# Assertions produce a verification VERDICT (the system-under-test did/did not
+# meet the condition) rather than an engine error — execute_and_record tags
+# their failures with assertion_failed so callers/--json can disambiguate.
+ASSERTION_ACTIONS = {
+    "assert_exist",
+    "assert_not_exist",
+    "assert_text_equals",
+    "assert_text_contains",
+    "assert_value",
+    "assert_url",
+}
+# assert_url reads page.url, not an element — it needs no locator (web-global).
+GLOBAL_ACTIONS = {"goto", "swipe", "press", "assert_url"}
+# Actions with a clean Playwright API but no robust coordinate-free mobile
+# equivalent. Engaged only on web; on android/ios the handler fails honestly
+# rather than emitting a brittle coordinate-based step (see P2 coordinate
+# honesty). assert_url is web-only for a different reason (reads page.url).
+WEB_ONLY_ACTIONS = {
+    "goto",
+    "scroll_into_view",
+    "select",
+    "upload",
+    "double_click",
+    "right_click",
+    "drag",
+    "assert_url",
+}
+ACTIONS_REQUIRING_EXTRA_VALUE = {
+    "goto",
+    "input",
+    "select",
+    "upload",
+    "drag",
+    "assert_text_equals",
+    "assert_text_contains",
+    "assert_value",
+    "assert_url",
+}
+CONTROL_PLANES = ("goal", "workflow", "action", "doctor")
+EXECUTION_MODES = (MODE_RUN, MODE_DOCTOR, MODE_PLAN_ONLY, MODE_DRY_RUN)
+
+# Which locator_type values actually resolve on each platform. This mirrors the
+# real executor / UI-compressor behavior, NOT aspiration:
+#   - web: compress_web_dom emits ref/bbox; LocatorBuilder maps css/text/desc.
+#   - android: utils_xml emits resource-id/text/content-desc (no ref/bbox).
+#   - ios: utils_ios maps text/desc -> label/name (no ref/bbox); resourceId->name.
+# An agent should read this instead of assuming `ref` works everywhere.
+LOCATORS_BY_PLATFORM = {
+    "web": ["css", "ref", "text", "description"],
+    "android": ["resourceId", "text", "description"],
+    "ios": ["text", "description"],
+}
+
+# Platform-gated location features. ref/bbox and the VLM visual fallback are
+# web-only (mobile UI-tree compressors don't emit ref/bbox, and the visual
+# fallback in executor.py is gated on platform == "web").
+FEATURES_BY_PLATFORM = {
+    "ref_bbox": ["web"],
+    "screenshot_annotation": ["web"],
+    "visual_fallback": ["web"],
+}
+
+
+def get_capabilities_payload() -> dict:
+    return {
+        "platforms": list(SUPPORTED_PLATFORMS),
+        "execution_modes": list(EXECUTION_MODES),
+        "control_planes": list(CONTROL_PLANES),
+        "supported_actions": list(SUPPORTED_ACTIONS),
+        "global_actions": sorted(GLOBAL_ACTIONS),
+        "web_only_actions": sorted(WEB_ONLY_ACTIONS),
+        "actions_requiring_extra_value": sorted(ACTIONS_REQUIRING_EXTRA_VALUE),
+        "locators": {p: list(v) for p, v in LOCATORS_BY_PLATFORM.items()},
+        "locator_priority": ["css", "resourceId", "text", "description"],
+        "features": {f: list(v) for f, v in FEATURES_BY_PLATFORM.items()},
+        "supports": {
+            "doctor": True,
+            "resume": True,
+            "workflow": True,
+            "workflow_vars": True,
+            "action": True,
+            "inspect_ui": True,
+            "case_memory": True,
+            "run_assets": True,
+            "load_run": True,
+            "tool_request": True,
+            "tool_stdin": True,
+            "mcp_server": True,
+            "json_events": True,
+            "goal_cli_human_mode_only": True,
+        },
+        "docs": {
+            "capability_matrix": "docs/capability-matrix.md",
+            "agent_guide": "docs/agent_guide.md",
+        },
+    }

common/case_memory.py

@@ -0,0 +1,327 @@
+import hashlib
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+import config.config as config
+from common.logs import log
+
+
+def _now_iso() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+def _normalize_text(value: str) -> str:
+    return str(value or "").strip()
+
+
+def _slugify(value: str) -> str:
+    text = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff]+", "-", _normalize_text(value)).strip("-")
+    return text.lower() or "memory"
+
+
+def _build_memory_id(
+    platform: str,
+    control_kind: str,
+    control_label: str,
+    source_ref: str,
+) -> str:
+    identity_seed = f"{platform}|{control_kind}|{control_label}|{source_ref}"
+    digest = hashlib.sha1(identity_seed.encode("utf-8")).hexdigest()[:10]
+    human_label = _slugify(source_ref or control_label)[:48]
+    return f"{platform}:{control_kind}:{human_label}:{digest}"
+
+
+def _merge_unique_strings(existing: list[str], incoming: list[str]) -> list[str]:
+    merged = []
+    seen = set()
+    for item in [*existing, *incoming]:
+        normalized = _normalize_text(item)
+        if not normalized or normalized in seen:
+            continue
+        merged.append(normalized)
+        seen.add(normalized)
+    return merged
+
+
+def _merge_locator_hints(
+    existing: list["LocatorHint"],
+    incoming: list["LocatorHint"],
+) -> list["LocatorHint"]:
+    merged: list["LocatorHint"] = []
+    seen = set()
+    for item in [*existing, *incoming]:
+        key = (
+            _normalize_text(item.action),
+            _normalize_text(item.locator_type),
+            _normalize_text(item.locator_value),
+        )
+        if not all(key) or key in seen:
+            continue
+        merged.append(
+            LocatorHint(
+                action=key[0],
+                locator_type=key[1],
+                locator_value=key[2],
+            )
+        )
+        seen.add(key)
+    return merged
+
+
+class LocatorHint(BaseModel):
+    action: str = ""
+    locator_type: str = ""
+    locator_value: str = ""
+
+
+class CaseMemoryEntry(BaseModel):
+    memory_id: str
+    platform: str
+    control_kind: str
+    control_label: str
+    source_ref: str = ""
+    success_count: int = 0
+    failure_count: int = 0
+    last_status: str = ""
+    last_run_id: str = ""
+    last_used_at: str = ""
+    successful_actions: list[str] = Field(default_factory=list)
+    locator_hints: list[LocatorHint] = Field(default_factory=list)
+    pytest_asset: dict[str, Any] = Field(default_factory=dict)
+    recommended_next_step: dict[str, Any] | None = None
+
+
+class CaseMemoryDocument(BaseModel):
+    version: int = 1
+    updated_at: str = ""
+    entries: list[CaseMemoryEntry] = Field(default_factory=list)
+
+
+def _collect_successful_actions(step_records: list[dict[str, Any]]) -> list[str]:
+    return [
+        _normalize_text(item.get("action_description", ""))
+        for item in step_records
+        if item.get("event") == "action_executed"
+        and item.get("success") is True
+        and _normalize_text(item.get("action_description", ""))
+    ]
+
+
+def _collect_locator_hints(
+    summary: dict[str, Any],
+    step_records: list[dict[str, Any]],
+) -> list[LocatorHint]:
+    hints: list[LocatorHint] = []
+    control_summary = summary.get("control_summary", {}) or {}
+    tuples_seen = set()
+
+    def _append_hint(action: str, locator_type: str, locator_value: str) -> None:
+        normalized_action = _normalize_text(action)
+        normalized_type = _normalize_text(locator_type)
+        normalized_value = _normalize_text(locator_value)
+        if (
+            not normalized_action
+            or not normalized_type
+            or normalized_type.lower() == "global"
+            or not normalized_value
+            or normalized_value.lower() == "global"
+        ):
+            return
+        key = (normalized_action, normalized_type, normalized_value)
+        if key in tuples_seen:
+            return
+        tuples_seen.add(key)
+        hints.append(
+            LocatorHint(
+                action=normalized_action,
+                locator_type=normalized_type,
+                locator_value=normalized_value,
+            )
+        )
+
+    _append_hint(
+        control_summary.get("action", ""),
+        control_summary.get("locator_type", ""),
+        control_summary.get("locator_value", ""),
+    )
+
+    for item in step_records:
+        _append_hint(
+            item.get("action", ""),
+            item.get("locator_type", ""),
+            item.get("locator_value", ""),
+        )
+
+    return hints
+
+
+class CaseMemoryStore:
+    def __init__(self, file_path: str | Path | None = None):
+        self._file_path = Path(file_path or config.CASE_MEMORY_PATH).expanduser()
+
+    @property
+    def file_path(self) -> Path:
+        return self._file_path
+
+    def load_document(self) -> CaseMemoryDocument:
+        if not self._file_path.exists():
+            return CaseMemoryDocument(updated_at=_now_iso())
+
+        try:
+            payload = json.loads(self._file_path.read_text(encoding="utf-8"))
+            return CaseMemoryDocument.model_validate(payload)
+        except Exception as e:
+            log.warning(f"[Warning] Failed to read case memory, falling back to empty store: {e}")
+            return CaseMemoryDocument(updated_at=_now_iso())
+
+    def save_document(self, document: CaseMemoryDocument) -> None:
+        self._file_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = self._file_path.with_suffix(self._file_path.suffix + ".tmp")
+        tmp_path.write_text(
+            json.dumps(document.model_dump(), ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        tmp_path.replace(self._file_path)
+
+    def query_entries(
+        self,
+        platform: str = "",
+        control_kind: str = "",
+        query: str = "",
+        source_ref: str = "",
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        document = self.load_document()
+        normalized_platform = _normalize_text(platform).lower()
+        normalized_kind = _normalize_text(control_kind).lower()
+        normalized_query = _normalize_text(query).lower()
+        normalized_source_ref = _normalize_text(source_ref)
+        limit = max(1, int(limit or 20))
+
+        matched_entries = []
+        for entry in document.entries:
+            if normalized_platform and entry.platform.lower() != normalized_platform:
+                continue
+            if normalized_kind and entry.control_kind.lower() != normalized_kind:
+                continue
+            if normalized_source_ref and entry.source_ref != normalized_source_ref:
+                continue
+            if normalized_query:
+                haystacks = [
+                    entry.control_label.lower(),
+                    entry.source_ref.lower(),
+                    " ".join(entry.successful_actions).lower(),
+                ]
+                if not any(normalized_query in haystack for haystack in haystacks):
+                    continue
+            matched_entries.append(entry.model_dump())
+
+        matched_entries.sort(
+            key=lambda item: (
+                item.get("last_used_at", ""),
+                item.get("success_count", 0),
+            ),
+            reverse=True,
+        )
+        return matched_entries[:limit]
+
+    def find_entry(
+        self,
+        platform: str,
+        control_kind: str,
+        control_label: str,
+        source_ref: str = "",
+    ) -> dict[str, Any] | None:
+        normalized_platform = _normalize_text(platform).lower()
+        normalized_kind = _normalize_text(control_kind).lower()
+        normalized_label = _normalize_text(control_label)
+        normalized_source_ref = _normalize_text(source_ref)
+        document = self.load_document()
+
+        for entry in document.entries:
+            if entry.platform.lower() != normalized_platform:
+                continue
+            if entry.control_kind.lower() != normalized_kind:
+                continue
+            if normalized_source_ref and entry.source_ref == normalized_source_ref:
+                return entry.model_dump()
+            if normalized_label and entry.control_label == normalized_label:
+                return entry.model_dump()
+        return None
+
+    def upsert_from_run(
+        self,
+        summary: dict[str, Any],
+        step_records: list[dict[str, Any]],
+    ) -> dict[str, Any] | None:
+        if _normalize_text(summary.get("execution_mode", "")) != "run":
+            return None
+
+        control_summary = summary.get("control_summary", {}) or {}
+        control_kind = _normalize_text(control_summary.get("control_kind", ""))
+        if not control_kind or control_kind == "doctor":
+            return None
+
+        platform = _normalize_text(summary.get("platform", ""))
+        control_label = _normalize_text(control_summary.get("control_label", "")) or _normalize_text(
+            summary.get("goal", "")
+        )
+        source_ref = _normalize_text(control_summary.get("source_ref", ""))
+        if not platform or not control_label:
+            return None
+
+        document = self.load_document()
+        existing_entry = None
+        for entry in document.entries:
+            if entry.platform != platform or entry.control_kind != control_kind:
+                continue
+            if source_ref and entry.source_ref == source_ref:
+                existing_entry = entry
+                break
+            if entry.control_label == control_label:
+                existing_entry = entry
+                break
+
+        if existing_entry is None:
+            existing_entry = CaseMemoryEntry(
+                memory_id=_build_memory_id(
+                    platform=platform,
+                    control_kind=control_kind,
+                    control_label=control_label,
+                    source_ref=source_ref,
+                ),
+                platform=platform,
+                control_kind=control_kind,
+                control_label=control_label,
+                source_ref=source_ref,
+            )
+            document.entries.append(existing_entry)
+
+        status = _normalize_text(summary.get("status", ""))
+        if status == "success":
+            existing_entry.success_count += 1
+        else:
+            existing_entry.failure_count += 1
+
+        existing_entry.last_status = status
+        existing_entry.last_run_id = _normalize_text(summary.get("run_id", ""))
+        existing_entry.last_used_at = _normalize_text(summary.get("finished_at", "")) or _now_iso()
+        existing_entry.successful_actions = _merge_unique_strings(
+            existing_entry.successful_actions,
+            _collect_successful_actions(step_records),
+        )
+        existing_entry.locator_hints = _merge_locator_hints(
+            existing_entry.locator_hints,
+            _collect_locator_hints(summary, step_records),
+        )
+        existing_entry.pytest_asset = dict(summary.get("pytest_asset", {}) or {})
+        existing_entry.recommended_next_step = dict(summary.get("failure_analysis", {}) or {}) or None
+
+        document.updated_at = existing_entry.last_used_at
+        self.save_document(document)
+        return existing_entry.model_dump()

common/error_codes.py

@@ -0,0 +1,61 @@
+"""Single source of truth for agent-facing error codes.
+
+Both the stderr log (`log.error(format_log("E037"))`) and the `--action --json`
+failure payload read message + fix from this one table, so the two channels can
+never drift. Scope is deliberately narrow: only the locate/action codes an agent
+hits on the `--action` path. Connection codes (E04x/E05x) and `--goal`-only codes
+(E02x stagnation / circuit-breaker / max-steps) are intentionally NOT here — this
+iteration does not touch those paths.
+"""
+
+# code -> (message, fix)
+ERROR_CODES: dict[str, tuple[str, str]] = {
+    "E030": (
+        "Ref not found in cache.",
+        "Run inspect_ui first to refresh the element cache.",
+    ),
+    "E031": (
+        "Unsupported action type.",
+        "See `screenforge --capabilities` for the supported action list.",
+    ),
+    "E032": (
+        "Element action missing locator_type.",
+        "Provide locator_type (css/text/resourceId/description).",
+    ),
+    "E033": (
+        "Element locator is empty after resolution.",
+        "Verify the target exists on the current page via inspect_ui.",
+    ),
+    "E035": (
+        "AI returned empty action type.",
+        "Check that MODEL_NAME supports structured JSON output.",
+    ),
+    "E036": (
+        "Ref has no stable locator (only coordinates).",
+        "Re-inspect; use a text/css locator instead of a coordinate-only ref.",
+    ),
+    "E037": (
+        "Element could not be located for the action.",
+        "Re-inspect, scroll the target into view, or add --vision.",
+    ),
+    "E038": (
+        "Element located but the action failed or was blocked.",
+        "Check for overlays; ensure the element is enabled and in the viewport.",
+    ),
+}
+
+_GENERIC = (
+    "Action failed.",
+    "Re-inspect via inspect_ui and adjust strategy.",
+)
+
+
+def lookup(code: str) -> tuple[str, str]:
+    """Return (message, fix) for a code; unknown codes get a generic, non-raising fallback."""
+    return ERROR_CODES.get(code, _GENERIC)
+
+
+def format_log(code: str) -> str:
+    """Format a code for stderr: '[E037] <message> Fix: <fix>'."""
+    msg, fix = lookup(code)
+    return f"[{code}] {msg} Fix: {fix}"

common/exceptions.py

@@ -0,0 +1,18 @@
+class UIAgentError(Exception):
+    pass
+
+
+class AdapterError(UIAgentError):
+    pass
+
+
+class AIError(UIAgentError):
+    pass
+
+
+class CacheError(UIAgentError):
+    pass
+
+
+class ExecutorError(UIAgentError):
+    pass