macos-computer-use-kit 0.2.0__py3-none-any.whl

macos_computer_use/__init__.py

@@ -0,0 +1,9 @@
+"""AX-first computer-use toolkit for AI agents on macOS.
+
+The public surface is the ``macos-cu`` command line; every subcommand prints a
+single JSON object (or JSON lines) so an agent can consume the result directly.
+"""
+
+__version__ = "0.2.0"
+
+__all__ = ["__version__"]

macos_computer_use/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

macos_computer_use/ax.py

@@ -0,0 +1,363 @@
+"""Accessibility (AX) tree reading, semantic targeting, and native AX actions.
+
+Inspired by Codex's CUA repl: instead of screenshot -> vision -> estimate
+coordinates, read the macOS accessibility tree and get exact element geometry.
+`press`/`setvalue` additionally verify that the action changed observable state
+before reporting success.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import time
+from typing import Any
+
+from . import darwin
+
+
+def _ax():
+    AS, _NSWorkspace, _Quartz = darwin._pyobjc()  # noqa: N806
+    return AS
+
+
+def attr(el, name):
+    AS = _ax()  # noqa: N806
+    err, value = AS.AXUIElementCopyAttributeValue(el, name, None)
+    if err != 0:
+        return None
+    return value
+
+
+def point_of(el):
+    AS = _ax()  # noqa: N806
+    value = attr(el, AS.kAXPositionAttribute)
+    if value is None:
+        return None
+    ok, pt = AS.AXValueGetValue(value, AS.kAXValueCGPointType, None)
+    if not ok:
+        return None
+    return (int(pt.x), int(pt.y))
+
+
+def size_of(el):
+    AS = _ax()  # noqa: N806
+    value = attr(el, AS.kAXSizeAttribute)
+    if value is None:
+        return None
+    ok, sz = AS.AXValueGetValue(value, AS.kAXValueCGSizeType, None)
+    if not ok:
+        return None
+    return (int(sz.width), int(sz.height))
+
+
+def walk(el, depth, max_depth, out, role_filter=None, path="0"):
+    AS = _ax()  # noqa: N806
+    if len(out) >= 4000 or depth > max_depth:
+        return
+    role = attr(el, AS.kAXRoleAttribute) or ""
+    title = attr(el, AS.kAXTitleAttribute) or ""
+    desc = attr(el, AS.kAXDescriptionAttribute) or ""
+    value = attr(el, AS.kAXValueAttribute)
+    value = value.replace("\n", " ")[:60] if isinstance(value, str) else None
+    pos = point_of(el)
+    size = size_of(el)
+    children = attr(el, AS.kAXChildrenAttribute) or []
+    matches = (not role_filter) or (role_filter.lower() in role.lower())
+    if matches and (title or desc or value or (size and size[0] > 4 and size[1] > 4)):
+        out.append(
+            {
+                "id": path,
+                "role": role,
+                "title": title,
+                "desc": desc,
+                "value": value,
+                "pos": pos,
+                "size": size,
+                "depth": depth,
+                "children": len(children),
+            }
+        )
+    for i, child in enumerate(children[:200]):
+        walk(child, depth + 1, max_depth, out, role_filter, f"{path}.{i}")
+
+
+def element_at_path(el, path):
+    """Resolve a snapshot-style path id (e.g. '0.1.0.6.0') to a live AXUIElement."""
+    AS = _ax()  # noqa: N806
+    parts = str(path).split(".")
+    if not parts or parts[0] != "0":
+        return None
+    cur = el
+    for p in parts[1:]:
+        children = attr(cur, AS.kAXChildrenAttribute) or []
+        try:
+            i = int(p)
+        except ValueError:
+            return None
+        if i >= len(children):
+            return None
+        cur = children[i]
+    return cur
+
+
+def text_signature(root, limit=80):
+    """Hash of the visible text in a window: catches pane/content changes."""
+    AS = _ax()  # noqa: N806
+    texts: list[str] = []
+
+    def collect(el, depth=0):
+        if depth > 12 or len(texts) >= limit:
+            return
+        role = attr(el, AS.kAXRoleAttribute)
+        if role in ("AXStaticText", "AXButton", "AXTextField", "AXTextArea"):
+            v = attr(el, AS.kAXValueAttribute) or attr(el, AS.kAXTitleAttribute)
+            if v:
+                texts.append(str(v)[:40])
+        for c in (attr(el, AS.kAXChildrenAttribute) or [])[:80]:
+            collect(c, depth + 1)
+
+    collect(root)
+    return hashlib.md5("\n".join(texts).encode()).hexdigest()[:12], len(texts)
+
+
+def app_fingerprint(root) -> dict[str, Any]:
+    """Small observable state used to verify that an AX action had an effect."""
+    AS = _ax()  # noqa: N806
+    f: dict[str, Any] = {}
+    focused = attr(root, AS.kAXFocusedUIElementAttribute)
+    if focused is not None:
+        f["focused_role"] = attr(focused, AS.kAXRoleAttribute)
+        f["focused_value"] = str(attr(focused, AS.kAXValueAttribute) or "")[:80]
+    windows = attr(root, AS.kAXWindowsAttribute) or []
+    if windows:
+        w = windows[0]
+        f["window_title"] = attr(w, AS.kAXTitleAttribute)
+        kids = attr(w, AS.kAXChildrenAttribute) or []
+        if kids:
+            k = kids[0]
+            f["pane_role"] = attr(k, AS.kAXRoleAttribute)
+            f["pane_children"] = len(attr(k, AS.kAXChildrenAttribute) or [])
+        sig, n = text_signature(w)
+        f["text_sig"] = sig
+        f["text_count"] = n
+    return f
+
+
+def locate(root, args):
+    """Return (element, entry) for --id or role/title filters."""
+    AS = _ax()  # noqa: N806
+    if getattr(args, "id", None):
+        el = element_at_path(root, args.id)
+        if el is None:
+            return None, None
+        entry = {
+            "id": args.id,
+            "role": attr(el, AS.kAXRoleAttribute),
+            "title": attr(el, AS.kAXTitleAttribute),
+            "value": attr(el, AS.kAXValueAttribute),
+        }
+        return el, entry
+    found: list[dict[str, Any]] = []
+    walk(root, 0, args.depth, found, args.role)
+    if args.title:
+        needle = args.title.lower()
+        found = [
+            e
+            for e in found
+            if needle in (e["title"] or "").lower()
+            or needle in (e["desc"] or "").lower()
+            or needle in (e["value"] or "").lower()
+        ]
+    if not found:
+        return None, None
+    entry = found[0]
+    return element_at_path(root, entry["id"]), entry
+
+
+def run_action(root, args) -> dict[str, Any]:
+    AS = _ax()  # noqa: N806
+    el, entry = locate(root, args)
+    if el is None:
+        return {"ok": False, "reason": "element_not_found"}
+
+    coords = None
+    pos = point_of(el)
+    size = size_of(el)
+    if pos and size:
+        coords = [pos[0] + size[0] // 2, pos[1] + size[1] // 2]
+
+    before = app_fingerprint(root)
+
+    if args.cmd == "press":
+        err = AS.AXUIElementPerformAction(el, AS.kAXPressAction)
+        time.sleep(0.4)
+        after = app_fingerprint(root)
+        changed = before != after
+        result = {
+            "ok": True,
+            "action": "ax_press",
+            "err": int(err),
+            "verified": bool(err == 0 and changed),
+            "state_changed": changed,
+            "before": before,
+            "after": after,
+            "coords": coords,
+            "element": {"id": entry.get("id"), "role": entry.get("role"), "title": entry.get("title")},
+        }
+        if not result["verified"]:
+            result["hint"] = (
+                "AXPress did not change observable state; the element may be custom-drawn. "
+                "Fall back to a coordinate click (macos-cu input click) after re-reading geometry."
+            )
+        return result
+
+    err = AS.AXUIElementSetAttributeValue(el, AS.kAXValueAttribute, args.text)
+    time.sleep(0.2)
+    readback = attr(el, AS.kAXValueAttribute)
+    result = {
+        "ok": True,
+        "action": "ax_set_value",
+        "err": int(err),
+        "verified": bool(err == 0 and str(readback) == args.text),
+        "readback": str(readback)[:80] if readback is not None else None,
+        "coords": coords,
+        "element": {"id": entry.get("id"), "role": entry.get("role"), "title": entry.get("title")},
+    }
+    if not result["verified"]:
+        result["hint"] = (
+            "AXValue is not settable here; use `macos-cu paste` (clipboard paste + verify) instead."
+        )
+    return result
+
+
+def snapshot_cache_dir() -> str:
+    base = os.environ.get("MACOS_CU_CACHE_DIR") or os.path.join(
+        os.path.expanduser("~"), ".cache", "macos-computer-use"
+    )
+    path = os.path.join(base, "snapshots")
+    os.makedirs(path, exist_ok=True)
+    return path
+
+
+def app_root(args):
+    """Resolve --pid/--app to an AX application element."""
+    AS = _ax()  # noqa: N806
+    if getattr(args, "pid", None):
+        return AS.AXUIElementCreateApplication(int(args.pid)), int(args.pid)
+    app = darwin.find_app(args.app)
+    if app is None:
+        return None, None
+    return AS.AXUIElementCreateApplication(app.processIdentifier()), app
+
+
+def run(args) -> int:
+    out: list[dict[str, Any]] = []
+
+    if args.cmd == "resolve":
+        if not args.file or not args.id:
+            print(json.dumps({"error": "resolve needs --file and --id"}), file=sys.stderr)
+            return 2
+        with open(args.file) as fh:
+            data = json.load(fh)
+        for e in data["elements"]:
+            if e["id"] == args.id:
+                print(json.dumps(e, ensure_ascii=False))
+                return 0
+        print(json.dumps({"error": "id_not_found", "id": args.id}), file=sys.stderr)
+        return 3
+
+    if not darwin.permissions()["accessibility"]:
+        print(json.dumps({"error": "accessibility_not_granted", "hint": darwin.permission_hint("accessibility")}), file=sys.stderr)
+        return 2
+
+    if args.cmd in ("press", "setvalue") and not (args.pid or args.app):
+        print(json.dumps({"error": "press/setvalue need --app or --pid"}), file=sys.stderr)
+        return 2
+    if args.cmd == "setvalue" and args.text is None:
+        print(json.dumps({"error": "setvalue needs --text"}), file=sys.stderr)
+        return 2
+
+    root, app = app_root(args)
+    if root is None:
+        print(json.dumps({"error": "app_not_found", "app": args.app}), file=sys.stderr)
+        return 2
+
+    if args.cmd in ("press", "setvalue"):
+        print(json.dumps(run_action(root, args), ensure_ascii=False))
+        return 0
+
+    walk(root, 0, args.depth, out, args.role)
+
+    if args.title:
+        needle = args.title.lower()
+        out = [
+            e
+            for e in out
+            if needle in (e["title"] or "").lower()
+            or needle in (e["desc"] or "").lower()
+            or needle in (e["value"] or "").lower()
+        ]
+
+    if args.cmd == "click-info" and args.index is not None:
+        out = [out[args.index]] if 0 <= args.index < len(out) else []
+    elif args.cmd in ("find", "tree", "click-info"):
+        # Cap results for these modes; `snapshot` is bounded by --budget instead.
+        out = out[: args.max]
+
+    app_name = (
+        app.localizedName()
+        if hasattr(app, "localizedName")
+        else darwin.find_app(args.app).localizedName() if args.app else f"pid:{args.pid}"
+    )
+
+    result = []
+    for i, e in enumerate(out):
+        entry = dict(e)
+        entry["idx"] = i
+        if e["pos"] and e["size"]:
+            cx = e["pos"][0] + e["size"][0] // 2
+            cy = e["pos"][1] + e["size"][1] // 2
+            entry["center_screen"] = [cx, cy]
+            # `shot` space only exists for harnesses whose screenshots are
+            # scaled differently from screen points; opt in with --shot-scale.
+            if args.shot_scale:
+                entry["center_shot"] = [round(cx * args.shot_scale), round(cy * args.shot_scale)]
+        result.append(entry)
+
+    if args.json:
+        print(json.dumps({"app": app_name, "count": len(result), "elements": result}, ensure_ascii=False))
+        return 0
+
+    if args.cmd == "snapshot":
+        lines = []
+        used = 0
+        omitted = 0
+        for e in result:
+            t = (e["title"] or e["desc"] or e["value"] or "")[:60]
+            loc = f"@screen{e['center_screen']}" if e.get("center_screen") else ""
+            sz = f"{e['size'][0]}x{e['size'][1]}" if e["size"] else "-"
+            line = f"[{e['id']}] {e['role']:<22} {sz:>10} {loc:<22} {t}"
+            if used + len(line) + 1 > args.budget:
+                omitted += 1
+                continue
+            lines.append(line)
+            used += len(line) + 1
+        cache = os.path.join(snapshot_cache_dir(), f"{app_name}-{int(time.time())}.json")
+        with open(cache, "w") as fh:
+            json.dump({"app": app_name, "elements": result}, fh, ensure_ascii=False)
+        print(f"# snapshot: {cache} | elements={len(result)} shown={len(lines)} omitted={omitted} (budget={args.budget})")
+        print("\n".join(lines))
+        return 0
+
+    print(f"app={app_name} elements={len(result)}")
+    for e in result:
+        t = (e["title"] or e["desc"] or e["value"] or "")[:50]
+        loc = f"@screen{e['center_screen']}" if e.get("center_screen") else ""
+        if e.get("center_shot"):
+            loc += f" shot{e['center_shot']}"
+        sz = f"{e['size'][0]}x{e['size'][1]}" if e["size"] else "-"
+        print(f"[{e['idx']:>3}] {e['role']:<24} {sz:>10} {loc:<34} {t}")
+    return 0

macos_computer_use/cli.py

@@ -0,0 +1,204 @@
+"""Command line interface: ``macos-cu <group> <command> [options]``.
+
+Every command prints JSON so an agent can consume it directly. Exit codes are
+stable and meaningful:
+
+- 0 success
+- 2 usage error, unsupported platform, or missing permission
+- 3 not found / capture failed / upstream HTTP error
+- 4 target app not found (paste) or capture failed
+- 5 target_changed (window signature mismatch) or paste conflict
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any
+
+from . import __version__, darwin
+
+EXIT_USAGE = 2
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--json", action="store_true", help="emit structured JSON (where the command has a text mode)")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(
+        prog="macos-cu",
+        description="AX-first computer-use toolkit for macOS agents.",
+        epilog="Run `macos-cu doctor` first to check permissions and display geometry.",
+    )
+    ap.add_argument("--version", action="version", version=f"macos-computer-use-kit {__version__}")
+    groups = ap.add_subparsers(dest="group", required=True)
+
+    # ---------------------------------------------------------------- ax
+    ax = groups.add_parser("ax", help="read the accessibility tree and run native AX actions")
+    _add_common(ax)
+    ax.add_argument("cmd", choices=["tree", "find", "click-info", "snapshot", "resolve", "press", "setvalue"])
+    ax.add_argument("--app", default=None, help="app name or bundle id (substring match)")
+    ax.add_argument("--pid", type=int, default=None, help="target a specific process instead of --app")
+    ax.add_argument("--depth", type=int, default=16)
+    ax.add_argument("--max", type=int, default=120, help="cap rows printed (tree/find)")
+    ax.add_argument("--role", default=None, help="AX role filter, e.g. AXButton")
+    ax.add_argument("--title", default=None, help="substring match on title/description/value")
+    ax.add_argument("--index", type=int, default=None, help="element index for click-info")
+    ax.add_argument(
+        "--shot-scale",
+        type=float,
+        default=None,
+        help="also emit center_shot = round(center_screen * SCALE). Only needed when your harness's "
+        "screenshots use a different scale than screen points; there is no safe default.",
+    )
+    ax.add_argument("--budget", type=int, default=4000, help="max characters of snapshot text")
+    ax.add_argument("--file", default=None, help="snapshot cache file (for resolve)")
+    ax.add_argument("--id", default=None, help="element id inside a snapshot (for resolve)")
+    ax.add_argument("--text", default=None, help="text to write (for setvalue)")
+
+    # ------------------------------------------------------------- input
+    inp = groups.add_parser("input", help="process/window-scoped input (the physical cursor never moves)")
+    inp.add_argument("cmd", choices=["windows", "cursor", "pid", "click", "key", "scroll", "move"])
+    inp.add_argument("--app")
+    inp.add_argument("--pid", type=int)
+    inp.add_argument("--window-id", type=int, help="target a window; --x/--y become window-relative")
+    inp.add_argument("--expect", help="expected signature pid:wid:x:y:w:h; mismatch refuses with target_changed")
+    inp.add_argument("--show", action="store_true", help="draw a visual ring at the action point")
+    inp.add_argument("--x", type=int)
+    inp.add_argument("--y", type=int)
+    inp.add_argument("--button", default="left", choices=["left", "right", "middle"])
+    inp.add_argument("--count", type=int, default=1)
+    inp.add_argument("--key")
+    inp.add_argument("--flags", default="", help="modifier flags, e.g. cmd+shift")
+    inp.add_argument("--amount", type=int, default=-5, help="scroll lines: negative = up, positive = down")
+
+    # ------------------------------------------------------------- paste
+    paste = groups.add_parser("paste", help="clipboard-safe paste that restores the user's clipboard")
+    paste.add_argument("--app")
+    paste.add_argument("--pid", type=int)
+    paste.add_argument("--text", required=True)
+    paste.add_argument("--mode", choices=["pid", "hid"], default="pid",
+                       help="pid = post to the app (cursor untouched); hid = system-wide")
+    paste.add_argument("--wait", type=float, default=1.5, help="seconds to wait before checking consumption")
+    paste.add_argument("--keep", action="store_true", help="leave our text on the clipboard")
+
+    # -------------------------------------------------------------- shot
+    shot = groups.add_parser("shot", help="screenshots with blank-frame detection")
+    shot.add_argument("cmd", choices=["capture", "check", "windows"])
+    shot.add_argument("--file")
+    shot.add_argument("--out")
+    shot.add_argument("--window-id", type=int)
+    shot.add_argument("--app")
+    shot.add_argument("--region", help="x,y,w,h")
+
+    # ----------------------------------------------------------- overlay
+    overlay = groups.add_parser("overlay", help="transient visual feedback ring")
+    overlay.add_argument("cmd", choices=["show", "clear"])
+    overlay.add_argument("--x", type=int)
+    overlay.add_argument("--y", type=int)
+    overlay.add_argument("--label", default="")
+    overlay.add_argument("--duration", type=float, default=1.5)
+    overlay.add_argument("--color", default="cyan", choices=["cyan", "green", "orange", "red"])
+
+    # --------------------------------------------------------------- jev
+    jev = groups.add_parser("jev", help="optional TypeSafe System One semantic guards (JSON on stdin)")
+    jev.add_argument("cmd", choices=["guard", "select"])
+
+    # ------------------------------------------------------------ doctor
+    groups.add_parser("doctor", help="diagnose permissions, displays, dependencies, and Jev setup")
+
+    return ap
+
+
+def _pyobjc_version() -> str | None:
+    try:
+        import objc
+
+        return getattr(objc, "__version__", None)
+    except Exception:
+        return None
+
+
+def doctor() -> int:
+    perms = {"accessibility": False, "screen_recording": False}
+    perm_error = None
+    try:
+        perms = darwin.permissions()
+    except SystemExit as exc:  # dependency missing
+        perm_error = str(exc.code)
+
+    info: dict[str, Any] = {
+        "version": __version__,
+        "platform": {"system": sys.platform, "macos": darwin.macos_version(), "arch": os.uname().machine},
+        "python": sys.version.split()[0],
+        "pyobjc": _pyobjc_version(),
+        "permissions": perms,
+        "displays": [],
+        "jev": {"key_present": bool(darwin.jev_key()), "model": os.environ.get("TYPESAFE_MODEL", "jev-latest")},
+        "hints": [],
+    }
+    try:
+        info["displays"] = darwin.displays()
+    except Exception as exc:  # pragma: no cover - defensive
+        info["displays_error"] = str(exc)[:200]
+
+    if perm_error:
+        info["hints"].append("Install dependencies: pip install 'macos-computer-use-kit'")
+    else:
+        if not perms.get("accessibility"):
+            info["hints"].append(darwin.permission_hint("accessibility"))
+        if not perms.get("screen_recording"):
+            info["hints"].append(darwin.permission_hint("screen_recording"))
+    if not info["jev"]["key_present"]:
+        info["hints"].append(
+            "Jev is optional. To enable semantic guards, set TYPESAFE_API_KEY or write "
+            "~/.config/typesafe/api_key (https://console.typesafe.ai/keys)."
+        )
+    if not info["hints"]:
+        info["hints"].append("All checks passed. Try: macos-cu ax tree --app Finder --max 20")
+
+    print(json.dumps(info, ensure_ascii=False, indent=2))
+    return 0 if (perms.get("accessibility") and perms.get("screen_recording")) else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.group == "doctor":
+        return doctor()
+
+    darwin.require_macos()
+
+    if args.group == "ax":
+        from . import ax
+
+        return ax.run(args)
+    if args.group == "input":
+        from . import input_events
+
+        return input_events.run(args)
+    if args.group == "paste":
+        from . import paste
+
+        return paste.run(args)
+    if args.group == "shot":
+        from . import shot
+
+        return shot.run(args)
+    if args.group == "overlay":
+        from . import overlay
+
+        return overlay.run(args)
+    if args.group == "jev":
+        from . import jev
+
+        return jev.run(args)
+
+    return EXIT_USAGE
+
+
+if __name__ == "__main__":
+    sys.exit(main())