macos-computer-use-kit 0.2.0__py3-none-any.whl

macos_computer_use/darwin.py

@@ -0,0 +1,261 @@
+"""macOS platform layer: guards, permissions, app/window resolution, displays.
+
+Everything macOS-specific that more than one command needs lives here, so the
+other modules stay portable and testable.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import subprocess
+import sys
+from typing import Any
+
+
+def require_macos() -> None:
+    """Exit with an actionable message when not running on macOS."""
+    if sys.platform != "darwin":
+        print(
+            json.dumps(
+                {
+                    "error": "unsupported_platform",
+                    "platform": sys.platform,
+                    "detail": "this toolkit drives macOS Accessibility and CoreGraphics APIs",
+                }
+            ),
+            file=sys.stderr,
+        )
+        raise SystemExit(2)
+
+
+def _pyobjc():
+    try:
+        import ApplicationServices as AS  # noqa: N813
+        from AppKit import NSWorkspace
+        import Quartz
+    except ImportError as exc:  # pragma: no cover - depends on host setup
+        print(
+            json.dumps(
+                {
+                    "error": "missing_dependency",
+                    "detail": f"{exc}. Install with: pip install 'macos-computer-use-kit' "
+                    "(or: pip install pyobjc-framework-Quartz pyobjc-framework-Cocoa "
+                    "pyobjc-framework-ApplicationServices)",
+                }
+            ),
+            file=sys.stderr,
+        )
+        raise SystemExit(2) from exc
+    return AS, NSWorkspace, Quartz
+
+
+def permissions() -> dict[str, bool]:
+    """Report the two TCC permissions this toolkit needs."""
+    AS, _NSWorkspace, Quartz = _pyobjc()  # noqa: N806
+    accessibility = bool(AS.AXIsProcessTrusted())
+    try:
+        screen_recording = bool(Quartz.CGPreflightScreenCaptureAccess())
+    except AttributeError:  # pragma: no cover - older SDKs
+        screen_recording = bool(Quartz.CGWindowListCopyWindowInfo(
+            Quartz.kCGWindowListOptionOnScreenOnly, Quartz.kCGNullWindowID
+        ))
+    return {"accessibility": accessibility, "screen_recording": screen_recording}
+
+
+def permission_hint(kind: str) -> str:
+    app = _frontmost_owner_name()
+    return (
+        f"{kind} permission is not granted. Open System Settings -> Privacy & Security -> "
+        f"{'Accessibility' if kind == 'accessibility' else 'Screen Recording'} and enable "
+        f"the process running this command{app}. Then rerun `macos-cu doctor`."
+    )
+
+
+def _frontmost_owner_name() -> str:
+    """Best-effort hint at which app should be granted permission."""
+    for candidate in (
+        os.environ.get("TERM_PROGRAM"),
+        os.environ.get("__CFBundleIdentifier"),
+        None,
+    ):
+        if candidate:
+            return f" ({candidate})"
+    return ""
+
+
+def displays() -> list[dict[str, Any]]:
+    """Screen geometry in points, as AX reports it.
+
+    Note: AX/CoreGraphics use a top-left origin space whose (0,0) is the primary
+    display. Secondary displays placed left of or above the primary produce
+    negative coordinates, which is normal.
+    """
+    from AppKit import NSScreen
+
+    out = []
+    for i, screen in enumerate(NSScreen.screens()):
+        frame = screen.frame()
+        out.append(
+            {
+                "index": i,
+                "primary": i == 0,
+                "name": str(screen.localizedName()),
+                "points": [int(frame.size.width), int(frame.size.height)],
+                "backing_scale": int(screen.backingScaleFactor()),
+                "origin": [int(frame.origin.x), int(frame.origin.y)],
+            }
+        )
+    return out
+
+
+def primary_screen_point_size() -> tuple[int, int]:
+    """Point size of the primary display (the AX coordinate space origin)."""
+    from AppKit import NSScreen
+
+    frame = NSScreen.screens()[0].frame()
+    return int(frame.size.width), int(frame.size.height)
+
+
+def find_app(name: str | None):
+    """Resolve an NSRunningApplication by localized name or bundle id.
+
+    Match order: exact bundle id, exact localized name, case-insensitive exact,
+    then substring. First match in each tier wins, preferring frontmost apps.
+    """
+    _AS, NSWorkspace, _Quartz = _pyobjc()  # noqa: N806
+    workspace = NSWorkspace.sharedWorkspace()
+    if not name:
+        return workspace.frontmostApplication()
+
+    apps = list(workspace.runningApplications())
+    frontmost = workspace.frontmostApplication()
+    if frontmost is not None:
+        apps.sort(key=lambda a: 0 if a.processIdentifier() == frontmost.processIdentifier() else 1)
+
+    needle = name.lower()
+
+    def field(app, getter):
+        try:
+            return (getter(app) or "")
+        except Exception:  # pragma: no cover - defensive
+            return ""
+
+    for tier in (
+        lambda a: field(a, lambda x: x.bundleIdentifier()) == name,
+        lambda a: field(a, lambda x: x.localizedName()) == name,
+        lambda a: field(a, lambda x: x.bundleIdentifier()).lower() == needle,
+        lambda a: field(a, lambda x: x.localizedName()).lower() == needle,
+        lambda a: needle in field(a, lambda x: x.bundleIdentifier()).lower(),
+        lambda a: needle in field(a, lambda x: x.localizedName()).lower(),
+    ):
+        for app in apps:
+            if tier(app):
+                return app
+    return None
+
+
+def resolve_pid(app_name: str | None = None, pid: int | None = None) -> int | None:
+    if pid:
+        return int(pid)
+    app = find_app(app_name)
+    return int(app.processIdentifier()) if app is not None else None
+
+
+def all_windows(owner_filter: str | None = None) -> list[dict[str, Any]]:
+    """On-screen windows, main windows first."""
+    _AS, _NSWorkspace, Quartz = _pyobjc()  # noqa: N806
+    out = []
+    for w in Quartz.CGWindowListCopyWindowInfo(
+        Quartz.kCGWindowListOptionOnScreenOnly, Quartz.kCGNullWindowID
+    ):
+        owner = str(w.get("kCGWindowOwnerName") or "")
+        if owner_filter and owner_filter.lower() not in owner.lower():
+            continue
+        b = w.get("kCGWindowBounds") or {}
+        width, height = int(b.get("Width", 0)), int(b.get("Height", 0))
+        if width < 50 or height < 50:
+            continue
+        out.append(
+            {
+                "id": int(w.get("kCGWindowNumber", 0)),
+                "pid": int(w.get("kCGWindowOwnerPID", 0)),
+                "owner": owner,
+                "title": str(w.get("kCGWindowName") or ""),
+                "bounds": [int(b.get("X", 0)), int(b.get("Y", 0)), width, height],
+                "layer": int(w.get("kCGWindowLayer", 0)),
+            }
+        )
+    out.sort(key=lambda x: (x["layer"] != 0, -(x["bounds"][2] * x["bounds"][3])))
+    return out
+
+
+def window_info(window_id: int) -> dict[str, Any] | None:
+    for w in all_windows():
+        if w["id"] == int(window_id):
+            return w
+    return None
+
+
+def target_sig(win: dict[str, Any]) -> str:
+    """Stable window identity signature: pid:wid:x:y:w:h."""
+    x, y, w, h = win["bounds"]
+    return f'{win["pid"]}:{win["id"]}:{x}:{y}:{w}:{h}'
+
+
+def overlay_argv(x: float, y: float, label: str = "", duration: float = 1.2, color: str = "cyan") -> list[str]:
+    """Command to draw an overlay ring, for the current install method."""
+    return [
+        sys.executable,
+        "-m",
+        "macos_computer_use.cli",
+        "overlay",
+        "show",
+        "--x",
+        str(int(x)),
+        "--y",
+        str(int(y)),
+        "--label",
+        label or "",
+        "--duration",
+        str(duration),
+        "--color",
+        color,
+    ]
+
+
+def show_overlay(x: float, y: float, label: str = "", color: str = "cyan", duration: float = 1.2) -> bool:
+    """Fire-and-forget visual ring at an AX/global screen point."""
+    try:
+        subprocess.Popen(
+            overlay_argv(x, y, label, duration, color),
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            start_new_session=True,
+        )
+        return True
+    except Exception:  # pragma: no cover - best-effort affordance
+        return False
+
+
+def macos_version() -> str:
+    try:
+        return platform.mac_ver()[0] or "unknown"
+    except Exception:  # pragma: no cover - defensive
+        return "unknown"
+
+
+def jev_key() -> str | None:
+    """TypeSafe/Jev API key from the environment or the conventional files."""
+    key = os.environ.get("TYPESAFE_API_KEY")
+    if key:
+        return key.strip()
+    for path in ("~/.config/typesafe/api_key", "~/.typesafe/api_key"):
+        p = os.path.expanduser(path)
+        if os.path.exists(p):
+            with open(p) as fh:
+                value = fh.read().strip()
+            if value:
+                return value
+    return None

macos_computer_use/input_events.py

@@ -0,0 +1,172 @@
+"""Process- and window-scoped input for macOS.
+
+Delivers mouse/keyboard events straight to a target process's event queue with
+``CGEventPostToPid``, so the system cursor never moves and the user's physical
+mouse is untouched. Window-scoped actions take window-relative coordinates and
+can validate a target signature before acting, which turns "the window moved
+while I was aiming" into an explicit ``target_changed`` refusal instead of a
+misclick.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+from . import darwin
+
+
+def _q():
+    _AS, _NSWorkspace, Quartz = darwin._pyobjc()  # noqa: N806
+    return Quartz
+
+
+BUTTONS = {
+    "left": ("kCGEventLeftMouseDown", "kCGEventLeftMouseUp", "kCGMouseButtonLeft"),
+    "right": ("kCGEventRightMouseDown", "kCGEventRightMouseUp", "kCGMouseButtonRight"),
+    "middle": ("kCGEventOtherMouseDown", "kCGEventOtherMouseUp", "kCGMouseButtonCenter"),
+}
+
+KEYS = {
+    "return": 36, "enter": 36, "tab": 48, "space": 49, "delete": 51, "escape": 53, "esc": 53,
+    "left": 123, "right": 124, "down": 125, "up": 126, "home": 115, "end": 119,
+    "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
+    "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
+    "3": 20, "4": 21, "6": 22, "5": 23, "9": 25, "7": 26, "8": 28, "0": 29,
+    "o": 31, "u": 32, "i": 34, "p": 35, "l": 37, "j": 38, "k": 40, "n": 45, "m": 46,
+}
+
+FLAGS = {
+    "cmd": "kCGEventFlagMaskCommand",
+    "command": "kCGEventFlagMaskCommand",
+    "shift": "kCGEventFlagMaskShift",
+    "ctrl": "kCGEventFlagMaskControl",
+    "control": "kCGEventFlagMaskControl",
+    "alt": "kCGEventFlagMaskAlternate",
+    "option": "kCGEventFlagMaskAlternate",
+}
+
+
+def do_click(pid, x, y, button, count):
+    Quartz = _q()  # noqa: N806
+    down, up, btn = (getattr(Quartz, name) for name in BUTTONS[button])
+    for i in range(count):
+        for kind in (down, up):
+            ev = Quartz.CGEventCreateMouseEvent(None, kind, (x, y), btn)
+            Quartz.CGEventSetIntegerValueField(ev, Quartz.kCGMouseEventClickState, i + 1)
+            Quartz.CGEventPostToPid(pid, ev)
+        time.sleep(0.06)
+    return f"posted {count}x {button} click at ({x},{y}) to pid {pid}"
+
+
+def do_scroll(pid, x, y, amount):
+    Quartz = _q()  # noqa: N806
+    ev = Quartz.CGEventCreateScrollWheelEvent(None, Quartz.kCGScrollEventUnitLine, 1, amount)
+    Quartz.CGEventSetLocation(ev, (x, y))
+    Quartz.CGEventPostToPid(pid, ev)
+    return f"posted scroll {amount} at ({x},{y}) to pid {pid}"
+
+
+def do_move(pid, x, y):
+    Quartz = _q()  # noqa: N806
+    ev = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventMouseMoved, (x, y), Quartz.kCGMouseButtonLeft)
+    Quartz.CGEventPostToPid(pid, ev)
+    return f"posted move to ({x},{y}) to pid {pid}"
+
+
+def do_key(pid, key, flags):
+    Quartz = _q()  # noqa: N806
+    code = KEYS.get(key, None)
+    if code is None:
+        code = int(key)
+    flagmask = 0
+    for f in (flags.split("+") if flags else []):
+        flagmask |= getattr(Quartz, FLAGS[f])
+    for is_down in (True, False):
+        ev = Quartz.CGEventCreateKeyboardEvent(None, code, is_down)
+        if flagmask:
+            Quartz.CGEventSetFlags(ev, flagmask)
+        Quartz.CGEventPostToPid(pid, ev)
+        time.sleep(0.02)
+    return f"posted key {key} (code {code}) flags={flagmask} to pid {pid}"
+
+
+def cursor_position() -> tuple[int, int]:
+    Quartz = _q()  # noqa: N806
+    loc = Quartz.CGEventGetLocation(Quartz.CGEventCreate(None))
+    return int(loc.x), int(loc.y)
+
+
+def run(args) -> int:
+    if args.cmd == "cursor":
+        x, y = cursor_position()
+        print(json.dumps({"cursor": [x, y]}))
+        return 0
+
+    if not darwin.permissions()["accessibility"]:
+        print(json.dumps({"error": "accessibility_not_granted", "hint": darwin.permission_hint("accessibility")}), file=sys.stderr)
+        return 2
+
+    if args.cmd == "windows":
+        windows = darwin.all_windows(args.app)
+        if args.pid:
+            windows = [w for w in windows if w["pid"] == int(args.pid)]
+        print(json.dumps({"windows": windows}, ensure_ascii=False))
+        return 0
+
+    win = None
+    if args.window_id:
+        win = darwin.window_info(args.window_id)
+        if win is None:
+            print(json.dumps({"ok": False, "reason": "target_changed", "detail": "window not found"}, ensure_ascii=False))
+            return 5
+        if args.expect and args.expect != darwin.target_sig(win):
+            print(json.dumps(
+                {"ok": False, "reason": "target_changed", "expected": args.expect, "actual": darwin.target_sig(win)},
+                ensure_ascii=False,
+            ))
+            return 5
+        pid = win["pid"]
+        if args.x is not None and args.y is not None:
+            args.x = win["bounds"][0] + args.x
+            args.y = win["bounds"][1] + args.y
+    else:
+        pid = darwin.resolve_pid(args.app, args.pid)
+        if pid is None:
+            print(json.dumps({"ok": False, "reason": "app_not_found", "app": args.app}, ensure_ascii=False))
+            return 2
+
+    if args.cmd == "pid":
+        print(json.dumps({"pid": pid}))
+        return 0
+
+    if args.cmd in ("click", "move", "scroll") and (args.x is None or args.y is None):
+        print(json.dumps({"error": "--x/--y required for this command"}), file=sys.stderr)
+        return 2
+    if args.cmd == "key" and not args.key:
+        print(json.dumps({"error": "--key required"}), file=sys.stderr)
+        return 2
+
+    sig = darwin.target_sig(win) if win else None
+    if args.show and args.x is not None:
+        darwin.show_overlay(args.x, args.y, args.key or args.cmd)
+
+    result: dict[str, Any] = {
+        "ok": True,
+        "pid": pid,
+        "target": sig,
+        "global": [args.x, args.y] if args.x is not None else None,
+        "action_sent": True,
+    }
+    if args.cmd == "click":
+        result["detail"] = do_click(pid, args.x, args.y, args.button, args.count)
+    elif args.cmd == "move":
+        result["detail"] = do_move(pid, args.x, args.y)
+    elif args.cmd == "scroll":
+        result["detail"] = do_scroll(pid, args.x, args.y, args.amount)
+    elif args.cmd == "key":
+        result["detail"] = do_key(pid, args.key, args.flags)
+    print(json.dumps(result, ensure_ascii=False))
+    return 0

macos_computer_use/jev.py

@@ -0,0 +1,223 @@
+"""Jev (TypeSafe System One) integration: semantic guards for computer use.
+
+Two entry points, both reading JSON on stdin and printing JSON on stdout:
+
+- ``guard``  : one request fans out several independent yes/no and choice
+  judgments (right target / input correctness / blocker / next action) and the
+  *code* applies thresholds. This is the highest-ROI place for a small model in
+  a computer-use loop: immediately before an irreversible action.
+- ``select`` : pick one element out of AX candidates, with a ``none`` escape
+  hatch and a confidence gate.
+
+Jev returns typed judgments and calibrated probabilities, not text. Policy and
+side effects stay in code. API key: ``TYPESAFE_API_KEY`` or
+``~/.config/typesafe/api_key``. See https://docs.typesafe.ai
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from typing import Any
+
+from . import darwin
+
+API = "https://api.typesafe.ai/v1/systemone"
+DEFAULT_MODEL = "jev-latest"
+# Act only when the semantic checks clear the bar (tune on your own data).
+T_TARGET = float(os.environ.get("MACOS_CU_JEV_T_TARGET", "0.85"))
+T_INPUT = float(os.environ.get("MACOS_CU_JEV_T_INPUT", "0.85"))
+
+
+def model() -> str:
+    return os.environ.get("TYPESAFE_MODEL", DEFAULT_MODEL)
+
+
+def _post(body: dict[str, Any], key: str, timeout: float = 30.0) -> tuple[dict[str, Any] | None, int]:
+    req = urllib.request.Request(
+        API,
+        data=json.dumps(body).encode(),
+        headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.load(resp), 0
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": f"HTTP {e.code}", "body": e.read().decode()[:300]}), file=sys.stderr)
+        return None, 3
+    except urllib.error.URLError as e:
+        print(json.dumps({"error": "network_error", "detail": str(e.reason)[:200]}), file=sys.stderr)
+        return None, 3
+
+
+def _api_key_or_exit() -> str:
+    key = darwin.jev_key()
+    if not key:
+        print(
+            json.dumps(
+                {
+                    "error": "TYPESAFE_API_KEY not set",
+                    "hint": "export TYPESAFE_API_KEY=... or write it to ~/.config/typesafe/api_key "
+                    "(keys: https://console.typesafe.ai/keys)",
+                }
+            ),
+            file=sys.stderr,
+        )
+        raise SystemExit(2)
+    return key
+
+
+def decide(right: float, inp: float, blocker: str, nxt: str) -> str:
+    """Apply the guard policy. Pure function so it can be unit-tested.
+
+    Code owns the decision: proceed only when nothing blocks and both semantic
+    checks clear their thresholds. The model may only contribute the two safe
+    recoveries (they cannot send or submit anything); everything else asks the
+    user.
+    """
+    if blocker == "none" and right >= T_TARGET and inp >= T_INPUT:
+        return "proceed"
+    if nxt in ("switch_target", "retype_input"):
+        return nxt
+    return "ask_user"
+
+
+def guard(payload: dict[str, Any], key: str) -> int:
+    state = {
+        "task": payload.get("task", ""),
+        "expected": payload.get("expected", {}),
+        "observed": payload.get("observed", {}),
+    }
+    body = {
+        "state": state,
+        "model": model(),
+        "questions": {
+            "right_target": {
+                "type": "noul",
+                "instructions": (
+                    "Is the observed target (chat_title / window_title) the same logical target "
+                    "the task intends (expected)? Ignore cosmetic differences; answer about identity."
+                ),
+                "criteria": {
+                    "true": "observed target is the intended recipient/window",
+                    "false": "observed target is a different recipient/window",
+                },
+            },
+            "input_ok": {
+                "type": "noul",
+                "instructions": (
+                    "Does `observed.input_text` match `expected.message` (the message the task intends "
+                    "to send)? Allow minor whitespace differences, but answer false for empty, unrelated, "
+                    "garbled, or someone else's clipboard content."
+                ),
+                "criteria": {
+                    "true": "observed input is the intended message",
+                    "false": "observed input differs from expected.message, or is empty/unrelated/garbled",
+                },
+            },
+            "blocker": {
+                "type": "choice",
+                "instructions": "What, if anything, blocks proceeding with the action?",
+                "criteria": {
+                    "none": "Nothing blocks; safe to proceed",
+                    "wrong_target": "The active chat/window is not the intended target",
+                    "bad_input": "The input content is wrong, empty, or not the intended text",
+                    "login_required": "A login or permission screen is showing",
+                    "error_dialog": "An error or unexpected dialog is in the way",
+                    "unknown": "Not enough evidence to tell",
+                },
+            },
+            "next_action": {
+                "type": "choice",
+                "instructions": "What should the agent do next?",
+                "criteria": {
+                    "proceed": "Target and input are correct; perform the action",
+                    "switch_target": "Switch to the intended target first, then re-check",
+                    "retype_input": "Clear the input and re-enter the intended text",
+                    "ask_user": "Stop and ask the user",
+                },
+            },
+        },
+    }
+    data, code = _post(body, key)
+    if data is None:
+        return code
+
+    a = data.get("answers", {})
+    right = a.get("right_target", {}).get("noul", 0.0)
+    inp = a.get("input_ok", {}).get("noul", 0.0)
+    blocker = a.get("blocker", {}).get("choice", "unknown")
+    nxt = a.get("next_action", {}).get("choice", "ask_user")
+
+    # Policy: code owns the decision. See `decide`.
+    decision = decide(right, inp, blocker, nxt)
+
+    print(json.dumps({
+        "answers": {"right_target": right, "input_ok": inp, "blocker": blocker, "next_action": nxt},
+        "thresholds": {"right_target": T_TARGET, "input_ok": T_INPUT},
+        "decision": decision,
+        "model_suggested": nxt,
+        "usage": data.get("usage"),
+    }, ensure_ascii=False))
+    return 0
+
+
+def select(payload: dict[str, Any], key: str) -> int:
+    goal = payload.get("goal", "")
+    candidates = payload.get("candidates", [])
+    if not candidates:
+        print(json.dumps({"error": "no candidates supplied"}), file=sys.stderr)
+        return 2
+
+    index = {str(c.get("id")): (c.get("text") or "") for c in candidates}
+    choices = dict(index)
+    choices["none"] = "No candidate matches the goal"
+    body = {
+        "state": {"goal": goal, "candidates": [{"id": str(c.get("id")), "text": c.get("text") or ""} for c in candidates]},
+        "model": model(),
+        "questions": {
+            "pick": {
+                "type": "choice",
+                "instructions": (
+                    "Which candidate is the element described by `goal`? Answer `none` when no "
+                    "candidate matches. Judge by role, size, and the text/label it carries."
+                ),
+                "criteria": choices,
+            }
+        },
+    }
+    data, code = _post(body, key)
+    if data is None:
+        return code
+
+    picked = data.get("answers", {}).get("pick", {})
+    choice = str(picked.get("choice", "none"))
+    probs = picked.get("probabilities", {}) or {}
+    confidence = float(probs.get(choice, 0.0))
+
+    if choice == "none" or choice not in index:
+        gate = "no_match"
+    elif confidence >= float(os.environ.get("MACOS_CU_JEV_SELECT_GATE", "0.7")):
+        gate = "auto"
+    else:
+        gate = "low_confidence_review"
+
+    print(json.dumps({
+        "id": choice,
+        "confidence": round(confidence, 4),
+        "probabilities": {k: round(float(v), 4) for k, v in probs.items()},
+        "gate": gate,
+        "usage": data.get("usage"),
+    }, ensure_ascii=False))
+    return 0
+
+
+def run(args: argparse.Namespace) -> int:
+    key = _api_key_or_exit()
+    payload = json.load(sys.stdin)
+    return guard(payload, key) if args.cmd == "guard" else select(payload, key)