code-puppy-core-plugins 0.0.2__py3-none-any.whl

code_puppy_core_plugins/computer_use/commands.py

@@ -0,0 +1,62 @@
+"""Slash commands for explicit computer-use authorization."""
+
+from __future__ import annotations
+
+import shlex
+
+from code_puppy.messaging import emit_error, emit_info, emit_success, emit_warning
+
+from .policy import policy_store
+
+
+def command_help() -> list[tuple[str, str]]:
+    return [
+        (
+            "computer-use",
+            "Enable, disable, pause, or configure macOS Computer Use",
+        )
+    ]
+
+
+def handle_command(command: str, name: str) -> bool | None:
+    if name != "computer-use":
+        return None
+    try:
+        tokens = shlex.split(command)
+    except ValueError as exc:
+        emit_error(f"Invalid /computer-use command: {exc}")
+        return True
+    subcommand = tokens[1].casefold() if len(tokens) > 1 else "status"
+    if subcommand == "status":
+        emit_info(f"Computer Use policy: {policy_store.status()}")
+    elif subcommand in {"enable", "disable"}:
+        enabled = subcommand == "enable"
+        policy_store.set_enabled(enabled)
+        if enabled:
+            emit_success(
+                "Computer Use enabled. It can now view and control allowed apps."
+            )
+        else:
+            emit_warning("Computer Use disabled in settings.")
+    elif subcommand == "pause":
+        policy_store.set_paused(True)
+        emit_warning("Computer Use paused immediately.")
+    elif subcommand == "resume":
+        policy_store.set_paused(False)
+        emit_success("Computer Use resumed.")
+    elif subcommand in {"allow", "deny"}:
+        if len(tokens) < 3:
+            emit_error("Usage: /computer-use allow|deny BUNDLE_ID")
+            return True
+        bundle_id = tokens[2]
+        if subcommand == "deny":
+            policy_store.deny(bundle_id)
+            emit_warning(f"Computer Use denied for {bundle_id}.")
+        else:
+            policy_store.allow(bundle_id)
+            emit_success(f"Computer Use allowed for {bundle_id}.")
+    else:
+        emit_error(
+            "Usage: /computer-use [status|enable|disable|pause|resume|allow|deny]"
+        )
+    return True

code_puppy_core_plugins/computer_use/geometry.py

@@ -0,0 +1,81 @@
+"""Coordinate spaces used by macOS computer use."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .backend_types import ComputerUseError
+
+
+@dataclass(frozen=True)
+class Rect:
+    x: float
+    y: float
+    width: float
+    height: float
+
+    def as_dict(self) -> dict[str, float]:
+        return {
+            "x": self.x,
+            "y": self.y,
+            "width": self.width,
+            "height": self.height,
+        }
+
+
+@dataclass(frozen=True)
+class CaptureGeometry:
+    """Mapping between screenshot pixels and global Quartz screen points.
+
+    ScreenCaptureKit and CGWindow bounds use a top-left global coordinate
+    system. AppKit uses bottom-left coordinates; callers that provide AppKit
+    coordinates must opt into the explicit conversion helper below.
+    """
+
+    window_points: Rect
+    image_width_pixels: int
+    image_height_pixels: int
+    backing_scale: float
+
+    def __post_init__(self) -> None:
+        if self.window_points.width <= 0 or self.window_points.height <= 0:
+            raise ComputerUseError("Window bounds must have positive dimensions.")
+        if self.image_width_pixels <= 0 or self.image_height_pixels <= 0:
+            raise ComputerUseError("Screenshot dimensions must be positive.")
+        if self.backing_scale <= 0:
+            raise ComputerUseError("Backing scale must be positive.")
+
+    def screenshot_to_quartz(self, x: float, y: float) -> tuple[float, float]:
+        if not (0 <= x < self.image_width_pixels):
+            raise ComputerUseError(
+                f"Screenshot x={x} is outside 0..{self.image_width_pixels - 1}."
+            )
+        if not (0 <= y < self.image_height_pixels):
+            raise ComputerUseError(
+                f"Screenshot y={y} is outside 0..{self.image_height_pixels - 1}."
+            )
+        return (
+            self.window_points.x
+            + (float(x) / self.image_width_pixels) * self.window_points.width,
+            self.window_points.y
+            + (float(y) / self.image_height_pixels) * self.window_points.height,
+        )
+
+    @staticmethod
+    def appkit_to_quartz(
+        x: float, y: float, display_points: Rect
+    ) -> tuple[float, float]:
+        """Convert bottom-left AppKit coordinates to top-left Quartz points."""
+        return x, display_points.y + display_points.height - (y - display_points.y)
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "window_bounds_points": self.window_points.as_dict(),
+            "screenshot_size_pixels": {
+                "width": self.image_width_pixels,
+                "height": self.image_height_pixels,
+            },
+            "backing_scale": self.backing_scale,
+            "screenshot_coordinate_system": "top-left, window-local pixels",
+            "action_coordinate_system": "top-left, global Quartz points",
+        }

code_puppy_core_plugins/computer_use/inline_image.py

@@ -0,0 +1,56 @@
+"""Best-effort inline image display for modern terminals."""
+
+from __future__ import annotations
+
+import base64
+import os
+import sys
+from pathlib import Path
+
+
+def _terminal_kind() -> str | None:
+    program = os.environ.get("TERM_PROGRAM", "").casefold()
+    term = os.environ.get("TERM", "").casefold()
+    if os.environ.get("ITERM_SESSION_ID") or program == "iterm.app":
+        return "iterm"
+    if (
+        program in {"ghostty", "wezterm"}
+        or os.environ.get("KITTY_WINDOW_ID")
+        or "ghostty" in term
+        or "kitty" in term
+    ):
+        return "kitty"
+    return None
+
+
+def emit_inline_image(path: str | Path) -> bool:
+    """Render a PNG using iTerm2 or Kitty graphics escape sequences."""
+    image_path = Path(path)
+    kind = _terminal_kind()
+    if kind is None or not sys.stdout.isatty() or not image_path.is_file():
+        return False
+
+    try:
+        if kind == "iterm":
+            data = base64.b64encode(image_path.read_bytes()).decode("ascii")
+            name = base64.b64encode(image_path.name.encode()).decode("ascii")
+            sequence = (
+                f"\033]1337;File=name={name};inline=1;width=50%;"
+                f"preserveAspectRatio=1:{data}\a\n"
+            )
+        else:
+            encoded_path = base64.b64encode(str(image_path.resolve()).encode()).decode(
+                "ascii"
+            )
+            # Direct file transmission keeps the full screenshot out of the terminal
+            # stream; q=2 suppresses replies, c/r bound it to a compact PiP preview.
+            sequence = f"\033_Ga=T,t=f,f=100,q=2,c=64,r=22;{encoded_path}\033\\\n"
+
+        from code_puppy.messaging.run_ui import suspended_run_ui
+
+        with suspended_run_ui():
+            sys.stdout.write(sequence)
+            sys.stdout.flush()
+    except Exception:
+        return False
+    return True

code_puppy_core_plugins/computer_use/keycodes.py

@@ -0,0 +1,47 @@
+"""macOS virtual key codes for keys exposed by the computer-use tool."""
+
+KEY_CODES = {
+    "return": 36,
+    "enter": 36,
+    "tab": 48,
+    "space": 49,
+    "delete": 51,
+    "escape": 53,
+    "left": 123,
+    "right": 124,
+    "down": 125,
+    "up": 126,
+    **dict(
+        zip(
+            "abcdefghijklmnopqrstuvwxyz",
+            (
+                0,
+                11,
+                8,
+                2,
+                14,
+                3,
+                5,
+                4,
+                34,
+                38,
+                40,
+                37,
+                46,
+                45,
+                31,
+                35,
+                12,
+                15,
+                1,
+                17,
+                32,
+                9,
+                13,
+                7,
+                16,
+                6,
+            ),
+        )
+    ),
+}

code_puppy_core_plugins/computer_use/policy.py

@@ -0,0 +1,144 @@
+"""Persisted computer-use consent, explicit denials, and hard stops."""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+from code_puppy.config import CONFIG_DIR
+
+from .backend_types import ComputerUseError
+
+POLICY_PATH = Path(CONFIG_DIR) / "computer_use_policy.json"
+SYSTEM_DENYLIST = {
+    "com.apple.loginwindow",
+    "com.apple.securityagent",
+    "com.apple.systempreferences",
+    "com.apple.systemsettings",
+    "com.apple.keychainaccess",
+    "com.apple.authorizationhost",
+}
+
+
+class PolicyStore:
+    def __init__(self, path: Path = POLICY_PATH) -> None:
+        self.path = path
+        self._paused = False
+        self._lock = threading.RLock()
+
+    def _load(self) -> dict[str, Any]:
+        if not self.path.is_file():
+            return {"enabled": None, "denied": []}
+        try:
+            payload = json.loads(self.path.read_text())
+        except (OSError, json.JSONDecodeError):
+            return {"enabled": None, "denied": []}
+        enabled = payload.get("enabled")
+        return {
+            "enabled": enabled if isinstance(enabled, bool) else None,
+            "denied": list(payload.get("denied", [])),
+        }
+
+    def _save(self, payload: dict[str, Any]) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        temporary = self.path.with_suffix(".tmp")
+        temporary.write_text(json.dumps(payload, indent=2, sort_keys=True))
+        os.chmod(temporary, 0o600)
+        temporary.replace(self.path)
+
+    def allow(self, bundle_id: str) -> None:
+        """Remove an explicit denial; all other applications are allowed."""
+        normalized = bundle_id.casefold()
+        if normalized in SYSTEM_DENYLIST:
+            raise ComputerUseError(
+                f"Computer Use is never allowed for security process {bundle_id}."
+            )
+        with self._lock:
+            payload = self._load()
+            denied = {str(item).casefold() for item in payload["denied"]}
+            denied.discard(normalized)
+            payload["denied"] = sorted(denied)
+            self._save(payload)
+
+    def deny(self, bundle_id: str) -> None:
+        normalized = bundle_id.casefold()
+        with self._lock:
+            payload = self._load()
+            denied = {str(item).casefold() for item in payload["denied"]}
+            denied.add(normalized)
+            payload["denied"] = sorted(denied)
+            self._save(payload)
+
+    def set_enabled(self, enabled: bool) -> None:
+        with self._lock:
+            payload = self._load()
+            payload["enabled"] = enabled
+            self._save(payload)
+
+    def set_paused(self, paused: bool) -> None:
+        with self._lock:
+            self._paused = paused
+
+    def is_enabled(self) -> bool:
+        """Return whether the user explicitly opted in to Computer Use."""
+        with self._lock:
+            return self._load()["enabled"] is True
+
+    def require_enabled(self) -> None:
+        with self._lock:
+            if self._paused:
+                raise ComputerUseError(
+                    "Computer Use is paused by the emergency stop. Run "
+                    "`/computer-use resume` after confirming it is safe."
+                )
+            payload = self._load()
+            if payload["enabled"] is None:
+                raise ComputerUseError(
+                    "Computer Use needs your one-time permission before its first "
+                    "use. It can view screenshots and control apps on this Mac. "
+                    "Run `/computer-use enable` to allow it, or "
+                    "`/computer-use disable` to keep it off. You can change this "
+                    "setting later with the same commands."
+                )
+            if payload["enabled"] is False:
+                raise ComputerUseError(
+                    "Computer Use is disabled in settings. Run "
+                    "`/computer-use enable` to turn it on."
+                )
+
+    def require(self, bundle_id: str) -> None:
+        self.require_enabled()
+        normalized = bundle_id.casefold()
+        with self._lock:
+            if normalized in SYSTEM_DENYLIST:
+                raise ComputerUseError(
+                    f"Computer Use is blocked for security process {bundle_id}."
+                )
+            payload = self._load()
+            denied = {str(item).casefold() for item in payload["denied"]}
+            if normalized in denied:
+                raise ComputerUseError(
+                    f"Computer Use is denied for application {bundle_id}."
+                )
+
+    def status(self) -> dict[str, Any]:
+        with self._lock:
+            payload = self._load()
+            return {
+                "mode": (
+                    "enabled"
+                    if payload["enabled"] is True
+                    else "disabled"
+                    if payload["enabled"] is False
+                    else "needs-first-use-consent"
+                ),
+                "paused": self._paused,
+                "denied": payload["denied"],
+                "system_denied": sorted(SYSTEM_DENYLIST),
+            }
+
+
+policy_store = PolicyStore()

code_puppy_core_plugins/computer_use/register_callbacks.py

@@ -0,0 +1,96 @@
+"""Register the macOS computer-use tools as an optional builtin plugin."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+from code_puppy.callbacks import register_callback
+from code_puppy.i18n import t
+from code_puppy.messaging import emit_info
+
+from .policy import policy_store
+
+_TOOL_NAMES = (
+    "computer_get_app_state",
+    "computer_snapshot",
+    "computer_click",
+    "computer_set_value",
+    "computer_perform_action",
+    "computer_select_text",
+    "computer_press_key",
+    "computer_type_text",
+    "computer_scroll",
+    "computer_drag",
+    "computer_screenshot",
+    "computer_use_batch",
+)
+
+
+def _available() -> bool:
+    return sys.platform == "darwin"
+
+
+def _enabled() -> bool:
+    return _available() and policy_store.is_enabled()
+
+
+def _register_tools() -> list[dict[str, Any]]:
+    if not _enabled():
+        return []
+
+    from .tools import REGISTRARS
+
+    return [{"name": name, "register_func": REGISTRARS[name]} for name in _TOOL_NAMES]
+
+
+def _register_agent_tools(agent_name: str | None = None) -> list[str]:
+    del agent_name
+    return list(_TOOL_NAMES) if _enabled() else []
+
+
+def _load_prompt() -> str | None:
+    if not _enabled():
+        return None
+    return (
+        "macOS Computer Use requires persisted one-time user consent. If a tool "
+        "reports that consent is unset or disabled, show its exact slash command "
+        "to the user and do not work around it. Slash commands are not visible in "
+        "conversation history, so never repeat an old consent error from memory. "
+        "When the user repeats the request, always call the appropriate Computer "
+        "Use state tool again; the tool's current result is authoritative. Once "
+        "enabled, do not ask for per-application approval. Prefer "
+        "computer_get_app_state and element IDs over screen coordinates. Every "
+        "standalone mutation needs its current state_revision; guarded batches "
+        "return fresh state after deterministic UI settling. For pixel-based "
+        "focus-and-type workflows, use one guarded batch instead of guessing across "
+        "separate revisions. Verify the returned state and recover until the "
+        "requested outcome is visibly complete. Do not stop merely because the user "
+        "moves the pointer or types. The emergency stop and hard macOS "
+        "security-process denylist remain enforced."
+    )
+
+
+def _startup() -> None:
+    if _available() and not policy_store.is_enabled():
+        emit_info(t("computer_use.startup.opt_in"))
+
+
+def _custom_help():
+    from .commands import command_help
+
+    return command_help()
+
+
+def _custom_command(command: str, name: str):
+    from .commands import handle_command
+
+    return handle_command(command, name)
+
+
+register_callback("startup", _startup)
+register_callback("register_tools", _register_tools)
+register_callback("register_agent_tools", _register_agent_tools)
+register_callback("load_prompt", _load_prompt)
+register_callback("custom_command_help", _custom_help)
+register_callback("custom_command", _custom_command)

code_puppy_core_plugins/computer_use/safety.py

@@ -0,0 +1,18 @@
+"""Shared policy and stale-state enforcement."""
+
+from __future__ import annotations
+
+from .policy import policy_store
+from .state import AppState, state_store
+
+
+def require_safe_state(
+    revision: str,
+    *,
+    consume: bool = False,
+) -> AppState:
+    state = state_store.require(revision)
+    policy_store.require(state.bundle_id)
+    if consume:
+        return state_store.require(revision, consume=True)
+    return state

code_puppy_core_plugins/computer_use/settle.py

@@ -0,0 +1,64 @@
+"""Deterministic UI settling based on accessibility-state stability."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from typing import Any, Callable
+
+
+def _fingerprint(snapshot: dict[str, Any]) -> str:
+    normalized = [
+        {
+            key: node.get(key)
+            for key in (
+                "role",
+                "title",
+                "description",
+                "value",
+                "enabled",
+                "focused",
+                "actions",
+            )
+        }
+        for node in snapshot.get("nodes", [])
+    ]
+    encoded = json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def wait_for_ui_settle(
+    snapshotter: Callable[[str, int], dict[str, Any]],
+    app_name: str,
+    timeout: float = 3.0,
+    interval: float = 0.1,
+    stable_observations: int = 3,
+) -> dict[str, Any]:
+    """Return after the focused accessibility state repeats consecutively."""
+    deadline = time.monotonic() + max(0.1, timeout)
+    previous = None
+    stable = 0
+    observations = 0
+    while time.monotonic() < deadline:
+        snapshot = snapshotter(app_name, 120)
+        observations += 1
+        fingerprint = _fingerprint(snapshot)
+        if fingerprint == previous:
+            stable += 1
+            if stable >= stable_observations:
+                return {
+                    "settled": True,
+                    "observations": observations,
+                    "fingerprint": fingerprint,
+                }
+        else:
+            previous = fingerprint
+            stable = 0
+        time.sleep(max(0.01, interval))
+    return {
+        "settled": False,
+        "observations": observations,
+        "fingerprint": previous,
+        "reason": "timeout",
+    }

code_puppy_core_plugins/computer_use/state.py

@@ -0,0 +1,86 @@
+"""Revisioned computer-use state shared by snapshots and actions."""
+
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+from .backend_types import ComputerUseError
+from .geometry import CaptureGeometry
+
+
+@dataclass
+class AppState:
+    revision: str
+    application: str
+    bundle_id: str
+    pid: int
+    window_id: int
+    window_title: str
+    geometry: CaptureGeometry
+    screenshot_path: str
+    elements: dict[int, Any] = field(repr=False)
+    created_at: float = field(default_factory=time.monotonic)
+    consumed: bool = False
+
+    def public_metadata(self) -> dict[str, Any]:
+        return {
+            "state_revision": self.revision,
+            "application": self.application,
+            "bundle_id": self.bundle_id,
+            "pid": self.pid,
+            "window_id": self.window_id,
+            "window_title": self.window_title,
+            "screenshot_path": self.screenshot_path,
+            **self.geometry.as_dict(),
+        }
+
+
+class StateStore:
+    def __init__(self, max_age_seconds: float = 120.0) -> None:
+        self._state: AppState | None = None
+        self._lock = threading.RLock()
+        self.max_age_seconds = max_age_seconds
+
+    def create(self, **kwargs: Any) -> AppState:
+        state = AppState(revision=uuid.uuid4().hex, **kwargs)
+        with self._lock:
+            self._state = state
+        return state
+
+    def require(self, revision: str, *, consume: bool = False) -> AppState:
+        if not revision:
+            raise ComputerUseError(
+                "state_revision is required. Call computer_get_app_state first."
+            )
+        with self._lock:
+            state = self._state
+            if state is None or state.revision != revision:
+                raise ComputerUseError(
+                    "Stale or unknown state_revision. Call computer_get_app_state again."
+                )
+            if time.monotonic() - state.created_at > self.max_age_seconds:
+                raise ComputerUseError(
+                    "The app state expired. Call computer_get_app_state again."
+                )
+            if state.consumed:
+                raise ComputerUseError(
+                    "The app state changed after an action. Fetch fresh app state."
+                )
+            if consume:
+                state.consumed = True
+            return state
+
+    def current(self) -> AppState | None:
+        with self._lock:
+            return self._state
+
+    def clear(self) -> None:
+        with self._lock:
+            self._state = None
+
+
+state_store = StateStore()