rea-agents 2.2.0 → 2.4.0

package/scripts/hopper-demo-x11.py

@@ -6,10 +6,14 @@ from __future__ import annotations
 import argparse
 import ctypes
 import hashlib
+import json
 import os
 from pathlib import Path
 import secrets
+import stat
 import subprocess
+import tempfile
+import threading
 import time
 
 
@@ -31,12 +35,60 @@ UNSUPPORTED_DEMO_DIALOG = 76
 UNEXPECTED_DISPLAY_GEOMETRY = 77
 X11_INPUT_FAILED = 78
 RUNTIME_DEPENDENCY_UNAVAILABLE = 79
+X11_SOCKET_DIRECTORY_UNUSABLE = 80
+DIAGNOSTIC_PREFIX = "REA_X11_DIAGNOSTIC_V1="
+X11_SOCKET_DIRECTORY = Path("/tmp/.X11-unix")
+XVFB_STDERR_LIMIT = 16 * 1024
 
 
 class AdapterFailure(Exception):
-    def __init__(self, code: int, message: str) -> None:
+    def __init__(
+        self,
+        code: int,
+        reason: str,
+        message: str,
+        stderr_bytes: int = 0,
+        stderr_truncated: bool = False,
+    ) -> None:
         super().__init__(message)
         self.code = code
+        self.reason = reason
+        self.stderr_bytes = stderr_bytes
+        self.stderr_truncated = stderr_truncated
+
+    def with_stderr(self, capture: BoundedStderr) -> AdapterFailure:
+        self.stderr_bytes, self.stderr_truncated, _ = capture.snapshot()
+        return self
+
+
+class BoundedStderr:
+    def __init__(self, stream: object) -> None:
+        self._stream = stream
+        self._retained = bytearray()
+        self._bytes = 0
+        self._lock = threading.Lock()
+        self._thread = threading.Thread(target=self._drain, daemon=True)
+        self._thread.start()
+
+    def _drain(self) -> None:
+        while True:
+            chunk = self._stream.read(4096)
+            if not chunk:
+                return
+            with self._lock:
+                self._bytes += len(chunk)
+                remaining = XVFB_STDERR_LIMIT - len(self._retained)
+                if remaining > 0:
+                    self._retained.extend(chunk[:remaining])
+
+    def snapshot(self) -> tuple[int, bool, str]:
+        with self._lock:
+            retained = bytes(self._retained)
+            total = self._bytes
+        return total, total > len(retained), retained.decode("utf-8", "replace")
+
+    def close(self) -> None:
+        self._thread.join(timeout=0.5)
 
 
 class XWindowAttributes(ctypes.Structure):
@@ -69,8 +121,18 @@ class XWindowAttributes(ctypes.Structure):
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
-    parser.add_argument("--hopper", required=True)
-    parser.add_argument("--socket", required=True)
+    parser.add_argument("--probe", action="store_true")
+    parser.add_argument("--hopper")
+    parser.add_argument("--socket")
+    parser.add_argument(
+        "--strategy",
+        choices=("direct", "user-mount-namespace"),
+        default="direct",
+    )
+    parser.add_argument("--mount-private-x11", action="store_true")
+    parser.add_argument("--xvfb", default="/usr/bin/Xvfb")
+    parser.add_argument("--xauth", default="/usr/bin/xauth")
+    parser.add_argument("--display-number", type=int)
     parser.add_argument("command", nargs=argparse.REMAINDER)
     return parser.parse_args()
 
@@ -189,66 +251,369 @@ def configure_x11(x11: ctypes.CDLL, xtst: ctypes.CDLL) -> None:
     xtst.XTestFakeButtonEvent.restype = ctypes.c_int
 
 
-def start_xvfb(session: Path) -> tuple[subprocess.Popen[bytes], str, Path]:
+def is_wsl() -> bool:
+    for path in (Path("/proc/sys/kernel/osrelease"), Path("/proc/version")):
+        try:
+            if "microsoft" in path.read_text().lower():
+                return True
+        except OSError:
+            continue
+    return False
+
+
+def socket_directory_state() -> tuple[str | None, bool | None, bool]:
+    try:
+        value = X11_SOCKET_DIRECTORY.lstat()
+    except FileNotFoundError:
+        try:
+            read_only = bool(os.statvfs(X11_SOCKET_DIRECTORY.parent).f_flag & os.ST_RDONLY)
+        except OSError:
+            read_only = None
+        return None, read_only, False
+    except OSError:
+        return None, None, False
+    try:
+        read_only = bool(os.statvfs(X11_SOCKET_DIRECTORY).f_flag & os.ST_RDONLY)
+    except OSError:
+        read_only = None
+    return f"{stat.S_IMODE(value.st_mode):04o}", read_only, stat.S_ISDIR(value.st_mode)
+
+
+def diagnostic(
+    args: argparse.Namespace,
+    operation: str,
+    status: str,
+    failure: AdapterFailure | None = None,
+    capture: BoundedStderr | None = None,
+) -> dict[str, object]:
+    mode, read_only, _ = socket_directory_state()
+    captured_bytes, captured_truncated, _ = (
+        (0, False, "") if capture is None else capture.snapshot()
+    )
+    return {
+        "schema_version": 1,
+        "component": "hopper_private_display",
+        "operation": operation,
+        "status": status,
+        "failure_code": None if failure is None else failure_name(failure.code),
+        "reason": "ready" if failure is None else failure.reason,
+        "socket_directory": str(X11_SOCKET_DIRECTORY),
+        "socket_directory_mode": mode,
+        "mount_read_only": read_only,
+        "effective_socket_directory_mode": mode,
+        "effective_mount_read_only": read_only,
+        "wsl": is_wsl(),
+        "strategy": args.strategy,
+        "fallback_reason": None,
+        "xvfb_stderr_bytes": captured_bytes
+        if failure is None
+        else failure.stderr_bytes,
+        "xvfb_stderr_truncated": captured_truncated
+        if failure is None
+        else failure.stderr_truncated,
+    }
+
+
+def failure_name(code: int) -> str:
+    return {
+        PRIVATE_DISPLAY_UNAVAILABLE: "private_display_unavailable",
+        X11_AUTHORIZATION_FAILED: "x11_authorization_failed",
+        UNSUPPORTED_HOPPER_BUILD: "unsupported_hopper_build",
+        INVALID_LAUNCH_COMMAND: "invalid_launch_command",
+        PROCESS_OWNERSHIP_MISMATCH: "process_ownership_mismatch",
+        HOPPER_EXITED_DURING_STARTUP: "hopper_exited_during_startup",
+        UNSUPPORTED_DEMO_DIALOG: "unsupported_demo_dialog",
+        UNEXPECTED_DISPLAY_GEOMETRY: "unexpected_display_geometry",
+        X11_INPUT_FAILED: "x11_input_failed",
+        RUNTIME_DEPENDENCY_UNAVAILABLE: "runtime_dependency_unavailable",
+        X11_SOCKET_DIRECTORY_UNUSABLE: "x11_socket_directory_unusable",
+    }[code]
+
+
+def emit_diagnostic(value: dict[str, object]) -> None:
+    print(
+        DIAGNOSTIC_PREFIX + json.dumps(value, sort_keys=True, separators=(",", ":")),
+        file=os.sys.stderr,
+    )
+
+
+def mount_private_x11() -> None:
+    try:
+        mounted = subprocess.run(
+            [
+                "/usr/bin/mount",
+                "-t",
+                "tmpfs",
+                "-o",
+                "mode=1777,nosuid,nodev",
+                "rea-x11",
+                str(X11_SOCKET_DIRECTORY),
+            ],
+            check=False,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            timeout=2,
+        )
+    except FileNotFoundError as error:
+        raise AdapterFailure(
+            RUNTIME_DEPENDENCY_UNAVAILABLE,
+            "missing_mount",
+            "mount executable unavailable",
+        ) from error
+    except subprocess.TimeoutExpired as error:
+        raise AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "mount_failed",
+            "private X11 mount timed out",
+        ) from error
+    if mounted.returncode != 0:
+        raise AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "mount_failed",
+            "private X11 mount failed",
+        )
+    mode, read_only, directory = socket_directory_state()
+    if not directory or mode != "1777" or read_only is not False:
+        raise AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "mount_validation_failed",
+            "private X11 mount validation failed",
+        )
+
+
+def stop_process(process: subprocess.Popen[bytes]) -> None:
+    if process.poll() is not None:
+        return
+    process.terminate()
+    try:
+        process.wait(timeout=2)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait()
+
+
+def authorize_display(
+    authority: Path,
+    display_name: str,
+    xauth_path: str,
+) -> None:
+    try:
+        authorized = subprocess.run(
+            [
+                xauth_path,
+                "-f",
+                str(authority),
+                "add",
+                display_name,
+                ".",
+                secrets.token_hex(16),
+            ],
+            check=False,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            timeout=2,
+        )
+    except FileNotFoundError as error:
+        raise AdapterFailure(
+            RUNTIME_DEPENDENCY_UNAVAILABLE,
+            "missing_xauth",
+            "xauth executable unavailable",
+        ) from error
+    except subprocess.TimeoutExpired as error:
+        raise AdapterFailure(
+            X11_AUTHORIZATION_FAILED,
+            "xauth_failed",
+            "private X authorization timed out",
+        ) from error
+    if authorized.returncode != 0:
+        raise AdapterFailure(
+            X11_AUTHORIZATION_FAILED,
+            "xauth_failed",
+            "private X authorization failed",
+        )
+    try:
+        authority.chmod(0o600)
+    except OSError as error:
+        raise AdapterFailure(
+            X11_AUTHORIZATION_FAILED,
+            "xauth_failed",
+            "private X authorization file unavailable",
+        ) from error
+
+
+def classify_xvfb_failure(capture: BoundedStderr) -> AdapterFailure:
+    stderr_bytes, stderr_truncated, text = capture.snapshot()
+    mode, read_only, directory = socket_directory_state()
+    if read_only is True:
+        return AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "socket_directory_read_only",
+            "X11 socket directory is read-only",
+            stderr_bytes,
+            stderr_truncated,
+        )
+    if directory and mode != "1777":
+        return AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "socket_directory_mode",
+            "X11 socket directory mode is not 1777",
+            stderr_bytes,
+            stderr_truncated,
+        )
+    if mode is not None and not directory:
+        return AdapterFailure(
+            X11_SOCKET_DIRECTORY_UNUSABLE,
+            "socket_directory_type",
+            "X11 socket path is not a directory",
+            stderr_bytes,
+            stderr_truncated,
+        )
+    lowered = text.lower()
+    if "error while loading shared libraries" in lowered:
+        return AdapterFailure(
+            RUNTIME_DEPENDENCY_UNAVAILABLE,
+            "missing_library",
+            "Xvfb shared library unavailable",
+            stderr_bytes,
+            stderr_truncated,
+        )
+    if any(
+        marker in lowered
+        for marker in (
+            "server is already active for display",
+            "address already in use",
+            "cannot establish any listening sockets",
+        )
+    ):
+        return AdapterFailure(
+            PRIVATE_DISPLAY_UNAVAILABLE,
+            "address_collision",
+            "private display address collision",
+            stderr_bytes,
+            stderr_truncated,
+        )
+    return AdapterFailure(
+        PRIVATE_DISPLAY_UNAVAILABLE,
+        "xvfb_startup_failed",
+        "private X display unavailable",
+        stderr_bytes,
+        stderr_truncated,
+    )
+
+
+def start_xvfb(
+    session: Path,
+    args: argparse.Namespace,
+) -> tuple[subprocess.Popen[bytes], str, Path, BoundedStderr]:
     authority = session / "Xauthority"
-    for _ in range(20):
-        display_number = 100 + secrets.randbelow(900)
+    collision: AdapterFailure | None = None
+    for _ in range(3):
+        display_number = args.display_number or (100 + secrets.randbelow(900))
         display_name = f":{display_number}"
-        if Path(f"/tmp/.X11-unix/X{display_number}").exists():
+        if Path(f"/tmp/.X11-unix/X{display_number}").exists() or Path(
+            f"/tmp/.X{display_number}-lock"
+        ).exists():
+            collision = AdapterFailure(
+                PRIVATE_DISPLAY_UNAVAILABLE,
+                "address_collision",
+                "private display address collision",
+            )
+            if args.display_number is not None:
+                raise collision
             continue
-        cookie = secrets.token_hex(16)
+        authorize_display(authority, display_name, args.xauth)
         try:
-            subprocess.run(
+            process = subprocess.Popen(
                 [
-                    "/usr/bin/xauth", "-f", str(authority), "add",
-                    display_name, ".", cookie,
+                    args.xvfb,
+                    display_name,
+                    "-screen",
+                    "0",
+                    "1280x1024x24",
+                    "-nolisten",
+                    "tcp",
+                    "-auth",
+                    str(authority),
                 ],
-                check=True,
                 stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
             )
-        except (OSError, subprocess.CalledProcessError) as error:
+        except FileNotFoundError as error:
             raise AdapterFailure(
-                X11_AUTHORIZATION_FAILED, "private X authorization failed"
+                RUNTIME_DEPENDENCY_UNAVAILABLE,
+                "missing_xvfb",
+                "Xvfb executable unavailable",
             ) from error
-        authority.chmod(0o600)
-        process = subprocess.Popen(
-            [
-                "/usr/bin/Xvfb",
-                display_name,
-                "-screen",
-                "0",
-                "1280x1024x24",
-                "-nolisten",
-                "tcp",
-                "-auth",
-                str(authority),
-            ],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-        )
+        if process.stderr is None:
+            stop_process(process)
+            raise AdapterFailure(
+                PRIVATE_DISPLAY_UNAVAILABLE,
+                "xvfb_startup_failed",
+                "Xvfb stderr unavailable",
+            )
+        capture = BoundedStderr(process.stderr)
         deadline = time.monotonic() + 2
         socket = Path(f"/tmp/.X11-unix/X{display_number}")
         while time.monotonic() < deadline and process.poll() is None:
-            if socket.exists():
-                return process, display_name, authority
+            if socket.is_socket():
+                return process, display_name, authority, capture
             time.sleep(0.05)
-        if process.poll() is None:
-            process.terminate()
-        process.wait()
-    raise AdapterFailure(PRIVATE_DISPLAY_UNAVAILABLE, "private X display unavailable")
+        stop_process(process)
+        capture.close()
+        failure = classify_xvfb_failure(capture)
+        if failure.reason != "address_collision" or args.display_number is not None:
+            raise failure
+        collision = failure
+    raise collision or AdapterFailure(
+        PRIVATE_DISPLAY_UNAVAILABLE,
+        "address_collision",
+        "private display address collision",
+    )
 
 
-def main() -> int:
-    args = parse_args()
+def run_probe(args: argparse.Namespace) -> dict[str, object]:
+    with tempfile.TemporaryDirectory(prefix="rea-x11-probe-", dir="/tmp") as root:
+        xvfb, _, _, capture = start_xvfb(Path(root), args)
+        try:
+            stop_process(xvfb)
+            capture.close()
+            return diagnostic(args, "probe", "ready", capture=capture)
+        finally:
+            stop_process(xvfb)
+            capture.close()
+
+
+def run_launch(args: argparse.Namespace) -> int:
+    if args.hopper is None or args.socket is None:
+        raise AdapterFailure(
+            INVALID_LAUNCH_COMMAND,
+            "invalid_launch_command",
+            "missing Hopper launch coordinates",
+        )
     hopper = Path(args.hopper)
     socket = Path(args.socket)
-    if digest(hopper) not in SUPPORTED_HOPPER_SHA256:
-        raise AdapterFailure(UNSUPPORTED_HOPPER_BUILD, "unsupported Hopper binary")
+    try:
+        supported = digest(hopper) in SUPPORTED_HOPPER_SHA256
+    except OSError as error:
+        raise AdapterFailure(
+            INVALID_LAUNCH_COMMAND,
+            "invalid_launch_command",
+            "Hopper binary unavailable",
+        ) from error
+    if not supported:
+        raise AdapterFailure(
+            UNSUPPORTED_HOPPER_BUILD,
+            "unsupported_hopper_build",
+            "unsupported Hopper binary",
+        )
     command = args.command[1:] if args.command[:1] == ["--"] else args.command
     if not command or Path(command[0]).resolve() != hopper.resolve():
-        raise AdapterFailure(INVALID_LAUNCH_COMMAND, "invalid Hopper launch command")
-    xvfb, display_name, authority = start_xvfb(socket.parent)
+        raise AdapterFailure(
+            INVALID_LAUNCH_COMMAND,
+            "invalid_launch_command",
+            "invalid Hopper launch command",
+        )
+    xvfb, display_name, authority, capture = start_xvfb(socket.parent, args)
     child: subprocess.Popen[bytes] | None = None
     display: int | None = None
     try:
@@ -273,19 +638,27 @@ def main() -> int:
             xtst = ctypes.CDLL("libXtst.so.6")
         except OSError as error:
             raise AdapterFailure(
-                RUNTIME_DEPENDENCY_UNAVAILABLE, "X11 runtime unavailable"
+                RUNTIME_DEPENDENCY_UNAVAILABLE,
+                "missing_library",
+                "X11 runtime unavailable",
             ) from error
         configure_x11(x11, xtst)
         display = x11.XOpenDisplay(display_name.encode())
         if not display:
-            raise AdapterFailure(PRIVATE_DISPLAY_UNAVAILABLE, "private X display unavailable")
+            raise AdapterFailure(
+                PRIVATE_DISPLAY_UNAVAILABLE,
+                "x11_connection_failed",
+                "private X display unavailable",
+            )
         root = x11.XDefaultRootWindow(display)
         if (
             x11.XDisplayWidth(display, 0),
             x11.XDisplayHeight(display, 0),
         ) != EXPECTED_SCREEN:
             raise AdapterFailure(
-                UNEXPECTED_DISPLAY_GEOMETRY, "unexpected private display geometry"
+                UNEXPECTED_DISPLAY_GEOMETRY,
+                "unexpected_display_geometry",
+                "unexpected private display geometry",
             )
 
         deadline = time.monotonic() + 10
@@ -300,7 +673,9 @@ def main() -> int:
                     time.sleep(0.1)
                     continue
                 raise AdapterFailure(
-                    HOPPER_EXITED_DURING_STARTUP, "owned Hopper process exited"
+                    HOPPER_EXITED_DURING_STARTUP,
+                    "hopper_exited_during_startup",
+                    "owned Hopper process exited",
                 )
             observed = windows(x11, display, root)
             if any(window[1:] == EXPECTED_DIALOG for window in observed):
@@ -309,10 +684,14 @@ def main() -> int:
         else:
             if not hopper_is_owned(leader, hopper):
                 raise AdapterFailure(
-                    PROCESS_OWNERSHIP_MISMATCH, "Hopper process ownership mismatch"
+                    PROCESS_OWNERSHIP_MISMATCH,
+                    "process_ownership_mismatch",
+                    "Hopper process ownership mismatch",
                 )
             raise AdapterFailure(
-                UNSUPPORTED_DEMO_DIALOG, "expected Hopper demo dialog not found"
+                UNSUPPORTED_DEMO_DIALOG,
+                "unsupported_demo_dialog",
+                "expected Hopper demo dialog not found",
             )
 
         input_results = (
@@ -321,29 +700,48 @@ def main() -> int:
             xtst.XTestFakeButtonEvent(display, 1, False, 0),
         )
         if not all(input_results):
-            raise AdapterFailure(X11_INPUT_FAILED, "private X input failed")
+            raise AdapterFailure(
+                X11_INPUT_FAILED,
+                "x11_input_failed",
+                "private X input failed",
+            )
         x11.XFlush(display)
         x11.XCloseDisplay(display)
         display = None
         return child.wait()
+    except AdapterFailure as error:
+        raise error.with_stderr(capture)
     finally:
         if display is not None:
             x11.XCloseDisplay(display)
         if child is not None and child.poll() is None:
             child.terminate()
             child.wait()
-        if xvfb.poll() is None:
-            xvfb.terminate()
-            try:
-                xvfb.wait(timeout=2)
-            except subprocess.TimeoutExpired:
-                xvfb.kill()
-                xvfb.wait()
+        stop_process(xvfb)
+        capture.close()
 
 
-if __name__ == "__main__":
+def main() -> int:
+    args = parse_args()
     try:
-        raise SystemExit(main())
+        if args.mount_private_x11:
+            mount_private_x11()
+        if args.probe:
+            emit_diagnostic(run_probe(args))
+            return 0
+        return run_launch(args)
     except AdapterFailure as error:
-        print(str(error), file=os.sys.stderr)
-        raise SystemExit(error.code) from None
+        emit_diagnostic(diagnostic(args, "probe" if args.probe else "launch", "error", error))
+        return error.code
+    except Exception:
+        failure = AdapterFailure(
+            PRIVATE_DISPLAY_UNAVAILABLE,
+            "unexpected_failure",
+            "private display adapter failed",
+        )
+        emit_diagnostic(diagnostic(args, "probe" if args.probe else "launch", "error", failure))
+        return failure.code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/prepack.mjs

@@ -0,0 +1,55 @@
+import { spawnSync } from "node:child_process";
+import { access } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+const pathExists = async (path) => {
+  try {
+    await access(path);
+    return true;
+  } catch (cause) {
+    if (
+      typeof cause === "object" &&
+      cause !== null &&
+      "code" in cause &&
+      cause.code === "ENOENT"
+    )
+      return false;
+    throw cause;
+  }
+};
+
+const run = (command, args) => {
+  const result = spawnSync(command, args, {
+    cwd: root,
+    encoding: "utf8",
+    maxBuffer: 4 * 1_024 * 1_024,
+  });
+  if (result.error !== undefined) throw result.error;
+  if (result.status !== 0) {
+    if (result.stdout.length > 0) process.stdout.write(result.stdout);
+    if (result.stderr.length > 0) process.stderr.write(result.stderr);
+    throw new Error(
+      `Prepack command failed with status ${String(result.status)}: ${command} ${args.join(" ")}`,
+    );
+  }
+};
+
+if (await pathExists(join(root, "src"))) {
+  const packageManagerPath = process.env.npm_execpath;
+  if (packageManagerPath === undefined)
+    throw new Error("Prepack could not resolve the active package manager");
+  run(process.execPath, [packageManagerPath, "run", "build:cached"]);
+  run(process.execPath, ["scripts/generate-skill-metadata.mjs", "--check"]);
+} else {
+  for (const path of [
+    "dist/main.js",
+    "dist/mcpDoctor.js",
+    "dist/cli.js",
+    "dist/cliOutput.js",
+    "skills/reverse-engineer-anything/SKILL.md",
+  ])
+    await access(join(root, path));
+}