vadgr-computer-use 0.1.0__py3-none-any.whl

computer_use/__init__.py

@@ -0,0 +1,50 @@
+"""Computer Use Engine -- gives LLM agents eyes and hands.
+
+Library mode (agent calls engine):
+    from computer_use import ComputerUseEngine
+    engine = ComputerUseEngine()
+    screen = engine.screenshot()
+    engine.click(500, 300)
+    engine.type_text("hello")
+
+Autonomous mode (engine calls LLM):
+    from computer_use import ComputerUseEngine
+    engine = ComputerUseEngine(provider="anthropic")
+    results = engine.run_task("Open Notepad and type hello")
+"""
+
+from computer_use.core.engine import ComputerUseEngine
+from computer_use.core.errors import (
+    ActionError,
+    ActionTimeoutError,
+    ComputerUseError,
+    ConfigError,
+    PlatformNotSupportedError,
+    ProviderError,
+    ScreenCaptureError,
+)
+from computer_use.core.types import (
+    Action,
+    ActionType,
+    Platform,
+    Region,
+    ScreenState,
+    StepResult,
+)
+
+__all__ = [
+    "ComputerUseEngine",
+    "Action",
+    "ActionType",
+    "Platform",
+    "Region",
+    "ScreenState",
+    "StepResult",
+    "ComputerUseError",
+    "ScreenCaptureError",
+    "ActionError",
+    "ActionTimeoutError",
+    "ProviderError",
+    "ConfigError",
+    "PlatformNotSupportedError",
+]

computer_use/__main__.py

@@ -0,0 +1,134 @@
+"""CLI entry point for autonomous mode.
+
+Usage:
+    python -m computer_use "Open Chrome and search for Vadgr"
+    python -m computer_use --provider anthropic "Open Notepad"
+    python -m computer_use --config custom.yaml "Click the Start menu"
+    python -m computer_use --info  # Show platform info
+"""
+
+import argparse
+import logging
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Computer Use Engine -- autonomous desktop control."
+    )
+    parser.add_argument(
+        "task",
+        nargs="?",
+        help="Natural-language task description.",
+    )
+    parser.add_argument(
+        "--provider",
+        default=None,
+        help="LLM provider (anthropic, openai).",
+    )
+    parser.add_argument(
+        "--config",
+        default=None,
+        help="Path to config.yaml.",
+    )
+    parser.add_argument(
+        "--max-steps",
+        type=int,
+        default=50,
+        help="Maximum steps before stopping.",
+    )
+    parser.add_argument(
+        "--no-verify",
+        action="store_true",
+        help="Skip action verification.",
+    )
+    parser.add_argument(
+        "--info",
+        action="store_true",
+        help="Show platform info and exit.",
+    )
+    parser.add_argument(
+        "--screenshot",
+        type=str,
+        default=None,
+        metavar="PATH",
+        help="Take a screenshot and save to PATH.",
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Enable debug logging.",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+    )
+
+    from computer_use.core.engine import ComputerUseEngine
+
+    # Info mode
+    if args.info:
+        engine = ComputerUseEngine(config_path=args.config)
+        info = engine.get_platform_info()
+        print(f"Platform: {info['platform']}")
+        print(f"Backend available: {info['backend_available']}")
+        print(f"Accessibility API: {info['accessibility']['api_name']}")
+        print(f"Accessibility available: {info['accessibility']['available']}")
+        if info["accessibility"].get("notes"):
+            print(f"Notes: {info['accessibility']['notes']}")
+        screen_w, screen_h = engine.get_screen_size()
+        print(f"Screen size: {screen_w}x{screen_h}")
+        return
+
+    # Screenshot mode
+    if args.screenshot:
+        engine = ComputerUseEngine(config_path=args.config)
+        screen = engine.screenshot()
+        with open(args.screenshot, "wb") as f:
+            f.write(screen.image_bytes)
+        print(
+            f"Screenshot saved to {args.screenshot} "
+            f"({screen.width}x{screen.height})"
+        )
+        return
+
+    # Task mode (autonomous)
+    if not args.task:
+        parser.error("A task description is required (or use --info / --screenshot)")
+
+    provider = args.provider or "anthropic"
+    engine = ComputerUseEngine(
+        config_path=args.config,
+        provider=provider,
+    )
+
+    print(f"Platform: {engine.get_platform().value}")
+    print(f"Provider: {provider}")
+    print(f"Task: {args.task}")
+    print(f"Max steps: {args.max_steps}")
+    print("---")
+
+    results = engine.run_task(
+        task=args.task,
+        max_steps=args.max_steps,
+        verify=not args.no_verify,
+    )
+
+    print(f"\nCompleted {len(results)} steps.")
+    for i, r in enumerate(results, 1):
+        status = "OK" if r.success else "FAILED"
+        action = r.action_taken.action_type.value
+        reason = r.reasoning[:80] if r.reasoning else ""
+        print(f"  Step {i}: [{status}] {action} -- {reason}")
+        if r.error:
+            print(f"          Error: {r.error}")
+
+    success_count = sum(1 for r in results if r.success)
+    print(f"\n{success_count}/{len(results)} steps succeeded.")
+    sys.exit(0 if all(r.success for r in results) else 1)
+
+
+if __name__ == "__main__":
+    main()

computer_use/bridge/actions.py

@@ -0,0 +1,76 @@
+"""Action execution via the bridge daemon."""
+
+import logging
+
+from computer_use.bridge.client import BridgeClient, BridgeError
+from computer_use.core.actions import ActionExecutor
+from computer_use.core.errors import ActionError
+
+logger = logging.getLogger("computer_use.bridge.actions")
+
+
+class BridgeActionExecutor(ActionExecutor):
+    """Implements ActionExecutor by delegating to the bridge daemon over TCP.
+
+    Accepts an optional *fallback* ActionExecutor (typically WSL2ActionExecutor).
+    When the bridge raises an error for text-input methods (type_text, key_press),
+    the fallback executor is tried before propagating the failure.
+    """
+
+    def __init__(self, client: BridgeClient, fallback: ActionExecutor | None = None):
+        self._client = client
+        self._fallback = fallback
+
+    def _act(self, method: str, params: dict) -> None:
+        try:
+            self._client.call(method, params, timeout=10.0)
+        except BridgeError as e:
+            raise ActionError(f"Bridge {method} failed: {e}") from e
+
+    def move_mouse(self, x: int, y: int) -> None:
+        self._act("move_mouse", {"x": x, "y": y})
+
+    def click(self, x: int, y: int, button: str = "left") -> None:
+        self._act("click", {"x": x, "y": y, "button": button})
+
+    def double_click(self, x: int, y: int) -> None:
+        self._act("double_click", {"x": x, "y": y})
+
+    def type_text(self, text: str) -> None:
+        try:
+            self._act("type_text", {"text": text})
+        except ActionError:
+            if self._fallback is not None:
+                logger.warning("Bridge type_text failed, using PowerShell fallback")
+                self._fallback.type_text(text)
+            else:
+                raise
+
+    def key_press(self, keys: list[str]) -> None:
+        try:
+            self._act("key_press", {"keys": keys})
+        except ActionError:
+            if self._fallback is not None:
+                logger.warning("Bridge key_press failed, using PowerShell fallback")
+                self._fallback.key_press(keys)
+            else:
+                raise
+
+    def scroll(self, x: int, y: int, amount: int) -> None:
+        self._act("scroll", {"x": x, "y": y, "amount": amount})
+
+    def drag(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration: float = 0.5,
+    ) -> None:
+        self._act("drag", {
+            "start_x": start_x,
+            "start_y": start_y,
+            "end_x": end_x,
+            "end_y": end_y,
+            "duration": duration,
+        })

computer_use/bridge/capture.py

@@ -0,0 +1,70 @@
+"""Screenshot capture via the bridge daemon."""
+
+import base64
+
+from computer_use.bridge.client import BridgeClient, BridgeError
+from computer_use.core.errors import ScreenCaptureError
+from computer_use.core.screenshot import ScreenCapture
+from computer_use.core.types import Region, ScreenState
+
+
+class BridgeScreenCapture(ScreenCapture):
+    """Implements ScreenCapture by delegating to the bridge daemon over TCP."""
+
+    def __init__(self, client: BridgeClient, quality: int = 85):
+        self._client = client
+        self._quality = quality
+
+    def capture_full(self) -> ScreenState:
+        try:
+            result = self._client.call(
+                "screenshot_full", {"quality": self._quality}, timeout=5.0
+            )
+        except BridgeError as e:
+            raise ScreenCaptureError(f"Bridge screenshot failed: {e}") from e
+
+        return ScreenState(
+            image_bytes=base64.b64decode(result["image_b64"]),
+            width=result["width"],
+            height=result["height"],
+            scale_factor=result.get("scale_factor", 1.0),
+            offset_x=result.get("offset_x", 0),
+            offset_y=result.get("offset_y", 0),
+        )
+
+    def capture_region(self, region: Region) -> ScreenState:
+        try:
+            result = self._client.call(
+                "screenshot_region",
+                {
+                    "x": region.x,
+                    "y": region.y,
+                    "width": region.width,
+                    "height": region.height,
+                    "quality": self._quality,
+                },
+                timeout=5.0,
+            )
+        except BridgeError as e:
+            raise ScreenCaptureError(f"Bridge region screenshot failed: {e}") from e
+
+        return ScreenState(
+            image_bytes=base64.b64decode(result["image_b64"]),
+            width=result["width"],
+            height=result["height"],
+            scale_factor=result.get("scale_factor", 1.0),
+        )
+
+    def get_screen_size(self) -> tuple[int, int]:
+        try:
+            result = self._client.call("screen_size", timeout=3.0)
+            return (result["width"], result["height"])
+        except BridgeError as e:
+            raise ScreenCaptureError(f"Bridge screen_size failed: {e}") from e
+
+    def get_scale_factor(self) -> float:
+        try:
+            result = self._client.call("scale_factor", timeout=3.0)
+            return result.get("factor", 1.0)
+        except BridgeError:
+            return 1.0

computer_use/bridge/client.py

@@ -0,0 +1,187 @@
+"""TCP client for the Windows bridge daemon."""
+
+import json
+import logging
+import os
+import platform
+import socket
+import subprocess
+import threading
+from pathlib import Path
+
+from computer_use.bridge.protocol import (
+    HEADER_SIZE,
+    decode_header,
+    encode_message,
+    get_port,
+    make_request,
+)
+
+logger = logging.getLogger("computer_use.bridge.client")
+
+
+def _is_wsl2_mirrored() -> bool:
+    """Check if WSL2 is using mirrored networking mode.
+
+    With mirrored networking, WSL2 shares the host's network stack so
+    localhost reaches Windows directly -- no need for the gateway IP.
+    """
+    try:
+        wslconfig = Path("/mnt/c/Users") / os.environ.get("USER", "") / ".wslconfig"
+        if not wslconfig.exists():
+            # Try via powershell
+            result = subprocess.run(
+                ["powershell.exe", "-NoProfile", "-Command", "$env:USERPROFILE"],
+                capture_output=True, text=True, timeout=3,
+            )
+            if result.returncode == 0:
+                win_profile = result.stdout.strip()
+                wsl_path = subprocess.run(
+                    ["wslpath", "-u", win_profile],
+                    capture_output=True, text=True,
+                ).stdout.strip()
+                wslconfig = Path(wsl_path) / ".wslconfig"
+
+        if wslconfig.exists():
+            content = wslconfig.read_text().lower()
+            return "networkingmode=mirrored" in content.replace(" ", "")
+    except Exception:
+        pass
+    return False
+
+
+def _detect_windows_host() -> str:
+    """Auto-detect the Windows host IP when running inside WSL2.
+
+    On standard WSL2, 127.0.0.1 points to the Linux VM, not Windows.
+    The Windows host IP is the default gateway (WSL2 vEthernet adapter).
+
+    On mirrored networking (networkingMode=mirrored in .wslconfig),
+    localhost reaches Windows directly, so 127.0.0.1 works.
+
+    Returns '127.0.0.1' on non-WSL2 platforms.
+    """
+    try:
+        if "microsoft" in platform.release().lower():
+            if _is_wsl2_mirrored():
+                logger.debug("WSL2 mirrored networking, using 127.0.0.1")
+                return "127.0.0.1"
+
+            # Default gateway is the Windows host on the WSL2 virtual network
+            with open("/proc/net/route") as f:
+                for line in f:
+                    fields = line.strip().split()
+                    if fields[1] == "00000000":  # default route
+                        # Gateway is in hex, little-endian
+                        gw_hex = fields[2]
+                        gw_bytes = bytes.fromhex(gw_hex)
+                        ip = f"{gw_bytes[3]}.{gw_bytes[2]}.{gw_bytes[1]}.{gw_bytes[0]}"
+                        logger.debug("WSL2 detected, Windows host IP: %s", ip)
+                        return ip
+    except Exception:
+        pass
+    return "127.0.0.1"
+
+
+class BridgeError(Exception):
+    pass
+
+
+class BridgeClient:
+    """Connects to the Windows bridge daemon over TCP.
+
+    On WSL2, auto-discovers the Windows host IP.
+    Thread-safe. Reconnects automatically on socket errors (one retry).
+    """
+
+    def __init__(self, host: str | None = None, port: int | None = None):
+        self._host = host or _detect_windows_host()
+        self._port = port or get_port()
+        self._sock: socket.socket | None = None
+        self._lock = threading.Lock()
+
+    def is_available(self) -> bool:
+        try:
+            result = self.call("ping", timeout=2.0)
+            return result.get("pong", False)
+        except Exception:
+            return False
+
+    def handshake(self) -> dict | None:
+        """Return the raw `ping` response dict, or None if the daemon is unreachable.
+
+        Separate from `is_available()` because callers (supervisor) need the
+        full response to read `version_hash` for drift detection. A running
+        daemon returns at least `{"pong": True}`; absent fields are treated
+        as drift by the supervisor.
+        """
+        try:
+            return self.call("ping", timeout=2.0)
+        except Exception:
+            return None
+
+    def call(self, method: str, params: dict | None = None, timeout: float = 10.0) -> dict:
+        with self._lock:
+            return self._call_locked(method, params, timeout, retry=True)
+
+    def _call_locked(
+        self, method: str, params: dict | None, timeout: float, retry: bool
+    ) -> dict:
+        try:
+            self._ensure_connected(timeout)
+            request = make_request(method, params)
+            self._send(encode_message(request))
+            response = self._receive(timeout)
+            if not response.get("ok", False):
+                raise BridgeError(response.get("error", "Unknown daemon error"))
+            return response.get("result", {})
+        except (socket.error, OSError, ConnectionError) as e:
+            self._close_socket()
+            if retry:
+                logger.debug("Connection lost, retrying: %s", e)
+                return self._call_locked(method, params, timeout, retry=False)
+            raise BridgeError(f"Bridge connection failed: {e}") from e
+
+    def _ensure_connected(self, timeout: float) -> None:
+        if self._sock is not None:
+            return
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(timeout)
+        sock.connect((self._host, self._port))
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        self._sock = sock
+        logger.debug("Connected to bridge at %s:%d", self._host, self._port)
+
+    def _send(self, data: bytes) -> None:
+        self._sock.sendall(data)
+
+    def _receive(self, timeout: float) -> dict:
+        self._sock.settimeout(timeout)
+        header = self._recv_exact(HEADER_SIZE)
+        length = decode_header(header)
+        payload = self._recv_exact(length)
+        return json.loads(payload)
+
+    def _recv_exact(self, n: int) -> bytes:
+        buf = bytearray()
+        while len(buf) < n:
+            chunk = self._sock.recv(n - len(buf))
+            if not chunk:
+                raise ConnectionError("Bridge daemon closed connection")
+            buf.extend(chunk)
+        return bytes(buf)
+
+    def _close_socket(self) -> None:
+        if self._sock is not None:
+            try:
+                self._sock.close()
+            except OSError:
+                pass
+            self._sock = None
+
+    def close(self) -> None:
+        with self._lock:
+            self._close_socket()
+
+    def __del__(self):
+        self._close_socket()