use-computer-cli 0.1.0__py3-none-any.whl

use_computer/__init__.py

@@ -0,0 +1,93 @@
+"""use-computer: execute input on a screen for computer-use agents.
+
+The acting half of a pair. ui-locator answers where; this performs the action there.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from use_computer.actions import (
+    Action,
+    ClickAction,
+    DoubleClickAction,
+    DragAction,
+    KeyAction,
+    MouseButton,
+    MoveAction,
+    RightClickAction,
+    ScreenshotAction,
+    ScrollAction,
+    ScrollDirection,
+    TypeAction,
+)
+from use_computer.backends import Backend, create_backend
+from use_computer.compare import ChangeReport, Screenshot, compare
+from use_computer.config import BackendProfile, ResolvedConfig, Settings
+from use_computer.config import load as load_config
+from use_computer.coordinates import Coordinate, CoordinateSpace, ScreenInfo, convert
+from use_computer.errors import (
+    ActionFailedError,
+    BackendNotAvailableError,
+    ConfigError,
+    CoordinateSpaceError,
+    KeySyntaxError,
+    PermissionDeniedError,
+    UseComputerError,
+)
+from use_computer.keys import KeyCombo, parse_combo
+from use_computer.runner import ActionResult, ErrorInfo, RunResult, Session, run_actions
+
+#: The distribution name on PyPI, which differs from the import package: `use-computer` was
+#: already taken there. importlib.metadata is keyed by the distribution, so this is the name
+#: that must appear here.
+DISTRIBUTION = "use-computer-cli"
+
+try:
+    __version__ = version(DISTRIBUTION)
+except PackageNotFoundError:  # pragma: no cover - source checkout without an install
+    __version__ = "0.0.0"
+
+__all__ = [
+    "DISTRIBUTION",
+    "Action",
+    "ActionFailedError",
+    "ActionResult",
+    "Backend",
+    "BackendNotAvailableError",
+    "BackendProfile",
+    "ChangeReport",
+    "ClickAction",
+    "ConfigError",
+    "Coordinate",
+    "CoordinateSpace",
+    "CoordinateSpaceError",
+    "DoubleClickAction",
+    "DragAction",
+    "ErrorInfo",
+    "KeyAction",
+    "KeyCombo",
+    "KeySyntaxError",
+    "MouseButton",
+    "MoveAction",
+    "PermissionDeniedError",
+    "ResolvedConfig",
+    "RightClickAction",
+    "RunResult",
+    "ScreenInfo",
+    "Screenshot",
+    "ScreenshotAction",
+    "ScrollAction",
+    "ScrollDirection",
+    "Session",
+    "Settings",
+    "TypeAction",
+    "UseComputerError",
+    "__version__",
+    "compare",
+    "convert",
+    "create_backend",
+    "load_config",
+    "parse_combo",
+    "run_actions",
+]

use_computer/actions.py

@@ -0,0 +1,184 @@
+"""The action models, and the resolution that happens before a backend sees them.
+
+The action set is the whole surface of what use-computer does to a screen. Coordinate scaling
+and key parsing are applied here, so that everything crossing the backend boundary is already
+in actuation units and canonical key names.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
+
+from use_computer.coordinates import Coordinate, CoordinateSpace, ScreenInfo, convert
+from use_computer.keys import KeyCombo, canonical, parse_combo
+
+
+class MouseButton(str, Enum):
+    LEFT = "left"
+    RIGHT = "right"
+    MIDDLE = "middle"
+
+
+class ScrollDirection(str, Enum):
+    UP = "up"
+    DOWN = "down"
+    LEFT = "left"
+    RIGHT = "right"
+
+
+class BaseAction(BaseModel):
+    """Fields every action shares."""
+
+    delay: float | None = Field(default=None, ge=0, description="Seconds to wait afterwards.")
+    verify: bool = Field(default=False, description="Compare the screen before and after.")
+
+    @property
+    def target(self) -> Coordinate | None:
+        """The point this action acts on, or None if it has no coordinate."""
+        return None
+
+    @property
+    def origin(self) -> Coordinate | None:
+        """The point this action starts from -- only a drag has one."""
+        return None
+
+
+class _Positioned(BaseAction):
+    """An action that may carry a coordinate. Omitting it acts where the pointer already is."""
+
+    x: int | None = None
+    y: int | None = None
+    space: CoordinateSpace | None = None
+
+    @property
+    def target(self) -> Coordinate | None:
+        if self.x is None or self.y is None:
+            return None
+        space = self.space if self.space is not None else CoordinateSpace.SCREENSHOT
+        return Coordinate(x=self.x, y=self.y, space=space)
+
+
+class MoveAction(_Positioned):
+    action: Literal["move"] = "move"
+    x: int
+    y: int
+
+
+class ClickAction(_Positioned):
+    action: Literal["click"] = "click"
+    button: MouseButton = MouseButton.LEFT
+
+
+class DoubleClickAction(_Positioned):
+    action: Literal["double_click"] = "double_click"
+    button: MouseButton = MouseButton.LEFT
+
+
+class RightClickAction(_Positioned):
+    action: Literal["right_click"] = "right_click"
+
+
+class DragAction(BaseAction):
+    action: Literal["drag"] = "drag"
+    from_x: int
+    from_y: int
+    to_x: int
+    to_y: int
+    space: CoordinateSpace | None = None
+    button: MouseButton = MouseButton.LEFT
+
+    @property
+    def origin(self) -> Coordinate:
+        return Coordinate(x=self.from_x, y=self.from_y, space=self._space)
+
+    @property
+    def target(self) -> Coordinate:
+        return Coordinate(x=self.to_x, y=self.to_y, space=self._space)
+
+    @property
+    def _space(self) -> CoordinateSpace:
+        return self.space if self.space is not None else CoordinateSpace.SCREENSHOT
+
+
+class ScrollAction(_Positioned):
+    action: Literal["scroll"] = "scroll"
+    amount: int
+    direction: ScrollDirection = ScrollDirection.DOWN
+
+
+class TypeAction(BaseAction):
+    action: Literal["type"] = "type"
+    text: str
+    rate: float | None = Field(
+        default=None,
+        ge=0,
+        description="Seconds between keystrokes; None uses the configured rate.",
+    )
+
+
+class KeyAction(BaseAction):
+    action: Literal["key"] = "key"
+    combo: str
+
+    @field_validator("combo")
+    @classmethod
+    def _canonicalise(cls, value: str) -> str:
+        # Parse at construction: an unknown key name must fail before a connection is opened,
+        # and the canonical spelling is what appears in results and logs.
+        return canonical(value)
+
+    @property
+    def key_combo(self) -> KeyCombo:
+        return parse_combo(self.combo)
+
+
+class ScreenshotAction(BaseAction):
+    action: Literal["screenshot"] = "screenshot"
+    out: Path | None = None
+    base64: bool = False
+
+
+Action = Annotated[
+    MoveAction
+    | ClickAction
+    | DoubleClickAction
+    | RightClickAction
+    | DragAction
+    | ScrollAction
+    | TypeAction
+    | KeyAction
+    | ScreenshotAction,
+    Field(discriminator="action"),
+]
+
+#: Parses a batch file: a JSON array of action objects, discriminated on `action`.
+ActionListAdapter: TypeAdapter[list[Action]] = TypeAdapter(list[Action])
+
+#: Parses a single action object.
+ActionAdapter: TypeAdapter[Action] = TypeAdapter(Action)
+
+
+def with_default_space(action: Action, space: CoordinateSpace) -> Action:
+    """Fill in the coordinate space an action did not state, from configuration."""
+    if getattr(action, "space", "missing") is None:
+        return action.model_copy(update={"space": space})
+    return action
+
+
+def resolve(action: Action, screen: ScreenInfo) -> tuple[Coordinate | None, Coordinate | None]:
+    """Convert an action's coordinates into actuation units.
+
+    Returns ``(target, origin)``; either may be ``None`` when the action carries no coordinate.
+
+    Raises:
+        CoordinateSpaceError: when the scale needed for the conversion is unknown.
+    """
+    target = action.target
+    origin = action.origin
+    resolved_target = convert(target, CoordinateSpace.ACTUATION, screen) if target else None
+    resolved_origin = convert(origin, CoordinateSpace.ACTUATION, screen) if origin else None
+    return resolved_target, resolved_origin

use_computer/backends/__init__.py

@@ -0,0 +1,31 @@
+"""Backend selection.
+
+Adding a backend is a module here plus a name in the config file -- never a change to the
+action layer.
+"""
+
+from __future__ import annotations
+
+from use_computer.backends.base import Backend, BackendNotAvailableError, require
+from use_computer.config import BackendProfile
+from use_computer.errors import ConfigError
+
+__all__ = ["Backend", "BackendNotAvailableError", "create_backend", "require"]
+
+
+def create_backend(profile: BackendProfile) -> Backend:
+    """Construct the backend a profile names. The module is imported only when selected."""
+    if profile.backend == "local":
+        from use_computer.backends.local import LocalBackend
+
+        return LocalBackend(allow_local=profile.allow_local, scale=profile.scale)
+    if profile.backend == "vnc":
+        from use_computer.backends.vnc import VNCBackend
+
+        return VNCBackend(
+            host=profile.host,
+            port=profile.port,
+            password=profile.password,
+            scale=profile.scale,
+        )
+    raise ConfigError(f"unknown backend {profile.backend!r} in profile {profile.name!r}")

use_computer/backends/base.py

@@ -0,0 +1,66 @@
+"""The backend Protocol, and the lazy-import helper every backend constructs itself through.
+
+Two interchangeable backends sit behind this interface. Coordinates crossing it are always in
+*actuation* units -- conversion happens above. A backend's third-party dependency is imported
+inside its constructor, never at module import, so the package installs and `--help` works with
+no extras present.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from types import ModuleType
+from typing import Protocol, runtime_checkable
+
+from use_computer.actions import MouseButton, ScrollDirection
+from use_computer.compare import Screenshot
+from use_computer.coordinates import ScreenInfo
+from use_computer.errors import BackendNotAvailableError
+from use_computer.keys import KeyCombo
+
+
+@runtime_checkable
+class Backend(Protocol):
+    """What every backend implements. The fake backend used in tests satisfies it too."""
+
+    name: str
+
+    def screen_info(self) -> ScreenInfo:
+        """Report this backend's own coordinate spaces and screen size."""
+
+    def screenshot(self) -> Screenshot:
+        """Capture the current screen."""
+
+    def move(self, x: int, y: int) -> None: ...
+
+    def click(self, x: int | None, y: int | None, button: MouseButton, count: int) -> None: ...
+
+    def drag(
+        self, from_x: int, from_y: int, to_x: int, to_y: int, button: MouseButton
+    ) -> None: ...
+
+    def scroll(
+        self, amount: int, direction: ScrollDirection, x: int | None, y: int | None
+    ) -> None: ...
+
+    def type_text(self, text: str, rate: float) -> None: ...
+
+    def key(self, combo: KeyCombo) -> None: ...
+
+    def close(self) -> None:
+        """Release whatever the backend holds. Called even when an action failed."""
+
+
+def require(module: str, *, backend: str, extra: str) -> ModuleType:
+    """Import an optional dependency, or say exactly what to install.
+
+    Raises:
+        BackendNotAvailableError: naming the extra, not the import error.
+    """
+    try:
+        return import_module(module)
+    except ImportError as exc:
+        raise BackendNotAvailableError(backend=backend, extra=extra, missing=module) from exc
+
+
+__all__ = ["Backend", "BackendNotAvailableError", "require"]

use_computer/backends/local.py

@@ -0,0 +1,214 @@
+"""The local backend: drives the display of the machine use-computer runs on.
+
+Input via pynput, capture via mss. pyautogui is deliberately not used -- its last release,
+0.9.54, dates from 2023.
+
+This backend types on the user's own keyboard and moves their own pointer, so it refuses to act
+without an explicit opt-in, and it checks the macOS permissions that would otherwise make every
+action silently do nothing.
+"""
+
+from __future__ import annotations
+
+import io
+import sys
+import time
+from typing import Any
+
+from use_computer.actions import MouseButton, ScrollDirection
+from use_computer.backends.base import require
+from use_computer.compare import Screenshot
+from use_computer.coordinates import CoordinateSpace, ScreenInfo, derive_scale
+from use_computer.errors import ActionFailedError, ConfigError, PermissionDeniedError
+from use_computer.keys import PYNPUT_KEYS, KeyCombo
+
+#: Seconds held between press and release, and between the two clicks of a double-click.
+PRESS_HOLD = 0.02
+DOUBLE_CLICK_GAP = 0.08
+#: Steps a drag is interpolated over, so the application sees movement rather than a teleport.
+DRAG_STEPS = 24
+
+
+class LocalBackend:
+    """Drives this machine's display."""
+
+    name = "local"
+
+    def __init__(self, *, allow_local: bool = False, scale: float | None = None) -> None:
+        if not allow_local:
+            raise ConfigError(
+                "the local backend controls this machine's own keyboard and pointer and is "
+                "disabled by default. Enable it explicitly with `allow-local = true` in the "
+                "profile, or USE_COMPUTER_ALLOW_LOCAL=1."
+            )
+        self._explicit_scale = scale
+        pynput_mouse = require("pynput.mouse", backend=self.name, extra="local")
+        pynput_keyboard = require("pynput.keyboard", backend=self.name, extra="local")
+        mss = require("mss", backend=self.name, extra="local")
+
+        _check_macos_permissions()
+
+        self._mouse_module = pynput_mouse
+        self._key_module = pynput_keyboard
+        self._mouse = pynput_mouse.Controller()
+        self._keyboard = pynput_keyboard.Controller()
+        self._sct = mss.mss()
+        self._screen: ScreenInfo | None = None
+
+    # --- reporting ---------------------------------------------------------------------------
+
+    def screen_info(self) -> ScreenInfo:
+        if self._screen is not None:
+            return self._screen
+        monitor = self._sct.monitors[1]
+        width, height = int(monitor["width"]), int(monitor["height"])
+        shot = self._sct.grab(monitor)
+        # On a HiDPI display mss reports the monitor in points and grabs in pixels. That
+        # difference is the whole reason coordinates carry their space.
+        scale = self._explicit_scale
+        if scale is None:
+            scale = derive_scale(width, height, shot.width, shot.height)
+        self._screen = ScreenInfo(
+            width=width,
+            height=height,
+            screenshot_width=shot.width,
+            screenshot_height=shot.height,
+            scale=scale,
+        )
+        return self._screen
+
+    def screenshot(self) -> Screenshot:
+        monitor = self._sct.monitors[1]
+        shot = self._sct.grab(monitor)
+        image = _to_png(shot)
+        return Screenshot(
+            data=image,
+            width=shot.width,
+            height=shot.height,
+            space=CoordinateSpace.SCREENSHOT,
+        )
+
+    # --- acting ------------------------------------------------------------------------------
+
+    def move(self, x: int, y: int) -> None:
+        self._mouse.position = (x, y)
+
+    def click(self, x: int | None, y: int | None, button: MouseButton, count: int) -> None:
+        if x is not None and y is not None:
+            self.move(x, y)
+        pynput_button = self._button(button)
+        for index in range(count):
+            if index:
+                time.sleep(DOUBLE_CLICK_GAP)
+            self._mouse.press(pynput_button)
+            time.sleep(PRESS_HOLD)
+            self._mouse.release(pynput_button)
+
+    def drag(self, from_x: int, from_y: int, to_x: int, to_y: int, button: MouseButton) -> None:
+        pynput_button = self._button(button)
+        self.move(from_x, from_y)
+        self._mouse.press(pynput_button)
+        try:
+            for step in range(1, DRAG_STEPS + 1):
+                ratio = step / DRAG_STEPS
+                self.move(
+                    round(from_x + (to_x - from_x) * ratio),
+                    round(from_y + (to_y - from_y) * ratio),
+                )
+                time.sleep(PRESS_HOLD / 2)
+        finally:
+            self._mouse.release(pynput_button)
+
+    def scroll(
+        self, amount: int, direction: ScrollDirection, x: int | None, y: int | None
+    ) -> None:
+        if x is not None and y is not None:
+            self.move(x, y)
+        dx, dy = _scroll_vector(amount, direction)
+        self._mouse.scroll(dx, dy)
+
+    def type_text(self, text: str, rate: float) -> None:
+        # Typing as fast as the API allows loses characters in real applications.
+        for char in text:
+            self._keyboard.type(char)
+            if rate:
+                time.sleep(rate)
+
+    def key(self, combo: KeyCombo) -> None:
+        modifiers = [self._key(name) for name in combo.modifiers]
+        key = self._key(combo.key)
+        for modifier in modifiers:
+            self._keyboard.press(modifier)
+        try:
+            self._keyboard.press(key)
+            time.sleep(PRESS_HOLD)
+            self._keyboard.release(key)
+        finally:
+            for modifier in reversed(modifiers):
+                self._keyboard.release(modifier)
+
+    def close(self) -> None:
+        self._sct.close()
+
+    # --- mapping -----------------------------------------------------------------------------
+
+    def _button(self, button: MouseButton) -> Any:
+        return getattr(self._mouse_module.Button, button.value)
+
+    def _key(self, name: str) -> Any:
+        """Canonical key name -> pynput key. Single characters are themselves."""
+        attribute = PYNPUT_KEYS.get(name)
+        if attribute is None:
+            if len(name) == 1:
+                return name
+            raise ActionFailedError(f"the local backend has no mapping for key {name!r}")
+        return getattr(self._key_module.Key, attribute)
+
+
+def _scroll_vector(amount: int, direction: ScrollDirection) -> tuple[int, int]:
+    if direction is ScrollDirection.UP:
+        return 0, amount
+    if direction is ScrollDirection.DOWN:
+        return 0, -amount
+    if direction is ScrollDirection.LEFT:
+        return -amount, 0
+    return amount, 0
+
+
+def _to_png(shot: Any) -> bytes:
+    from PIL import Image
+
+    image = Image.frombytes("RGB", (shot.width, shot.height), shot.rgb)
+    buffer = io.BytesIO()
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def _check_macos_permissions() -> None:
+    """Fail loudly when macOS has denied what pynput and mss need.
+
+    Without the permission the libraries typically do nothing at all -- a click that never
+    happens and never errors, which is the worst possible failure for a calling agent.
+    """
+    if sys.platform != "darwin":  # pragma: no cover - platform specific
+        return
+    try:  # pragma: no cover - requires macOS
+        from ApplicationServices import AXIsProcessTrusted
+    except ImportError:  # pragma: no cover - pyobjc absent; let the action fail instead
+        return
+    if not AXIsProcessTrusted():  # pragma: no cover - requires macOS
+        raise PermissionDeniedError(
+            "Accessibility",
+            "Grant it in System Settings > Privacy & Security > Accessibility for the "
+            "terminal or application running use-computer, then start it again.",
+        )
+    try:  # pragma: no cover - requires macOS
+        from Quartz import CGPreflightScreenCaptureAccess
+    except ImportError:  # pragma: no cover
+        return
+    if not CGPreflightScreenCaptureAccess():  # pragma: no cover - requires macOS
+        raise PermissionDeniedError(
+            "Screen Recording",
+            "Grant it in System Settings > Privacy & Security > Screen Recording, then start "
+            "use-computer again.",
+        )