use-computer-cli 0.1.0__py3-none-any.whl

use_computer/coordinates.py

@@ -0,0 +1,104 @@
+"""Coordinate spaces and the conversion between them.
+
+A screenshot on a HiDPI display is larger than the space the operating system clicks in. An
+unhandled factor of two makes every click land in the wrong place, and nothing about the
+failure looks like a scaling bug -- so every coordinate carries the space it belongs to, and a
+conversion whose ratio is unknown raises instead of guessing.
+
+This module is pure: it knows nothing about backends.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from use_computer.errors import CoordinateSpaceError
+
+#: Relative disagreement tolerated between the horizontal and vertical ratios before the two
+#: reported screen sizes are called inconsistent.
+SCALE_TOLERANCE = 0.02
+
+
+class CoordinateSpace(str, Enum):
+    """The space a coordinate is expressed in."""
+
+    SCREENSHOT = "screenshot"
+    """Pixels of the captured image -- what ui-locator returns, because it looked at the image."""
+
+    ACTUATION = "actuation"
+    """The units the backend moves the pointer in."""
+
+
+class Coordinate(BaseModel):
+    """A point, together with the space it belongs to.
+
+    A bare pair of numbers is not a coordinate; the space is part of the value.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    x: int
+    y: int
+    space: CoordinateSpace
+
+
+class ScreenInfo(BaseModel):
+    """What a backend reports about its own screen."""
+
+    model_config = ConfigDict(frozen=True)
+
+    width: int = Field(description="Screen width in actuation units.")
+    height: int = Field(description="Screen height in actuation units.")
+    screenshot_width: int = Field(description="Screenshot width in pixels.")
+    screenshot_height: int = Field(description="Screenshot height in pixels.")
+    scale: float | None = Field(
+        default=None,
+        description=(
+            "Screenshot pixels per actuation unit. None means unknown -- conversion refuses."
+        ),
+    )
+
+    def size(self, space: CoordinateSpace) -> tuple[int, int]:
+        if space is CoordinateSpace.SCREENSHOT:
+            return self.screenshot_width, self.screenshot_height
+        return self.width, self.height
+
+
+def derive_scale(
+    width: int, height: int, screenshot_width: int, screenshot_height: int
+) -> float | None:
+    """Derive the screenshot/actuation ratio from two reported sizes.
+
+    Returns ``None`` when it cannot be derived -- a zero dimension, or horizontal and vertical
+    ratios that disagree by more than :data:`SCALE_TOLERANCE`. An inconsistent pair is not
+    averaged into a plausible-looking number; it is reported as unknown.
+    """
+    if width <= 0 or height <= 0 or screenshot_width <= 0 or screenshot_height <= 0:
+        return None
+    ratio_x = screenshot_width / width
+    ratio_y = screenshot_height / height
+    if abs(ratio_x - ratio_y) > SCALE_TOLERANCE * max(ratio_x, ratio_y):
+        return None
+    return (ratio_x + ratio_y) / 2
+
+
+def convert(coord: Coordinate, target: CoordinateSpace, screen: ScreenInfo) -> Coordinate:
+    """Convert ``coord`` into ``target`` space.
+
+    Raises:
+        CoordinateSpaceError: when the scale is unknown, so the conversion would be a guess.
+    """
+    if coord.space is target:
+        return coord
+    if screen.scale is None or screen.scale <= 0:
+        raise CoordinateSpaceError(
+            f"cannot convert {coord.x},{coord.y} from {coord.space.value} to {target.value}: "
+            f"the backend reports {screen.width}x{screen.height} actuation units and "
+            f"{screen.screenshot_width}x{screen.screenshot_height} screenshot pixels, from which "
+            "no consistent scale can be derived. Capture a screenshot to establish the ratio, or "
+            "set `scale` explicitly in the profile."
+        )
+    factor = 1 / screen.scale if coord.space is CoordinateSpace.SCREENSHOT else screen.scale
+    return Coordinate(x=round(coord.x * factor), y=round(coord.y * factor), space=target)

use_computer/errors.py

@@ -0,0 +1,53 @@
+"""Exception hierarchy.
+
+Every error carries what the caller needs to fix it: the extra to install, the OS permission
+to grant, the config key to set. A message that only says "failed" costs a debugging session.
+"""
+
+from __future__ import annotations
+
+
+class UseComputerError(Exception):
+    """Base class for every error raised by use-computer."""
+
+
+class BackendNotAvailableError(UseComputerError):
+    """A backend's optional dependency is not installed."""
+
+    def __init__(self, backend: str, extra: str, missing: str) -> None:
+        self.backend = backend
+        self.extra = extra
+        self.missing = missing
+        super().__init__(
+            f"backend {backend!r} needs {missing!r}, which is not installed. "
+            f'Install it with: pip install "use-computer-cli[{extra}]"'
+        )
+
+
+class PermissionDeniedError(UseComputerError):
+    """The host denied a permission the backend needs.
+
+    Without this error the underlying libraries typically do nothing at all, and a click that
+    never happened is reported as a click that worked.
+    """
+
+    def __init__(self, permission: str, hint: str) -> None:
+        self.permission = permission
+        self.hint = hint
+        super().__init__(f"{permission} permission denied. {hint}")
+
+
+class CoordinateSpaceError(UseComputerError):
+    """A coordinate cannot be converted because the scale is unknown or inconsistent."""
+
+
+class KeySyntaxError(UseComputerError):
+    """A key combination could not be parsed."""
+
+
+class ConfigError(UseComputerError):
+    """Configuration is missing, malformed, or forbids the requested operation."""
+
+
+class ActionFailedError(UseComputerError):
+    """The backend failed to perform an action."""

use_computer/keys.py

@@ -0,0 +1,209 @@
+"""One key-name syntax, normalised per backend.
+
+One spelling of a shortcut must work on both backends. This module owns the canonical
+vocabulary and one mapping table per backend, both keyed by the same canonical names -- so a
+missing entry is a visible hole rather than a divergence between backends.
+
+This module is pure: the tables hold strings, never imported symbols, so it stays importable
+with no extras installed.
+"""
+
+from __future__ import annotations
+
+import difflib
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict
+
+from use_computer.errors import KeySyntaxError
+
+#: Modifier aliases resolved to one canonical name.
+MODIFIER_ALIASES: Final[dict[str, str]] = {
+    "ctrl": "ctrl",
+    "control": "ctrl",
+    "alt": "alt",
+    "option": "alt",
+    "opt": "alt",
+    "shift": "shift",
+    "cmd": "cmd",
+    "command": "cmd",
+    "super": "cmd",
+    "win": "cmd",
+    "meta": "cmd",
+}
+
+#: Canonical modifiers, in the order they are pressed.
+MODIFIER_ORDER: Final[tuple[str, ...]] = ("ctrl", "alt", "shift", "cmd")
+
+#: Named-key aliases resolved to one canonical name.
+KEY_ALIASES: Final[dict[str, str]] = {
+    "return": "enter",
+    "escape": "esc",
+    "del": "delete",
+    "ins": "insert",
+    "pgup": "pageup",
+    "page_up": "pageup",
+    "pgdn": "pagedown",
+    "pagedn": "pagedown",
+    "page_down": "pagedown",
+    "spacebar": "space",
+    "capslock": "caps_lock",
+    "printscreen": "print_screen",
+}
+
+_NAMED_KEYS: Final[tuple[str, ...]] = (
+    "enter",
+    "tab",
+    "esc",
+    "space",
+    "backspace",
+    "delete",
+    "insert",
+    "home",
+    "end",
+    "pageup",
+    "pagedown",
+    "up",
+    "down",
+    "left",
+    "right",
+    "caps_lock",
+    "print_screen",
+    *(f"f{n}" for n in range(1, 25)),
+)
+
+#: Every canonical named key. Single characters are keys too, but are not enumerated.
+NAMED_KEYS: Final[frozenset[str]] = frozenset(_NAMED_KEYS)
+
+
+class KeyCombo(BaseModel):
+    """A parsed key combination, in canonical names."""
+
+    model_config = ConfigDict(frozen=True)
+
+    modifiers: tuple[str, ...] = ()
+    key: str
+
+    def __str__(self) -> str:
+        return "+".join((*self.modifiers, self.key))
+
+
+def _suggest(name: str) -> str:
+    pool = [*MODIFIER_ALIASES, *NAMED_KEYS, *KEY_ALIASES]
+    close = difflib.get_close_matches(name, pool, n=3, cutoff=0.5)
+    if not close:
+        return ""
+    return " Did you mean: " + ", ".join(close) + "?"
+
+
+def parse_combo(spec: str) -> KeyCombo:
+    """Parse ``ctrl+shift+t`` into a :class:`KeyCombo` of canonical names.
+
+    Raises:
+        KeySyntaxError: on an empty spec, a repeated modifier, more than one non-modifier key,
+            or an unknown key name. An unknown name is never passed through to the backend to
+            fail obscurely there.
+    """
+    raw = spec.strip()
+    if not raw:
+        raise KeySyntaxError("empty key combination")
+    if raw == "+":
+        return KeyCombo(key="+")
+    # "ctrl++" means ctrl plus the literal "+" key; the split leaves an empty tail behind.
+    body = raw[:-1] if raw.endswith("+") else raw
+    parts = [p.strip().lower() for p in body.split("+")]
+    if raw.endswith("+"):
+        if parts[-1] != "":
+            # A trailing separator with nothing after it, e.g. "ctrl+".
+            raise KeySyntaxError(f"malformed key combination: {spec!r}")
+        parts[-1] = "+"
+    if any(not p for p in parts):
+        raise KeySyntaxError(f"malformed key combination: {spec!r}")
+
+    modifiers: list[str] = []
+    key: str | None = None
+    for part in parts:
+        if part in MODIFIER_ALIASES:
+            modifier = MODIFIER_ALIASES[part]
+            if modifier in modifiers:
+                raise KeySyntaxError(f"repeated modifier {modifier!r} in {spec!r}")
+            modifiers.append(modifier)
+            continue
+        if key is not None:
+            raise KeySyntaxError(
+                f"more than one non-modifier key in {spec!r}: {key!r} and {part!r}"
+            )
+        key = KEY_ALIASES.get(part, part)
+        if len(key) != 1 and key not in NAMED_KEYS:
+            raise KeySyntaxError(f"unknown key name {part!r} in {spec!r}.{_suggest(part)}")
+
+    if key is None:
+        # A lone modifier is a key in its own right: `use-computer key shift`.
+        if len(modifiers) == 1:
+            return KeyCombo(key=modifiers[0])
+        raise KeySyntaxError(f"{spec!r} is only modifiers -- no key to press")
+
+    ordered = tuple(m for m in MODIFIER_ORDER if m in modifiers)
+    return KeyCombo(modifiers=ordered, key=key)
+
+
+def canonical(spec: str) -> str:
+    """Return the canonical spelling of ``spec``, so a log line is reproducible input."""
+    return str(parse_combo(spec))
+
+
+# --- Per-backend tables ----------------------------------------------------------------------
+# Both keyed by the same canonical names. Values are the *names* the backend's library uses;
+# resolving a name to a symbol happens in the backend, so this module imports nothing optional.
+
+#: canonical name -> attribute of ``pynput.keyboard.Key``.
+PYNPUT_KEYS: Final[dict[str, str]] = {
+    "ctrl": "ctrl",
+    "alt": "alt",
+    "shift": "shift",
+    "cmd": "cmd",
+    "enter": "enter",
+    "tab": "tab",
+    "esc": "esc",
+    "space": "space",
+    "backspace": "backspace",
+    "delete": "delete",
+    "insert": "insert",
+    "home": "home",
+    "end": "end",
+    "pageup": "page_up",
+    "pagedown": "page_down",
+    "up": "up",
+    "down": "down",
+    "left": "left",
+    "right": "right",
+    "caps_lock": "caps_lock",
+    "print_screen": "print_screen",
+    **{f"f{n}": f"f{n}" for n in range(1, 21)},
+}
+
+#: canonical name -> X11 keysym name understood by vncdotool.
+VNC_KEYS: Final[dict[str, str]] = {
+    "ctrl": "ctrl",
+    "alt": "alt",
+    "shift": "shift",
+    "cmd": "super",
+    "enter": "return",
+    "tab": "tab",
+    "esc": "esc",
+    "space": "space",
+    "backspace": "bsp",
+    "delete": "delete",
+    "insert": "insert",
+    "home": "home",
+    "end": "end",
+    "pageup": "pgup",
+    "pagedown": "pgdn",
+    "up": "up",
+    "down": "down",
+    "left": "left",
+    "right": "right",
+    "caps_lock": "caps_lock",
+    "print_screen": "print",
+    **{f"f{n}": f"f{n}" for n in range(1, 21)},
+}

use_computer/runner.py

@@ -0,0 +1,304 @@
+"""Executing a batch: one backend, many actions, one connection.
+
+Opening a VNC connection dominates the cost of a single action, so the backend is constructed
+once and every action runs against it. A single action invoked directly is simply a batch of
+one and returns the same shape.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Sequence
+from types import TracebackType
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from use_computer.actions import (
+    Action,
+    DoubleClickAction,
+    DragAction,
+    KeyAction,
+    MouseButton,
+    MoveAction,
+    RightClickAction,
+    ScreenshotAction,
+    ScrollAction,
+    TypeAction,
+    resolve,
+    with_default_space,
+)
+from use_computer.backends import Backend, create_backend
+from use_computer.compare import ChangeReport, Screenshot, compare
+from use_computer.config import BackendProfile, ResolvedConfig, Settings
+from use_computer.config import load as load_config
+from use_computer.coordinates import Coordinate, ScreenInfo
+from use_computer.errors import UseComputerError
+
+
+class ErrorInfo(BaseModel):
+    """A failure, as the calling agent sees it."""
+
+    model_config = ConfigDict(frozen=True)
+
+    type: str
+    message: str
+
+    @classmethod
+    def of(cls, exc: BaseException) -> ErrorInfo:
+        return cls(type=type(exc).__name__, message=str(exc))
+
+
+class ActionResult(BaseModel):
+    """What one action did. Frozen: once it has run, what it did does not change."""
+
+    model_config = ConfigDict(frozen=True)
+
+    action: Action
+    resolved: Coordinate | None = Field(
+        default=None, description="The target, in actuation units."
+    )
+    resolved_from: Coordinate | None = Field(
+        default=None, description="A drag's origin, in actuation units."
+    )
+    performed: bool
+    duration_ms: float
+    change: ChangeReport | None = None
+    screenshot: Screenshot | None = None
+    error: ErrorInfo | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+
+class RunResult(BaseModel):
+    """One invocation. This is what the CLI serialises to stdout."""
+
+    model_config = ConfigDict(frozen=True)
+
+    profile: str
+    backend: str
+    screen: ScreenInfo
+    ok: bool
+    failed_index: int | None = None
+    results: list[ActionResult] = Field(default_factory=list)
+
+
+class Session:
+    """A backend held open across a batch of actions."""
+
+    def __init__(
+        self,
+        backend: Backend,
+        *,
+        profile: str,
+        settings: Settings | None = None,
+    ) -> None:
+        self._backend = backend
+        self._profile = profile
+        self._settings = settings or Settings()
+        self._screen = backend.screen_info()
+
+    @classmethod
+    def from_profile(
+        cls, name: str | None = None, *, config: ResolvedConfig | None = None
+    ) -> Session:
+        """Open a session from a named profile in the project config."""
+        resolved = config or load_config(profile=name)
+        profile: BackendProfile = resolved.profile
+        return cls(
+            create_backend(profile), profile=profile.name, settings=resolved.settings
+        )
+
+    # --- context manager ---------------------------------------------------------------------
+
+    def __enter__(self) -> Session:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        self._backend.close()
+
+    # --- running -----------------------------------------------------------------------------
+
+    @property
+    def screen(self) -> ScreenInfo:
+        return self._screen
+
+    def run(self, actions: Sequence[Action]) -> RunResult:
+        """Perform every action in order against the open backend.
+
+        Stops at the first failure unless ``continue_on_error`` is set: a later action usually
+        depends on the state the failed one was meant to produce.
+        """
+        settings = self._settings
+        results: list[ActionResult] = []
+        failed_index: int | None = None
+        carried: Screenshot | None = None
+
+        for index, raw in enumerate(actions):
+            action = with_default_space(raw, settings.space)
+            verify = action.verify or settings.verify
+            before = carried if verify else None
+            if verify and before is None:
+                before = self._safe_screenshot()
+
+            result = self._run_one(action, before=before, verify=verify)
+            results.append(result)
+            carried = result.screenshot if verify else None
+
+            if not result.ok:
+                failed_index = index
+                if not settings.continue_on_error:
+                    break
+
+        return RunResult(
+            profile=self._profile,
+            backend=self._backend.name,
+            screen=self._screen,
+            ok=failed_index is None,
+            failed_index=failed_index,
+            results=results,
+        )
+
+    def _run_one(
+        self, action: Action, *, before: Screenshot | None, verify: bool
+    ) -> ActionResult:
+        settings = self._settings
+        started = time.perf_counter()
+        target: Coordinate | None = None
+        origin: Coordinate | None = None
+        screenshot: Screenshot | None = None
+        change: ChangeReport | None = None
+        performed = False
+        error: ErrorInfo | None = None
+
+        try:
+            target, origin = resolve(action, self._screen)
+            if settings.dry_run:
+                # Everything above ran: the profile, the scaling, the key parsing. Only the
+                # actuation is skipped.
+                pass
+            else:
+                screenshot = self._perform(action, target, origin)
+                performed = True
+
+            delay = action.delay if action.delay is not None else settings.delay
+            if performed and delay:
+                time.sleep(delay)
+
+            if verify and performed:
+                after = self._safe_screenshot()
+                if before is not None and after is not None:
+                    change = compare(before, after, settings.verify_threshold)
+                is_capture = isinstance(action, ScreenshotAction)
+                screenshot = (screenshot or after) if is_capture else after
+        except UseComputerError as exc:
+            error = ErrorInfo.of(exc)
+        except Exception as exc:  # a backend can fail in its own vocabulary
+            error = ErrorInfo.of(exc)
+
+        return ActionResult(
+            action=action,
+            resolved=target,
+            resolved_from=origin,
+            performed=performed,
+            duration_ms=round((time.perf_counter() - started) * 1000, 3),
+            change=change,
+            screenshot=screenshot,
+            error=error,
+        )
+
+    def _perform(
+        self, action: Action, target: Coordinate | None, origin: Coordinate | None
+    ) -> Screenshot | None:
+        backend = self._backend
+        x, y = (target.x, target.y) if target else (None, None)
+
+        if isinstance(action, MoveAction):
+            assert x is not None and y is not None
+            backend.move(x, y)
+        elif isinstance(action, DoubleClickAction):
+            backend.click(x, y, action.button, 2)
+        elif isinstance(action, RightClickAction):
+            backend.click(x, y, MouseButton.RIGHT, 1)
+        elif isinstance(action, DragAction):
+            assert origin is not None and target is not None
+            backend.drag(origin.x, origin.y, target.x, target.y, action.button)
+        elif isinstance(action, ScrollAction):
+            backend.scroll(action.amount, action.direction, x, y)
+        elif isinstance(action, TypeAction):
+            rate = action.rate if action.rate is not None else self._settings.typing_rate
+            backend.type_text(action.text, rate)
+        elif isinstance(action, KeyAction):
+            backend.key(action.key_combo)
+        elif isinstance(action, ScreenshotAction):
+            return self._capture(action)
+        else:  # ClickAction, and anything else positional with a button
+            backend.click(x, y, getattr(action, "button", MouseButton.LEFT), 1)
+        return None
+
+    def _capture(self, action: ScreenshotAction) -> Screenshot:
+        shot = self._backend.screenshot()
+        if action.out is not None:
+            action.out.parent.mkdir(parents=True, exist_ok=True)
+            if shot.data is not None:
+                action.out.write_bytes(shot.data)
+            shot = shot.model_copy(update={"path": action.out})
+        if not action.base64:
+            shot = shot.model_copy(update={"data": None if action.out else shot.data})
+        return shot
+
+    def _safe_screenshot(self) -> Screenshot | None:
+        """Change detection is advisory; a capture that fails must not fail the action."""
+        try:
+            return self._backend.screenshot()
+        except Exception:
+            return None
+
+
+def run_actions(
+    actions: Sequence[Action],
+    *,
+    profile: str | None = None,
+    config: ResolvedConfig | None = None,
+) -> RunResult:
+    """Open a session, run a batch, close it -- including when an action failed."""
+    session = Session.from_profile(profile, config=config)
+    try:
+        return session.run(actions)
+    finally:
+        session.close()
+
+
+def as_json(result: RunResult) -> dict[str, Any]:
+    """The run payload, with screenshot bytes rendered as base64 where they were requested."""
+    payload = result.model_dump(mode="json", exclude={"results": {"__all__": {"screenshot"}}})
+    payload["results"] = [
+        {**item, "screenshot": _screenshot_json(action.screenshot)}
+        for item, action in zip(payload["results"], result.results, strict=True)
+    ]
+    return payload
+
+
+def _screenshot_json(shot: Screenshot | None) -> dict[str, Any] | None:
+    if shot is None:
+        return None
+    payload: dict[str, Any] = {
+        "path": str(shot.path) if shot.path else None,
+        "width": shot.width,
+        "height": shot.height,
+        "space": shot.space.value,
+        "captured_at": shot.captured_at.isoformat(),
+    }
+    if shot.data is not None:
+        payload["base64"] = shot.base64()
+    return payload