deskpilot 0.2.0__py3-none-any.whl

deskpilot/__init__.py

@@ -0,0 +1 @@
+"""Desktop control for agents on Hyprland."""

deskpilot/cli.py

@@ -0,0 +1,173 @@
+"""deskpilot: drive Hyprland desktop windows from an agent.
+
+Coordinates for move, click and scroll are pixels in the window's last `deskpilot shot`, so read the screenshot,
+pick a point, click it. Pass --scale 1 to give window pixels instead.
+
+Clicks, scrolls, and keys and text sent to a --window run inside the compositor through the deskpilot Hyprland plugin
+(`deskpilot plugin load`), so the user's pointer and keyboard focus stay where they are. --take-pointer and
+--take-focus move the user's own pointer or focus instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .desk import EXPECTED, Desk
+
+
+def _print(data, as_json: bool) -> None:
+    if as_json:
+        print(json.dumps(data, indent=2))
+    elif isinstance(data, list):
+        for row in data:
+            print(row if isinstance(row, str) else json.dumps(row))
+    else:
+        print(data if isinstance(data, str) else json.dumps(data))
+
+
+SHOW_HELP = "bring a window that is not on screen into view by focusing it, instead of refusing"
+
+
+def _point_parser(sub, name: str, helptext: str) -> argparse.ArgumentParser:
+    """A command that acts at a point of a window's screenshot."""
+    s = sub.add_parser(name, help=helptext)
+    s.add_argument("window")
+    s.add_argument("x", type=float)
+    s.add_argument("y", type=float)
+    s.add_argument("--scale", type=float, help="the scale the point is in; 1 for window pixels")
+    s.add_argument("--shot", help="the shot id the point is in (default: the window's latest shot)")
+    s.add_argument("--show", action="store_true", help=SHOW_HELP)
+    s.add_argument(
+        "--take-pointer", action="store_true", help="move the user's own pointer there instead of leaving it in place"
+    )
+    return s
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="deskpilot", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    p.add_argument("--json", action="store_true", help="print JSON")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("windows", help="list mapped windows with geometry")
+
+    s = sub.add_parser("focus", help="focus a window")
+    s.add_argument("window")
+
+    s = sub.add_parser("shot", help="screenshot a window; later clicks use its pixels")
+    s.add_argument("window")
+    s.add_argument("-o", "--out", type=Path)
+    s.add_argument("--max-width", type=int, default=1400, help="scale down wider windows to this width")
+    s.add_argument("--show", action="store_true", help=SHOW_HELP)
+
+    _point_parser(sub, "move", "show the agent cursor at a point")
+    s = _point_parser(sub, "click", "click a point")
+    s.add_argument("--button", choices=["left", "right", "middle"], default="left")
+    s.add_argument("--double", action="store_true")
+    s = _point_parser(sub, "scroll", "scroll at a point; positive is up")
+    s.add_argument("dy", type=int)
+
+    s = sub.add_parser("plugin", help="build, load or check the deskpilot Hyprland plugin")
+    s.add_argument("action", choices=["build", "load", "status"])
+
+    focus_help = "focus the window and press through the input device instead of leaving focus in place"
+    s = sub.add_parser("key", help="press a key or chord, e.g. Return, ctrl+a, alt+F4")
+    s.add_argument("combo")
+    s.add_argument("--window", help="press it in this window; without it the keys go to the focused window")
+    s.add_argument("--take-focus", action="store_true", help=focus_help)
+
+    s = sub.add_parser("type", help="type text (US layout)")
+    s.add_argument("text")
+    s.add_argument("--window", help="type into this window; without it the keys go to the focused window")
+    s.add_argument("--take-focus", action="store_true", help=focus_help)
+    s.add_argument(
+        "--delay", type=float, default=0.03, help="seconds each key is held and between keys; games want 0.1 or more"
+    )
+
+    s = sub.add_parser("wait-for", help="wait for a window to appear (or --gone to close)")
+    s.add_argument("window")
+    s.add_argument("--timeout", type=float, default=60)
+    s.add_argument("--gone", action="store_true")
+
+    sub.add_parser("cursor", help="print the cursor's global position")
+    sub.add_parser("mcp", help="run the MCP server on stdio")
+    return p
+
+
+def run(args: argparse.Namespace, desk: Desk) -> object:
+    match args.cmd:
+        case "windows":
+            return [w.to_dict() for w in desk.windows()]
+        case "focus":
+            return desk.focus(args.window).to_dict()
+        case "shot":
+            return desk.shot(args.window, args.out, args.max_width, args.show).to_dict()
+        case "move":
+            gx, gy = desk.move(
+                args.window, args.x, args.y, args.scale, args.take_pointer, shot=args.shot, show=args.show
+            )
+            return {"x": gx, "y": gy}
+        case "click":
+            count = 2 if args.double else 1
+            gx, gy = desk.click(
+                args.window,
+                args.x,
+                args.y,
+                args.button,
+                count,
+                args.scale,
+                args.take_pointer,
+                shot=args.shot,
+                show=args.show,
+            )
+            return {"x": gx, "y": gy}
+        case "scroll":
+            gx, gy = desk.scroll(
+                args.window, args.x, args.y, args.dy, args.scale, args.take_pointer, shot=args.shot, show=args.show
+            )
+            return {"x": gx, "y": gy}
+        case "key":
+            desk.key(args.combo, args.window, args.take_focus)
+            return "ok"
+        case "type":
+            desk.type(args.text, args.window, args.delay, args.take_focus)
+            return "ok"
+        case "wait-for":
+            win = desk.wait_for(args.window, args.timeout, args.gone)
+            return win.to_dict() if win else "gone"
+        case "plugin":
+            if args.action == "build":
+                return {"path": str(desk.plugin.build())}
+            if args.action == "load":
+                return {"path": str(desk.plugin.load()), **desk.plugin.status()}
+            return desk.plugin.status()
+        case "cursor":
+            x, y = desk.hypr.cursor()
+            return {"x": x, "y": y}
+    raise AssertionError(args.cmd)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.cmd == "mcp":
+        from .mcp_server import serve
+
+        serve()
+        return 0
+    desk = Desk()
+    try:
+        _print(run(args, desk), args.json)
+    except EXPECTED as error:
+        print(f"deskpilot: {error}", file=sys.stderr)
+        return 2
+    finally:
+        desk.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

deskpilot/desk.py

@@ -0,0 +1,221 @@
+"""The actions an agent takes, shared by the CLI and the MCP server.
+
+Clicks, scrolls and keys for a named window go through the deskpilot Hyprland plugin, which leaves the user's
+pointer and keyboard focus where they were; the agent cursor overlay shows where clicks land. `take_pointer` and
+`take_focus` instead move the user's own pointer or focus and press through the uinput device.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+from . import keys, screen
+from .hypr import AmbiguousWindow, HyprctlError, Hyprland, Window, WindowNotFound
+from .overlay import AgentCursor
+from .plugin import Plugin, PluginError
+
+
+class NotVisible(LookupError):
+    pass
+
+
+# Failures an agent should read and act on: a refusal, a missing window, a tool that failed. Anything else is a bug.
+EXPECTED = (
+    WindowNotFound,
+    AmbiguousWindow,
+    NotVisible,
+    HyprctlError,
+    PluginError,
+    ValueError,
+    TimeoutError,
+    RuntimeError,
+)
+
+# How long `show` waits for a focused window to come on screen: a workspace switch or a scrolling layout's
+# animation has to finish before the window's area is its own.
+SHOW_TIMEOUT = 3.0
+
+
+class Desk:
+    def __init__(self, hypr: Hyprland | None = None, input_factory=None, plugin=None, cursor=None):
+        self.hypr = hypr or Hyprland()
+        self.plugin = plugin or Plugin()
+        self.cursor = cursor or AgentCursor()
+        self._input_factory = input_factory
+        self._input = None
+
+    @property
+    def input(self):
+        if self._input is None:
+            if self._input_factory is None:
+                from .uinput import VirtualInput
+
+                self._input_factory = VirtualInput
+            self._input = self._input_factory()
+        return self._input
+
+    def close(self) -> None:
+        self.cursor.close()
+        if self._input is not None:
+            self._input.close()
+            self._input = None
+
+    def windows(self) -> list[Window]:
+        return self.hypr.windows()
+
+    def window(self, query: str) -> Window:
+        return self.hypr.find(query)
+
+    def focus(self, query: str) -> Window:
+        win = self.window(query)
+        self.hypr.focus(win)
+        return win
+
+    def _on_screen(self, win: Window, show: bool) -> Window:
+        """The window, on screen: a screenshot or a click reaches only what is showing. A hidden window is an error
+        unless `show`, which focuses it, since that switches the user's workspace or scrolls their layout."""
+        if win.visible:
+            return win
+        if not show:
+            raise NotVisible(
+                f"{win.cls} ({win.address}) is not on screen: its workspace {win.workspace} is not showing or it is "
+                "scrolled off its monitor. Pass --show (show over MCP) to bring it into view, which focuses it"
+            )
+        self.hypr.focus(win)
+        deadline = time.monotonic() + SHOW_TIMEOUT
+        while time.monotonic() < deadline:
+            win = self.window(win.address)
+            if win.visible:
+                return win
+            time.sleep(0.05)
+        raise NotVisible(f"{win.cls} ({win.address}) did not come on screen within {SHOW_TIMEOUT}s of focusing it")
+
+    def shot(
+        self, query: str, out: Path | None = None, max_width: int = screen.DEFAULT_MAX_WIDTH, show: bool = False
+    ) -> screen.Shot:
+        return screen.capture(self._on_screen(self.window(query), show), out, max_width)
+
+    def point(
+        self,
+        query: str,
+        x: float,
+        y: float,
+        scale: float | None = None,
+        shot: str | None = None,
+        show: bool = False,
+    ) -> tuple[Window, int, int]:
+        """Resolve a point to a global position. The point is in pixels of `shot`, or of the window's latest shot,
+        or at `scale` (1 for window pixels); with no shot at all it is in window pixels."""
+        win = self._on_screen(self.window(query), show)
+        if scale is None:
+            taken = screen.load(shot) if shot else screen.latest(win.address)
+            if taken:
+                taken.check(win)
+            scale = taken.scale if taken else 1.0
+        gx, gy = screen.to_global(win, x, y, scale)
+        return win, gx, gy
+
+    def _take_pointer(self, win: Window, gx: int, gy: int) -> None:
+        self.hypr.focus(win)
+        self.hypr.move_cursor(gx, gy)
+        time.sleep(0.15)
+        self.input.nudge()
+
+    def move(
+        self,
+        query: str,
+        x: float,
+        y: float,
+        scale: float | None = None,
+        take_pointer: bool = False,
+        *,
+        shot: str | None = None,
+        show: bool = False,
+    ) -> tuple[int, int]:
+        """Show the agent cursor at a point. take_pointer also moves the user's pointer, for a hover."""
+        win, gx, gy = self.point(query, x, y, scale, shot, show)
+        self.cursor.glide(gx, gy)
+        if take_pointer:
+            self._take_pointer(win, gx, gy)
+        return gx, gy
+
+    def click(
+        self,
+        query: str,
+        x: float,
+        y: float,
+        button: str = "left",
+        count: int = 1,
+        scale: float | None = None,
+        take_pointer: bool = False,
+        *,
+        shot: str | None = None,
+        show: bool = False,
+    ) -> tuple[int, int]:
+        win, gx, gy = self.point(query, x, y, scale, shot, show)
+        self.cursor.glide(gx, gy)
+        if take_pointer:
+            self._take_pointer(win, gx, gy)
+            self.input.click(button, count)
+        else:
+            self.plugin.click(win.address, gx, gy, button, count)
+        return gx, gy
+
+    def scroll(
+        self,
+        query: str,
+        x: float,
+        y: float,
+        dy: int,
+        scale: float | None = None,
+        take_pointer: bool = False,
+        *,
+        shot: str | None = None,
+        show: bool = False,
+    ) -> tuple[int, int]:
+        win, gx, gy = self.point(query, x, y, scale, shot, show)
+        self.cursor.glide(gx, gy)
+        if take_pointer:
+            self._take_pointer(win, gx, gy)
+            self.input.scroll(dy=dy)
+        else:
+            self.plugin.scroll(win.address, gx, gy, dy)
+        return gx, gy
+
+    def key(self, combo: str, query: str | None = None, take_focus: bool = False) -> None:
+        """Press a chord in a window without focusing it, or with take_focus focus it and press through uinput.
+        With no window the keys go to whatever has focus."""
+        if query and not take_focus:
+            self.plugin.keys(self.window(query).address, keys.chord_events(combo))
+            return
+        if query:
+            self.focus(query)
+            time.sleep(0.1)
+        self.input.chord(combo)
+
+    def type(self, text: str, query: str | None = None, delay: float = 0.03, take_focus: bool = False) -> None:
+        if query and not take_focus:
+            events = [event for code, shift in keys.text_strokes(text) for event in keys.stroke_events(code, shift)]
+            self.plugin.keys(self.window(query).address, events, hold=delay, gap=delay)
+            return
+        if query:
+            self.focus(query)
+            time.sleep(0.1)
+        self.input.type_text(text, delay)
+
+    def wait_for(self, query: str, timeout: float = 60, gone: bool = False) -> Window | None:
+        """Wait until a window matching `query` exists (or, with gone, no longer does)."""
+        deadline = time.monotonic() + timeout
+        while True:
+            try:
+                win = self.window(query)
+                if not gone:
+                    return win
+            except WindowNotFound:
+                if gone:
+                    return None
+            if time.monotonic() >= deadline:
+                state = "still open" if gone else "did not appear"
+                raise TimeoutError(f"window '{query}' {state} within {timeout:g}s")
+            time.sleep(0.5)

deskpilot/hypr.py

@@ -0,0 +1,222 @@
+"""Hyprland through hyprctl: windows, monitors, focus and the cursor.
+
+Hyprland 0.57 replaced the string dispatchers (`movecursor 10 20`) with Lua ones (`hl.dsp.cursor.move({x=10,y=20})`).
+Each dispatcher here tries the Lua form first and falls back to the string form, so both versions work.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import socket
+import subprocess
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+Runner = Callable[[list[str]], str]
+
+
+class HyprctlError(RuntimeError):
+    pass
+
+
+def socket_alive(path: Path) -> bool:
+    """Whether a Hyprland control socket accepts a connection. A crashed or ended session leaves its file behind."""
+    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+        sock.settimeout(1)
+        try:
+            sock.connect(str(path))
+        except OSError:
+            return False
+    return True
+
+
+def pick_instance(signature: str | None, hypr_dir: Path, alive: Callable[[Path], bool] = socket_alive) -> str:
+    """The Hyprland instance to talk to: the environment's, unless its socket is dead, then the only live one.
+
+    A shell started before Hyprland restarted keeps the old session's HYPRLAND_INSTANCE_SIGNATURE, and hyprctl then
+    fails on every call. Several live instances are an error naming them, never a guess.
+    """
+    if signature and alive(hypr_dir / signature / ".socket.sock"):
+        return signature
+    folders = sorted(p.name for p in hypr_dir.iterdir() if p.is_dir()) if hypr_dir.is_dir() else []
+    live = [name for name in folders if alive(hypr_dir / name / ".socket.sock")]
+    stale = f"HYPRLAND_INSTANCE_SIGNATURE={signature} has no live socket. " if signature else ""
+    if len(live) == 1:
+        return live[0]
+    if not live:
+        raise HyprctlError(f"{stale}No running Hyprland instance under {hypr_dir}.")
+    raise HyprctlError(
+        f"{stale}Several Hyprland instances are running, set HYPRLAND_INSTANCE_SIGNATURE to one: {', '.join(live)}"
+    )
+
+
+def current_instance() -> str:
+    """Chosen per call, not once: a long-lived MCP server outlives a Hyprland restart. It costs one connect."""
+    runtime = os.environ.get("XDG_RUNTIME_DIR") or f"/run/user/{os.getuid()}"
+    return pick_instance(os.environ.get("HYPRLAND_INSTANCE_SIGNATURE"), Path(runtime) / "hypr")
+
+
+def run_hyprctl(args: list[str]) -> str:
+    cmd = ["hyprctl", "-i", current_instance(), *args]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=10)
+    if result.returncode != 0:
+        detail = (result.stderr.strip() or result.stdout.strip() or "no output").splitlines()[0]
+        raise HyprctlError(f"hyprctl {' '.join(args)} exited {result.returncode}: {detail}")
+    return result.stdout
+
+
+@dataclass(frozen=True)
+class Window:
+    address: str
+    cls: str
+    title: str
+    x: int
+    y: int
+    width: int
+    height: int
+    monitor: int
+    workspace: str
+    xwayland: bool
+    pid: int
+    focused: bool = False
+    # The ext-foreign-toplevel identifier, which `grim -T` captures by. Empty on a Hyprland too old to report it.
+    stable_id: str = ""
+    # Some of the window lies on a monitor showing its workspace. False for a hidden workspace and for a column a
+    # scrolling layout has moved off every monitor.
+    visible: bool = True
+
+    def to_dict(self) -> dict:
+        return {
+            "address": self.address,
+            "class": self.cls,
+            "title": self.title,
+            "x": self.x,
+            "y": self.y,
+            "width": self.width,
+            "height": self.height,
+            "monitor": self.monitor,
+            "workspace": self.workspace,
+            "xwayland": self.xwayland,
+            "visible": self.visible,
+            "pid": self.pid,
+            "focused": self.focused,
+        }
+
+
+@dataclass(frozen=True)
+class Monitor:
+    """A monitor's area in global logical pixels, and the workspaces it shows."""
+
+    x: int
+    y: int
+    width: int
+    height: int
+    showing: frozenset[str]
+
+    @classmethod
+    def from_json(cls, m: dict) -> Monitor:
+        scale = m.get("scale") or 1
+        width, height = round(m["width"] / scale), round(m["height"] / scale)
+        if m.get("transform", 0) % 2:  # rotated a quarter turn, flipped or not
+            width, height = height, width
+        # A special workspace covers the monitor's normal one while it is open.
+        special = str((m.get("specialWorkspace") or {}).get("name") or "")
+        shown = special or str((m.get("activeWorkspace") or {}).get("name", ""))
+        return cls(m["x"], m["y"], width, height, frozenset({shown}))
+
+    def shows(self, workspace: str, x: int, y: int, width: int, height: int) -> bool:
+        overlaps = x < self.x + self.width and self.x < x + width and y < self.y + self.height and self.y < y + height
+        return workspace in self.showing and overlaps
+
+
+class WindowNotFound(LookupError):
+    pass
+
+
+class AmbiguousWindow(LookupError):
+    pass
+
+
+class Hyprland:
+    def __init__(self, runner: Runner = run_hyprctl):
+        self._run = runner
+
+    def _json(self, *args: str):
+        out = self._run(["-j", *args])
+        return json.loads(out) if out.strip() else None
+
+    def windows(self) -> list[Window]:
+        active = self._json("activewindow") or {}
+        monitors = [Monitor.from_json(m) for m in self._json("monitors") or [] if not m.get("disabled")]
+        found = []
+        for c in self._json("clients") or []:
+            if not c.get("mapped", True) or c.get("hidden"):
+                continue
+            x, y = c["at"]
+            w, h = c["size"]
+            workspace = str((c.get("workspace") or {}).get("name", ""))
+            found.append(
+                Window(
+                    address=c["address"],
+                    cls=c.get("class", ""),
+                    title=c.get("title", ""),
+                    x=x,
+                    y=y,
+                    width=w,
+                    height=h,
+                    monitor=c.get("monitor", -1),
+                    workspace=workspace,
+                    xwayland=bool(c.get("xwayland")),
+                    pid=c.get("pid", 0),
+                    focused=c["address"] == active.get("address"),
+                    stable_id=str(c.get("stableId") or ""),
+                    visible=any(m.shows(workspace, x, y, w, h) for m in monitors),
+                )
+            )
+        return found
+
+    def find(self, query: str) -> Window:
+        return find_window(self.windows(), query)
+
+    def cursor(self) -> tuple[int, int]:
+        pos = self._json("cursorpos") or {}
+        return int(pos.get("x", 0)), int(pos.get("y", 0))
+
+    def _dispatch(self, lua: str, legacy: str) -> None:
+        """Run the Lua dispatcher, or the string one on Hyprland before 0.57. Either failing exits non-zero."""
+        try:
+            if self._run(["dispatch", lua]).strip() == "ok":
+                return
+        except HyprctlError:
+            pass
+        out = self._run(["dispatch", *legacy.split(" ", 1)])
+        if out.strip() != "ok":
+            raise HyprctlError(f"hyprctl dispatch {legacy} failed: {out.strip()}")
+
+    def move_cursor(self, gx: int, gy: int) -> None:
+        self._dispatch(f"hl.dsp.cursor.move({{x={gx},y={gy}}})", f"movecursor {gx} {gy}")
+
+    def focus(self, window: Window) -> None:
+        self._dispatch(f'hl.dsp.focus({{window="address:{window.address}"}})', f"focuswindow address:{window.address}")
+
+
+def find_window(windows: list[Window], query: str) -> Window:
+    """Match an address exactly, then a class exactly, then a substring of class or title, all case-insensitive.
+
+    A query that matches more than one window at the same level is an error naming them, never a guess.
+    """
+    q = query.lower()
+    levels = [
+        [w for w in windows if w.address.lower() == q or w.address.lower() == f"0x{q}"],
+        [w for w in windows if w.cls.lower() == q],
+        [w for w in windows if q in w.cls.lower() or q in w.title.lower()],
+    ]
+    for matches in levels:
+        if len(matches) == 1:
+            return matches[0]
+        if len(matches) > 1:
+            names = "; ".join(f"{w.address} {w.cls} '{w.title[:40]}'" for w in matches)
+            raise AmbiguousWindow(f"'{query}' matches {len(matches)} windows, use an address: {names}")
+    raise WindowNotFound(f"no window matches '{query}'")