VirtualDesktop 1.6.5__py3-none-win_amd64.whl

VirtualDesktop/_desktop_runtime.py

@@ -0,0 +1,465 @@
+"""Runs **inside** a private desktop and answers commands from the outside.
+
+Started by :class:`VirtualDesktop.agent.AgentSession`; never started by hand.
+
+A private desktop is a real isolation boundary: from the outside, UI Automation returns
+``ElementNotAvailable`` and ``PrintWindow`` yields no pixels, so any real work has to execute *on*
+that desktop.  This script is the "inside half" of that arrangement.
+
+**Implemented with ``ctypes`` only.** No pywinauto, no pywin32: this library's promise is that a
+plain ``pip install`` is enough, and a runtime that needs an extra package would break it.  (An
+earlier revision imported pywinauto and failed inside the desktop with ``ModuleNotFoundError``.)
+Anyone who wants pywinauto can still have it — send ``exec`` with their own code, which runs in
+this process.
+
+Protocol: two files in a control directory (handles and pipes cannot cross a desktop boundary).
+
+======================  ==========================================================
+``info``                ``hwnd`` — title, class, rect, child count, button labels
+``text``                ``hwnd`` — window title
+``click``               ``hwnd`` + ``text`` (label substring) or ``index``
+``keys``                ``hwnd`` + ``text`` — WM_CHAR per character
+``screenshot``          ``hwnd`` + ``path`` — PrintWindow to a file on the host side
+``windows``             — every titled window on this desktop
+``exec``                ``code`` — arbitrary Python here, stdout returned
+``stop``                — exit
+======================  ==========================================================
+"""
+
+from __future__ import annotations
+
+import ctypes
+import ctypes.wintypes as wintypes
+import io
+import json
+import sys
+import time
+import traceback
+from contextlib import redirect_stdout
+from pathlib import Path
+
+# This process runs on a private desktop with a working directory that has nothing to do with the
+# caller's project, so ``import VirtualDesktop`` would fail (measured: ModuleNotFoundError) unless the
+# package root is put on the path explicitly.  The file lives inside the package, so both it and its
+# parent are known without relying on the environment — the same bootstrapping ``_helper.py`` does.
+_PACKAGE_DIR = Path(__file__).resolve().parent
+for _candidate in (str(_PACKAGE_DIR.parent), str(_PACKAGE_DIR)):
+    if _candidate not in sys.path:
+        sys.path.insert(0, _candidate)
+
+# -- Win32 -----------------------------------------------------------------------------
+
+_user32 = ctypes.WinDLL("user32", use_last_error=True)
+_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+
+_BM_CLICK = 0x00F5
+_WM_CHAR = 0x0102
+_GW_CHILD = 5
+_GW_HWNDNEXT = 2
+_MAX_TEXT = 512
+
+
+class _RECT(ctypes.Structure):
+    _fields_ = [
+        ("left", wintypes.LONG),
+        ("top", wintypes.LONG),
+        ("right", wintypes.LONG),
+        ("bottom", wintypes.LONG),
+    ]
+
+
+_user32.SendMessageW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM]
+_user32.SendMessageW.restype = wintypes.LPARAM
+_user32.GetWindowRect.argtypes = [wintypes.HWND, ctypes.POINTER(_RECT)]
+_user32.GetWindowRect.restype = wintypes.BOOL
+_user32.GetWindowTextW.argtypes = [wintypes.HWND, ctypes.c_wchar_p, ctypes.c_int]
+_user32.GetWindowTextW.restype = ctypes.c_int
+_user32.GetClassNameW.argtypes = [wintypes.HWND, ctypes.c_wchar_p, ctypes.c_int]
+_user32.GetClassNameW.restype = ctypes.c_int
+_user32.GetWindow.argtypes = [wintypes.HWND, wintypes.UINT]
+_user32.GetWindow.restype = wintypes.HWND
+_user32.IsWindow.argtypes = [wintypes.HWND]
+_user32.IsWindow.restype = wintypes.BOOL
+_user32.IsWindowVisible.argtypes = [wintypes.HWND]
+_user32.IsWindowVisible.restype = wintypes.BOOL
+_EnumProc = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM)
+_user32.EnumWindows.argtypes = [_EnumProc, wintypes.LPARAM]
+_user32.EnumWindows.restype = wintypes.BOOL
+
+
+def _title(hwnd: int) -> str:
+    buffer = ctypes.create_unicode_buffer(_MAX_TEXT)
+    _user32.GetWindowTextW(wintypes.HWND(hwnd), buffer, _MAX_TEXT)
+    return buffer.value
+
+
+def _class_name(hwnd: int) -> str:
+    buffer = ctypes.create_unicode_buffer(_MAX_TEXT)
+    _user32.GetClassNameW(wintypes.HWND(hwnd), buffer, _MAX_TEXT)
+    return buffer.value
+
+
+def _rect(hwnd: int) -> list:
+    rect = _RECT()
+    if not _user32.GetWindowRect(wintypes.HWND(hwnd), ctypes.byref(rect)):
+        return [0, 0, 0, 0]
+    return [rect.left, rect.top, rect.right, rect.bottom]
+
+
+def _children(hwnd: int, depth: int = 2) -> list:
+    """Direct children (and grandchildren): handle, class, text, rect."""
+    found = []
+    child = _user32.GetWindow(wintypes.HWND(hwnd), _GW_CHILD)
+    while child:
+        entry = {
+            "hwnd": hex(int(child)),
+            "class": _class_name(int(child)),
+            "text": _title(int(child)),
+            "rect": _rect(int(child)),
+            "visible": bool(_user32.IsWindowVisible(child)),
+        }
+        if depth > 1:
+            entry["children"] = _children(int(child), depth - 1)
+        found.append(entry)
+        child = _user32.GetWindow(child, _GW_HWNDNEXT)
+    return found
+
+
+def _button_descendants(hwnd: int) -> list:
+    """Every button below ``hwnd``, flattened (depth-first)."""
+    buttons = []
+    for child in _children(hwnd, depth=3):
+        if child["class"] == "Button":
+            buttons.append(child)
+        buttons.extend(
+            b for b in _flatten(child.get("children", [])) if b["class"] == "Button"
+        )
+    return buttons
+
+
+def _flatten(nodes: list) -> list:
+    out = []
+    for node in nodes:
+        out.append(node)
+        out.extend(_flatten(node.get("children", [])))
+    return out
+
+
+# -- commands --------------------------------------------------------------------------
+
+
+def _describe(hwnd: int) -> dict:
+    buttons = _button_descendants(hwnd)
+    rect = _rect(hwnd)
+    return {
+        "title": _title(hwnd),
+        "class": _class_name(hwnd),
+        "rect": rect,
+        "size": [rect[2] - rect[0], rect[3] - rect[1]],
+        "child_count": len(_children(hwnd, depth=3)),
+        "buttons": [b["text"] for b in buttons],
+    }
+
+
+def _click(hwnd: int, *, text=None, index: int = 0) -> dict:
+    """Click a button by label substring or position.
+
+    ``BM_CLICK`` rather than a real mouse event: on a private desktop the window is never
+    ``WS_VISIBLE`` and there is no active desktop to move a cursor on, so a message is the only
+    mechanism that works.
+    """
+    buttons = _button_descendants(hwnd)
+    if not buttons:
+        return {"ok": False, "error": "the window has no buttons", "class": _class_name(hwnd)}
+    if text is not None:
+        needle = str(text)
+        picked = next((b for b in buttons if needle in (b["text"] or "")), None)
+        if picked is None:
+            return {"ok": False, "error": f"no button matching {needle!r}",
+                    "buttons": [b["text"] for b in buttons]}
+    else:
+        if not 0 <= index < len(buttons):
+            return {"ok": False, "error": f"button index {index} out of range",
+                    "buttons": [b["text"] for b in buttons]}
+        picked = buttons[index]
+    _user32.SendMessageW(wintypes.HWND(int(picked["hwnd"], 16)), _BM_CLICK, 0, 0)
+    return {"ok": True, "clicked": picked["text"]}
+
+
+def _keys(hwnd: int, text: str) -> dict:
+    for character in text:
+        _user32.SendMessageW(wintypes.HWND(hwnd), _WM_CHAR, ord(character), 0)
+    return {"ok": True, "sent": len(text)}
+
+
+def _screenshot(hwnd: int, path: str) -> dict:
+    """Refused on purpose: a private desktop's windows are never composited.
+
+    Measured inside a private desktop on Windows 11: ``printwindow`` (both flag variants) and
+    ``wm_print`` all return variance 0, and ``bitblt`` fails outright.  Answering with a blank file
+    would be worse than refusing, because the caller cannot tell "black window" from "not supported".
+    This command exists only so the failure carries an explanation.
+    """
+    return {
+        "ok": False,
+        "error": "windows on a private desktop cannot be captured; there are no pixels to read",
+        "hint": "Use the shared workspace (VirtualWorkspace.SHARED) for screenshots, or read state "
+        "with the 'info' command.",
+    }
+
+
+def _windows() -> dict:
+    from VirtualDesktop import list_windows
+
+    return {
+        "ok": True,
+        "windows": [
+            {"hwnd": hex(r.hwnd), "title": r.title, "class": r.class_name, "rect": list(r.rect)}
+            for r in list_windows()
+            if r.title
+        ],
+    }
+
+
+def _exec(code: str) -> dict:
+    """Run Python here and return what it printed."""
+    buffer = io.StringIO()
+    with redirect_stdout(buffer):
+        exec(compile(code, "<agent>", "exec"), {"__name__": "__desktop_agent__"})
+    return {"ok": True, "stdout": buffer.getvalue()}
+
+
+def _uia_tree(hwnd: int, *, depth: int = 8, max_nodes: int = 500, view: str = "RawView") -> dict:
+    """Walk the UI Automation tree of ``hwnd`` **from inside this desktop**.
+
+    Uses ``ElementFromHandle`` rather than ``GetRootElement``: the root element belongs to the
+    *input* desktop, so from a private desktop it yields a single node and none of the application's
+    windows.  An element obtained from a handle is not restricted that way (measured: root -> 1
+    node, ``FromHandle`` -> the window's real subtree).
+
+    Running inside the desktop is the whole point: a UIA client elsewhere resolves elements against
+    the desktop it is attached to, so an outside dump returns nothing for a private desktop.
+    """
+    try:
+        import comtypes.client
+
+        # ``comtypes.gen.UIAutomationClient`` is generated on first use from the type library, so
+        # importing it directly fails on a machine that has never spoken to UIA (measured: a fresh
+        # venv dies with "cannot import name 'UIAutomationClient' from 'comtypes.gen'").  GetModule
+        # generates it if needed, and is what makes this work out of the box.
+        comtypes.client.GetModule("UIAutomationCore.dll")
+        from comtypes.gen import UIAutomationClient as UIA
+    except Exception as exc:  # noqa: BLE001 - comtypes is optional
+        return {
+            "ok": True,
+            "tree": {
+                "error": f"UI Automation unavailable inside the desktop: {type(exc).__name__}: {exc}",
+                "nodes": [],
+            },
+        }
+
+    automation = comtypes.client.CreateObject(UIA.CUIAutomation, interface=UIA.IUIAutomation)
+    root = automation.ElementFromHandle(hwnd)
+    if root is None:
+        return {"ok": True, "tree": {"error": "ElementFromHandle returned null", "nodes": []}}
+    walker = {
+        "RawView": automation.RawViewWalker,
+        "ControlView": automation.ControlViewWalker,
+        "ContentView": automation.ContentViewWalker,
+    }.get(view, automation.RawViewWalker)
+
+    nodes: list = []
+
+    def visit(element, level: int) -> None:
+        if level > depth or len(nodes) >= max_nodes:
+            return
+        try:
+            box = element.CurrentBoundingRectangle
+            nodes.append(
+                {
+                    "depth": level,
+                    "name": element.CurrentName or "",
+                    "control": str(element.CurrentControlType),
+                    "type_id": int(element.CurrentControlType),
+                    "automation": element.CurrentAutomationId or "",
+                    "class": element.CurrentClassName or "",
+                    "handle": int(element.CurrentNativeWindowHandle or 0),
+                    "x": int(box.left),
+                    "y": int(box.top),
+                    "w": int(box.right - box.left),
+                    "h": int(box.bottom - box.top),
+                    "enabled": bool(element.CurrentIsEnabled),
+                    "offscreen": bool(element.CurrentIsOffscreen),
+                    "focusable": bool(element.CurrentIsKeyboardFocusable),
+                }
+            )
+        except Exception:  # noqa: BLE001 - one dying element must not end the walk
+            return
+        try:
+            child = walker.GetFirstChildElement(element)
+        except Exception:  # noqa: BLE001
+            return
+        while child is not None and len(nodes) < max_nodes:
+            visit(child, level + 1)
+            try:
+                child = walker.GetNextSiblingElement(child)
+            except Exception:  # noqa: BLE001
+                break
+
+    visit(root, 0)
+    return {"ok": True, "tree": {"view": view, "count": len(nodes), "nodes": nodes}}
+
+
+def _capture(hwnd: int, path: str, *, client_only=None, methods=None, require_content=None) -> dict:
+    """Capture ``hwnd`` and write it to ``path``, **from inside this desktop**.
+
+    The capture has to happen here for the same reason the UIA walk does: ``IsWindow`` and
+    ``GetWindowRect`` are scoped to the calling thread's desktop and report
+    ``ERROR_INVALID_WINDOW_HANDLE`` for a window on another desktop, so the capture engine cannot
+    even validate a cross-desktop handle.  Run inside, the handle is ordinary and every backend
+    works — ``PrintWindow`` asks the owning process to paint itself, which needs no screen.
+
+    A window that was never shown still answers ``PrintWindow`` successfully and paints a solid
+    rectangle, so a blank frame is reported as an error rather than saved as a real screenshot.
+    """
+    from VirtualDesktop.capture import CaptureEngine
+
+    engine = CaptureEngine(methods=tuple(methods)) if methods else CaptureEngine()
+    engine.ensure_dpi_aware()
+
+    # A window that was just created may not have painted yet, and a window created with SW_HIDE
+    # never paints at all.  Retry briefly so the first case succeeds and the second still gets a
+    # clear "blank" answer instead of a generic failure.
+    #
+    # ``require_content=False`` here on purpose: the engine's own validation raises a generic
+    # CaptureError for a blank frame, which looks identical to a real backend failure.  Taking the
+    # frame and judging it ourselves is what makes "the window was never shown" reportable.
+    deadline = time.monotonic() + 6.0
+    last_error = ""
+    last_stats: dict = {}
+    last_method = ""
+    last_size: list = []
+    while True:
+        try:
+            frame = engine.capture(
+                hwnd,
+                client_only=client_only,
+                require_content=False,
+                save_to=path,
+            )
+        except Exception as exc:  # noqa: BLE001 - retried below, then reported
+            last_error = f"{type(exc).__name__}: {exc}"
+            if time.monotonic() >= deadline:
+                return {"ok": False, "error": last_error}
+            time.sleep(0.3)
+            continue
+
+        blank = bool(frame.analysis.blank) if frame.analysis is not None else False
+        last_stats = frame.analysis.to_dict() if frame.analysis is not None else {}
+        last_method = frame.method
+        last_size = list(frame.size)
+        if not blank:
+            return {
+                "ok": True,
+                "saved": path,
+                "method": frame.method,
+                "size": list(frame.size),
+                "blank": False,
+                "analysis": last_stats,
+            }
+        if time.monotonic() >= deadline:
+            return {
+                "ok": False,
+                "error": "the captured frame is blank; the window was probably never shown",
+                "hint": "A window created with SW_HIDE does not paint itself. Launch it with "
+                "show=True (agent.launch does this by default), or call ShowWindow(SW_SHOW) first.",
+                "blank": True,
+                "method": last_method,
+                "size": last_size,
+                "analysis": last_stats,
+            }
+        time.sleep(0.3)
+
+
+def _handle(request: dict) -> dict:
+    if not _kernel32.GetCurrentProcessId():  # pragma: no cover - keeps kernel32 referenced
+        pass
+    command = str(request.get("command", "")).lower()
+    if command == "info":
+        return {"ok": True, "window": _describe(int(request["hwnd"], 0))}
+    if command == "text":
+        return {"ok": True, "text": _title(int(request["hwnd"], 0))}
+    if command == "click":
+        return _click(int(request["hwnd"], 0), text=request.get("text"),
+                      index=int(request.get("index", 0)))
+    if command == "keys":
+        return _keys(int(request["hwnd"], 0), str(request.get("text", "")))
+    if command == "screenshot":
+        return _screenshot(int(request["hwnd"], 0), str(request["path"]))
+    if command == "capture":
+        return _capture(
+            int(request["hwnd"], 0),
+            str(request["path"]),
+            client_only=request.get("client_only"),
+            methods=request.get("methods"),
+            require_content=request.get("require_content"),
+        )
+    if command == "windows":
+        return _windows()
+    if command == "uia_tree":
+        return _uia_tree(
+            int(request["hwnd"], 0),
+            depth=int(request.get("depth", 8)),
+            max_nodes=int(request.get("max_nodes", 500)),
+            view=str(request.get("view", "RawView")),
+        )
+    if command == "exec":
+        return _exec(str(request.get("code", "")))
+    return {"ok": False, "error": f"unknown command {command!r}"}
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print("usage: desktop_runtime.py <control-dir>", file=sys.stderr)
+        return 2
+    control = Path(sys.argv[1])
+    request_file = control / "request.json"
+    response_file = control / "response.json"
+    control.mkdir(parents=True, exist_ok=True)
+    (control / "ready").write_text("1", encoding="utf-8")
+
+    while True:
+        if not request_file.is_file():
+            time.sleep(0.05)
+            continue
+        try:
+            request = json.loads(request_file.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001 - a partial write is not fatal
+            response_file.write_text(
+                json.dumps({"ok": False, "error": f"unreadable request: {exc}"}), encoding="utf-8"
+            )
+            request_file.unlink(missing_ok=True)
+            continue
+
+        if str(request.get("command", "")).lower() == "stop":
+            response_file.write_text(json.dumps({"ok": True, "stopping": True}), encoding="utf-8")
+            request_file.unlink(missing_ok=True)
+            return 0
+
+        try:
+            response = _handle(request)
+        except Exception as exc:  # noqa: BLE001 - errors travel back as data
+            response = {
+                "ok": False,
+                "error": f"{type(exc).__name__}: {exc}",
+                "traceback": traceback.format_exc()[-1500:],
+            }
+        # Write the response before acknowledging, so the host never sees a missing answer.
+        response_file.write_text(
+            json.dumps(response, ensure_ascii=False, default=str), encoding="utf-8"
+        )
+        request_file.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())