omnisight 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
omnisight/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """OmniSight: computer-use backend that can see and drive any Windows app window."""
omnisight/capture.py ADDED
@@ -0,0 +1,355 @@
1
+ """Window capture with several backends and automatic fallback.
2
+
3
+ Order tried for a window (unless a backend that worked before for the same
4
+ process is cached):
5
+
6
+ printwindow PrintWindow(PW_RENDERFULLCONTENT). Works for most GDI and many
7
+ DirectComposition windows (Chromium, Electron), including covered ones.
8
+ wgc Windows.Graphics.Capture. Sees GPU-rendered content (games,
9
+ UWP) and windows covered by other windows. Cannot see minimized
10
+ windows.
11
+ bitblt Classic window DC copy. Cheap; black for GPU content.
12
+ screen Crop of the desktop at the window rect. Only valid when nothing
13
+ covers the window; checked before use.
14
+
15
+ A frame that is (almost) one flat color is treated as a failed capture and the
16
+ next backend is tried.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import ctypes
22
+ import threading
23
+ import time
24
+ from ctypes import wintypes
25
+ from dataclasses import dataclass, field
26
+
27
+ import numpy as np
28
+
29
+ from . import winutil as wu
30
+
31
+ gdi32 = ctypes.WinDLL("gdi32")
32
+ user32 = wu.user32
33
+
34
+ PW_RENDERFULLCONTENT = 0x2
35
+ SRCCOPY = 0x00CC0020
36
+ DIB_RGB_COLORS = 0
37
+
38
+ user32.GetWindowDC.argtypes = [wintypes.HWND]
39
+ user32.GetWindowDC.restype = wintypes.HDC
40
+ user32.ReleaseDC.argtypes = [wintypes.HWND, wintypes.HDC]
41
+ user32.PrintWindow.argtypes = [wintypes.HWND, wintypes.HDC, wintypes.UINT]
42
+ gdi32.CreateCompatibleDC.argtypes = [wintypes.HDC]
43
+ gdi32.CreateCompatibleDC.restype = wintypes.HDC
44
+ gdi32.CreateCompatibleBitmap.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int]
45
+ gdi32.CreateCompatibleBitmap.restype = wintypes.HBITMAP
46
+ gdi32.SelectObject.argtypes = [wintypes.HDC, wintypes.HGDIOBJ]
47
+ gdi32.SelectObject.restype = wintypes.HGDIOBJ
48
+ gdi32.DeleteObject.argtypes = [wintypes.HGDIOBJ]
49
+ gdi32.DeleteDC.argtypes = [wintypes.HDC]
50
+ gdi32.BitBlt.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
51
+ wintypes.HDC, ctypes.c_int, ctypes.c_int, wintypes.DWORD]
52
+ gdi32.GetDIBits.argtypes = [wintypes.HDC, wintypes.HBITMAP, wintypes.UINT, wintypes.UINT,
53
+ ctypes.c_void_p, ctypes.c_void_p, wintypes.UINT]
54
+
55
+
56
+ class BITMAPINFOHEADER(ctypes.Structure):
57
+ _fields_ = [
58
+ ("biSize", wintypes.DWORD), ("biWidth", wintypes.LONG), ("biHeight", wintypes.LONG),
59
+ ("biPlanes", wintypes.WORD), ("biBitCount", wintypes.WORD),
60
+ ("biCompression", wintypes.DWORD), ("biSizeImage", wintypes.DWORD),
61
+ ("biXPelsPerMeter", wintypes.LONG), ("biYPelsPerMeter", wintypes.LONG),
62
+ ("biClrUsed", wintypes.DWORD), ("biClrImportant", wintypes.DWORD),
63
+ ]
64
+
65
+
66
+ class CaptureError(Exception):
67
+ pass
68
+
69
+
70
+ @dataclass
71
+ class Capture:
72
+ image: np.ndarray # H x W x 3, RGB, uint8
73
+ origin: tuple[int, int] # screen position of image pixel (0, 0)
74
+ backend: str
75
+ elapsed_ms: float
76
+ attempts: list[str] = field(default_factory=list) # "backend: why it was rejected"
77
+
78
+
79
+ def is_blank(img: np.ndarray) -> bool:
80
+ if img.size == 0:
81
+ return True
82
+ small = img[:: max(1, img.shape[0] // 64), :: max(1, img.shape[1] // 64)]
83
+ return float(small.reshape(-1, small.shape[-1]).std(axis=0).max()) < 1.5
84
+
85
+
86
+ SUSPECT_BLACK = 0.9
87
+
88
+
89
+ def black_ratio(img: np.ndarray) -> float:
90
+ small = img[:: max(1, img.shape[0] // 128), :: max(1, img.shape[1] // 128)]
91
+ return float((small.max(axis=2) < 8).mean())
92
+
93
+
94
+ def _bgra_to_rgb(buf: np.ndarray) -> np.ndarray:
95
+ return np.ascontiguousarray(buf[:, :, 2::-1])
96
+
97
+
98
+ # --------------------------------------------------------------------- GDI
99
+
100
+
101
+ def _gdi_grab(hwnd: int, use_printwindow: bool) -> tuple[np.ndarray, tuple[int, int]]:
102
+ wr = wintypes.RECT()
103
+ user32.GetWindowRect(hwnd, ctypes.byref(wr))
104
+ w, h = wr.right - wr.left, wr.bottom - wr.top
105
+ if w <= 0 or h <= 0:
106
+ raise CaptureError("window has no size")
107
+ wdc = user32.GetWindowDC(hwnd)
108
+ mdc = gdi32.CreateCompatibleDC(wdc)
109
+ bmp = gdi32.CreateCompatibleBitmap(wdc, w, h)
110
+ old = gdi32.SelectObject(mdc, bmp)
111
+ try:
112
+ if use_printwindow:
113
+ ok = user32.PrintWindow(hwnd, mdc, PW_RENDERFULLCONTENT)
114
+ else:
115
+ ok = gdi32.BitBlt(mdc, 0, 0, w, h, wdc, 0, 0, SRCCOPY)
116
+ if not ok:
117
+ raise CaptureError("call failed")
118
+ bi = BITMAPINFOHEADER(ctypes.sizeof(BITMAPINFOHEADER), w, -h, 1, 32, 0, 0, 0, 0, 0, 0)
119
+ buf = np.empty((h, w, 4), dtype=np.uint8)
120
+ if not gdi32.GetDIBits(mdc, bmp, 0, h, buf.ctypes.data, ctypes.byref(bi), DIB_RGB_COLORS):
121
+ raise CaptureError("GetDIBits failed")
122
+ finally:
123
+ gdi32.SelectObject(mdc, old)
124
+ gdi32.DeleteObject(bmp)
125
+ gdi32.DeleteDC(mdc)
126
+ user32.ReleaseDC(hwnd, wdc)
127
+ # Trim the invisible DWM resize border so the image matches the visible frame.
128
+ l, t, r, b = wu.window_rect(hwnd)
129
+ ox, oy = max(0, l - wr.left), max(0, t - wr.top)
130
+ buf = buf[oy: oy + (b - t), ox: ox + (r - l)]
131
+ return _bgra_to_rgb(buf), (l, t)
132
+
133
+
134
+ # --------------------------------------------------------------------- WGC
135
+
136
+
137
+ class _WgcSession:
138
+ """A live Windows.Graphics.Capture session for one window.
139
+
140
+ WGC only delivers a frame when the window content changes, so the newest
141
+ frame is kept and reused; repeated snapshots of a static window cost
142
+ nothing. Sessions close after IDLE_SECONDS without use.
143
+ """
144
+
145
+ def __init__(self, hwnd: int):
146
+ from windows_capture import WindowsCapture
147
+
148
+ self.hwnd = hwnd
149
+ self.frame: np.ndarray | None = None
150
+ self.frame_time = 0.0
151
+ self.closed = False
152
+ self.last_used = time.monotonic()
153
+ self.ready = threading.Event()
154
+ self._lock = threading.Lock()
155
+ # minimum_update_interval / draw_border are Windows 11 only; leave them unset.
156
+ cap = WindowsCapture(cursor_capture=False, window_hwnd=hwnd)
157
+
158
+ @cap.event
159
+ def on_frame_arrived(frame, control):
160
+ if self.closed:
161
+ control.stop()
162
+ return
163
+ img = frame.frame_buffer.copy() # native buffer is only valid inside the callback
164
+ with self._lock:
165
+ self.frame, self.frame_time = img, time.monotonic()
166
+ self.ready.set()
167
+
168
+ @cap.event
169
+ def on_closed():
170
+ self.closed = True
171
+ self.ready.set()
172
+
173
+ self.control = cap.start_free_threaded()
174
+
175
+ def latest(self, timeout: float) -> tuple[np.ndarray, float]:
176
+ self.last_used = time.monotonic()
177
+ if not self.ready.wait(timeout):
178
+ raise CaptureError("no frame within timeout")
179
+ with self._lock:
180
+ if self.frame is None:
181
+ raise CaptureError("session closed before first frame")
182
+ return self.frame, self.frame_time
183
+
184
+ def stop(self) -> None:
185
+ self.closed = True
186
+ try:
187
+ self.control.stop()
188
+ except Exception:
189
+ pass
190
+
191
+
192
+ class _WgcPool:
193
+ IDLE_SECONDS = 20.0
194
+
195
+ def __init__(self):
196
+ self.sessions: dict[int, _WgcSession] = {}
197
+ self.lock = threading.Lock()
198
+ threading.Thread(target=self._reaper, daemon=True).start()
199
+
200
+ def get(self, hwnd: int) -> _WgcSession:
201
+ with self.lock:
202
+ s = self.sessions.get(hwnd)
203
+ if s is None or s.closed:
204
+ s = self.sessions[hwnd] = _WgcSession(hwnd)
205
+ return s
206
+
207
+ def drop(self, hwnd: int) -> None:
208
+ with self.lock:
209
+ s = self.sessions.pop(hwnd, None)
210
+ if s:
211
+ s.stop()
212
+
213
+ def _reaper(self):
214
+ while True:
215
+ time.sleep(5)
216
+ now = time.monotonic()
217
+ with self.lock:
218
+ dead = [h for h, s in self.sessions.items()
219
+ if s.closed or now - s.last_used > self.IDLE_SECONDS or not wu.is_window(h)]
220
+ stale = [self.sessions.pop(h) for h in dead]
221
+ for s in stale:
222
+ s.stop()
223
+
224
+
225
+ _wgc_pool = _WgcPool()
226
+
227
+
228
+ def _wgc_grab(hwnd: int, timeout: float = 1.5) -> tuple[np.ndarray, tuple[int, int]]:
229
+ session = _wgc_pool.get(hwnd)
230
+ buf, _ = session.latest(timeout)
231
+ l, t, r, b = wu.window_rect(hwnd)
232
+ img = _bgra_to_rgb(buf)
233
+ # Frame may include the invisible border; centre-crop to the visible frame.
234
+ w, h = r - l, b - t
235
+ if img.shape[1] > w or img.shape[0] > h:
236
+ ox = max(0, (img.shape[1] - w) // 2)
237
+ img = img[: h, ox: ox + w]
238
+ return img, (l, t)
239
+
240
+
241
+ # ------------------------------------------------------------------- screen
242
+
243
+
244
+ def grab_screen_region(left: int, top: int, right: int, bottom: int) -> np.ndarray:
245
+ import mss
246
+
247
+ with mss.mss() as sct:
248
+ shot = sct.grab({"left": left, "top": top, "width": right - left, "height": bottom - top})
249
+ return _bgra_to_rgb(np.asarray(shot))
250
+
251
+
252
+ def virtual_screen() -> tuple[int, int, int, int]:
253
+ import mss
254
+
255
+ with mss.mss() as sct:
256
+ m = sct.monitors[0]
257
+ return (m["left"], m["top"], m["left"] + m["width"], m["top"] + m["height"])
258
+
259
+
260
+ def monitors() -> list[dict]:
261
+ import mss
262
+
263
+ with mss.mss() as sct:
264
+ return [dict(index=i, left=m["left"], top=m["top"], width=m["width"], height=m["height"])
265
+ for i, m in enumerate(sct.monitors[1:], start=1)]
266
+
267
+
268
+ def _unoccluded(hwnd: int) -> bool:
269
+ l, t, r, b = wu.window_rect(hwnd)
270
+ pts = [((l + r) // 2, (t + b) // 2)]
271
+ for fx in (0.1, 0.5, 0.9):
272
+ for fy in (0.1, 0.5, 0.9):
273
+ pts.append((int(l + (r - l) * fx), int(t + (b - t) * fy)))
274
+ return all(wu.root_at(x, y) == hwnd for x, y in pts)
275
+
276
+
277
+ def _screen_grab(hwnd: int) -> tuple[np.ndarray, tuple[int, int]]:
278
+ if not _unoccluded(hwnd):
279
+ raise CaptureError("window is covered by another window")
280
+ l, t, r, b = wu.window_rect(hwnd)
281
+ return grab_screen_region(l, t, r, b), (l, t)
282
+
283
+
284
+ # ------------------------------------------------------------------- driver
285
+
286
+ BACKENDS = {
287
+ "wgc": _wgc_grab,
288
+ "printwindow": lambda h: _gdi_grab(h, True),
289
+ "bitblt": lambda h: _gdi_grab(h, False),
290
+ "screen": _screen_grab,
291
+ }
292
+ # PrintWindow first: on Windows 10 a WGC session costs 150-600 ms to start and
293
+ # draws a yellow border around the window while it lives.
294
+ DEFAULT_ORDER = ["printwindow", "wgc", "bitblt", "screen"]
295
+
296
+ # exe name -> backend that last produced a usable frame
297
+ _preferred: dict[str, str] = {}
298
+
299
+
300
+ def capture_window(hwnd: int, backend: str = "auto", restore_minimized: bool = False) -> Capture:
301
+ info = wu.get_window(hwnd)
302
+ if info.capture_blocked:
303
+ raise CaptureError(
304
+ "This window has opted out of screen capture (SetWindowDisplayAffinity). "
305
+ "It is a deliberate protection, e.g. a password manager or DRM video; "
306
+ "use the element tree instead, or ask the user to read it.")
307
+ if info.minimized:
308
+ if not restore_minimized:
309
+ raise CaptureError("Window is minimized. Retry with restore_minimized=true to "
310
+ "restore it without stealing focus.")
311
+ wu.restore_no_activate(hwnd)
312
+ time.sleep(0.25)
313
+ if info.cloaked:
314
+ raise CaptureError("Window is cloaked (on another virtual desktop or suspended).")
315
+
316
+ if backend != "auto":
317
+ order = [backend]
318
+ else:
319
+ order = list(DEFAULT_ORDER)
320
+ pref = _preferred.get(info.process)
321
+ if pref in order:
322
+ order.remove(pref)
323
+ order.insert(0, pref)
324
+
325
+ attempts: list[str] = []
326
+ t0 = time.perf_counter()
327
+ suspects: list[tuple[float, Capture]] = []
328
+ for name in order:
329
+ try:
330
+ img, origin = BACKENDS[name](hwnd)
331
+ except Exception as e: # noqa: BLE001 - every backend failure is just "try the next one"
332
+ attempts.append(f"{name}: {e}")
333
+ if name == "wgc":
334
+ _wgc_pool.drop(hwnd)
335
+ continue
336
+ cap = Capture(img, origin, name, (time.perf_counter() - t0) * 1000, attempts)
337
+ black = black_ratio(img)
338
+ if is_blank(img) or black > SUSPECT_BLACK:
339
+ # GPU content read through GDI comes back black, often with only the
340
+ # title bar drawn. Keep it as a fallback; maybe the window is just dark.
341
+ attempts.append(f"{name}: {'blank' if is_blank(img) else f'{black:.0%} black'} frame")
342
+ suspects.append((1.0 if is_blank(img) else black, cap))
343
+ continue
344
+ _preferred[info.process] = name
345
+ return cap
346
+ if suspects:
347
+ return min(suspects, key=lambda s: s[0])[1]
348
+ raise CaptureError("all capture backends failed: " + "; ".join(attempts))
349
+
350
+
351
+ def capture_screen(region: tuple[int, int, int, int] | None = None) -> Capture:
352
+ t0 = time.perf_counter()
353
+ region = region or virtual_screen()
354
+ img = grab_screen_region(*region)
355
+ return Capture(img, (region[0], region[1]), "screen", (time.perf_counter() - t0) * 1000)
omnisight/inputs.py ADDED
@@ -0,0 +1,299 @@
1
+ """Mouse and keyboard input.
2
+
3
+ Two delivery paths:
4
+
5
+ foreground SendInput. Works with everything that reads real input (games,
6
+ Chromium, Qt), but the target must be the foreground window and the
7
+ mouse really moves. Keys can be sent as scan codes for games that
8
+ read DirectInput / Raw Input.
9
+ background PostMessage of WM_* messages to the window under the point. Does
10
+ not move the mouse or steal focus. Classic Win32 / WinForms / many
11
+ Qt apps accept it; Chromium and most games ignore it.
12
+
13
+ Windows blocks both paths into an elevated (admin) window when this process is
14
+ not elevated (UIPI). That is reported, not worked around here.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import ctypes
20
+ import time
21
+ from ctypes import wintypes
22
+
23
+ from . import winutil as wu
24
+
25
+ user32 = wu.user32
26
+
27
+ INPUT_MOUSE, INPUT_KEYBOARD = 0, 1
28
+ KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYEVENTF_SCANCODE = 0x1, 0x2, 0x4, 0x8
29
+ MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP = 0x2, 0x4
30
+ MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP = 0x8, 0x10
31
+ MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP = 0x20, 0x40
32
+ MOUSEEVENTF_WHEEL, MOUSEEVENTF_HWHEEL = 0x800, 0x1000
33
+ WHEEL_DELTA = 120
34
+
35
+ WM_MOUSEMOVE, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_LBUTTONDBLCLK = 0x200, 0x201, 0x202, 0x203
36
+ WM_RBUTTONDOWN, WM_RBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP = 0x204, 0x205, 0x207, 0x208
37
+ WM_MOUSEWHEEL = 0x20A
38
+ WM_KEYDOWN, WM_KEYUP, WM_CHAR = 0x100, 0x101, 0x102
39
+ MK_LBUTTON, MK_RBUTTON, MK_MBUTTON = 0x1, 0x2, 0x10
40
+ CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT = 0x1, 0x4
41
+ MAPVK_VK_TO_VSC = 0
42
+
43
+
44
+ class MOUSEINPUT(ctypes.Structure):
45
+ _fields_ = [("dx", wintypes.LONG), ("dy", wintypes.LONG), ("mouseData", wintypes.DWORD),
46
+ ("dwFlags", wintypes.DWORD), ("time", wintypes.DWORD),
47
+ ("dwExtraInfo", ctypes.c_size_t)]
48
+
49
+
50
+ class KEYBDINPUT(ctypes.Structure):
51
+ _fields_ = [("wVk", wintypes.WORD), ("wScan", wintypes.WORD), ("dwFlags", wintypes.DWORD),
52
+ ("time", wintypes.DWORD), ("dwExtraInfo", ctypes.c_size_t)]
53
+
54
+
55
+ class HARDWAREINPUT(ctypes.Structure):
56
+ _fields_ = [("uMsg", wintypes.DWORD), ("wParamL", wintypes.WORD), ("wParamH", wintypes.WORD)]
57
+
58
+
59
+ class _U(ctypes.Union):
60
+ _fields_ = [("mi", MOUSEINPUT), ("ki", KEYBDINPUT), ("hi", HARDWAREINPUT)]
61
+
62
+
63
+ class INPUT(ctypes.Structure):
64
+ _fields_ = [("type", wintypes.DWORD), ("u", _U)]
65
+
66
+
67
+ class GUITHREADINFO(ctypes.Structure):
68
+ _fields_ = [("cbSize", wintypes.DWORD), ("flags", wintypes.DWORD),
69
+ ("hwndActive", wintypes.HWND), ("hwndFocus", wintypes.HWND),
70
+ ("hwndCapture", wintypes.HWND), ("hwndMenuOwner", wintypes.HWND),
71
+ ("hwndMoveSize", wintypes.HWND), ("hwndCaret", wintypes.HWND),
72
+ ("rcCaret", wintypes.RECT)]
73
+
74
+
75
+ user32.SendInput.argtypes = [wintypes.UINT, ctypes.POINTER(INPUT), ctypes.c_int]
76
+ user32.SetCursorPos.argtypes = [ctypes.c_int, ctypes.c_int]
77
+ user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)]
78
+ user32.PostMessageW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM]
79
+ user32.SendMessageTimeoutW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM,
80
+ wintypes.UINT, wintypes.UINT, ctypes.POINTER(ctypes.c_size_t)]
81
+ SMTO_ABORTIFHUNG = 0x2
82
+ user32.ScreenToClient.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)]
83
+ user32.ChildWindowFromPointEx.argtypes = [wintypes.HWND, wintypes.POINT, wintypes.UINT]
84
+ user32.ChildWindowFromPointEx.restype = wintypes.HWND
85
+ user32.GetGUIThreadInfo.argtypes = [wintypes.DWORD, ctypes.POINTER(GUITHREADINFO)]
86
+ user32.MapVirtualKeyW.argtypes = [wintypes.UINT, wintypes.UINT]
87
+ user32.VkKeyScanW.argtypes = [wintypes.WCHAR]
88
+ user32.VkKeyScanW.restype = ctypes.c_short
89
+
90
+
91
+ class InputBlocked(Exception):
92
+ pass
93
+
94
+
95
+ def check_uipi(hwnd: int) -> None:
96
+ info = wu.get_window(hwnd)
97
+ if info.elevated and not wu.self_elevated():
98
+ raise InputBlocked(
99
+ f"{info.process} runs as administrator and this server does not. Windows (UIPI) "
100
+ "drops input sent to it. Restart the MCP server from an elevated terminal to "
101
+ "control this window.")
102
+
103
+
104
+ def _send(*inputs: INPUT) -> None:
105
+ arr = (INPUT * len(inputs))(*inputs)
106
+ if user32.SendInput(len(inputs), arr, ctypes.sizeof(INPUT)) != len(inputs):
107
+ raise InputBlocked(f"SendInput was blocked (error {ctypes.get_last_error()}); "
108
+ "the secure desktop (UAC, lock screen) may be active.")
109
+
110
+
111
+ def _mouse(flags: int, data: int = 0) -> INPUT:
112
+ return INPUT(INPUT_MOUSE, _U(mi=MOUSEINPUT(0, 0, ctypes.c_ulong(data & 0xFFFFFFFF).value, flags, 0, 0)))
113
+
114
+
115
+ def _key(vk: int = 0, scan: int = 0, flags: int = 0) -> INPUT:
116
+ return INPUT(INPUT_KEYBOARD, _U(ki=KEYBDINPUT(vk, scan, flags, 0, 0)))
117
+
118
+
119
+ _BUTTONS = {
120
+ "left": (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),
121
+ "right": (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),
122
+ "middle": (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),
123
+ }
124
+
125
+
126
+ def cursor_pos() -> tuple[int, int]:
127
+ p = wintypes.POINT()
128
+ user32.GetCursorPos(ctypes.byref(p))
129
+ return p.x, p.y
130
+
131
+
132
+ def ensure_foreground(hwnd: int) -> None:
133
+ if not wu.focus(hwnd):
134
+ raise InputBlocked("Windows refused to bring the window to the foreground. "
135
+ "Use background=true, or ask the user to click the window once.")
136
+ time.sleep(0.05)
137
+
138
+
139
+ # ---------------------------------------------------------------- foreground
140
+
141
+
142
+ def click(x: int, y: int, button: str = "left", count: int = 1) -> None:
143
+ down, up = _BUTTONS[button][:2]
144
+ user32.SetCursorPos(x, y)
145
+ time.sleep(0.01)
146
+ for _ in range(count):
147
+ _send(_mouse(down), _mouse(up))
148
+ time.sleep(0.03)
149
+
150
+
151
+ def move(x: int, y: int) -> None:
152
+ user32.SetCursorPos(x, y)
153
+
154
+
155
+ def drag(x1: int, y1: int, x2: int, y2: int, button: str = "left", steps: int = 20) -> None:
156
+ down, up = _BUTTONS[button][:2]
157
+ user32.SetCursorPos(x1, y1)
158
+ _send(_mouse(down))
159
+ time.sleep(0.05)
160
+ for i in range(1, steps + 1):
161
+ user32.SetCursorPos(x1 + (x2 - x1) * i // steps, y1 + (y2 - y1) * i // steps)
162
+ time.sleep(0.01)
163
+ time.sleep(0.05)
164
+ _send(_mouse(up))
165
+
166
+
167
+ def scroll(x: int, y: int, clicks: int, horizontal: bool = False) -> None:
168
+ user32.SetCursorPos(x, y)
169
+ _send(_mouse(MOUSEEVENTF_HWHEEL if horizontal else MOUSEEVENTF_WHEEL, clicks * WHEEL_DELTA))
170
+
171
+
172
+ def type_text(text: str) -> None:
173
+ """Unicode injection: layout independent, works for CJK and emoji."""
174
+ events = []
175
+ for ch in text:
176
+ if ch == "\n":
177
+ events += [_key(0x0D), _key(0x0D, flags=KEYEVENTF_KEYUP)]
178
+ continue
179
+ for unit in ch.encode("utf-16-le").hex(" ", 2).split():
180
+ code = int.from_bytes(bytes.fromhex(unit), "little")
181
+ events += [_key(0, code, KEYEVENTF_UNICODE), _key(0, code, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP)]
182
+ for i in range(0, len(events), 64):
183
+ _send(*events[i: i + 64])
184
+ time.sleep(0.005)
185
+
186
+
187
+ VK = {
188
+ "ctrl": 0x11, "control": 0x11, "shift": 0x10, "alt": 0x12, "win": 0x5B, "cmd": 0x5B,
189
+ "enter": 0x0D, "return": 0x0D, "tab": 0x09, "esc": 0x1B, "escape": 0x1B, "space": 0x20,
190
+ "backspace": 0x08, "delete": 0x2E, "del": 0x2E, "insert": 0x2D, "home": 0x24, "end": 0x23,
191
+ "pageup": 0x21, "pagedown": 0x22, "up": 0x26, "down": 0x28, "left": 0x25, "right": 0x27,
192
+ "capslock": 0x14, "printscreen": 0x2C, "menu": 0x5D, "apps": 0x5D,
193
+ **{f"f{i}": 0x6F + i for i in range(1, 25)},
194
+ }
195
+ _EXTENDED = {0x2E, 0x2D, 0x24, 0x23, 0x21, 0x22, 0x26, 0x28, 0x25, 0x27, 0x5B, 0x5D}
196
+
197
+
198
+ def _vk(name: str) -> int:
199
+ n = name.lower()
200
+ if n in VK:
201
+ return VK[n]
202
+ if len(name) == 1:
203
+ r = user32.VkKeyScanW(name)
204
+ if r != -1:
205
+ return r & 0xFF
206
+ raise ValueError(f"unknown key {name!r}")
207
+
208
+
209
+ def key(combo: str, scancode: bool = False, hold_ms: int = 0) -> None:
210
+ """Press a combination like 'ctrl+shift+s'. scancode=true for games."""
211
+ vks = [_vk(k) for k in combo.replace(" ", "").split("+") if k]
212
+
213
+ def ev(vk, release):
214
+ flags = KEYEVENTF_KEYUP if release else 0
215
+ if vk in _EXTENDED:
216
+ flags |= KEYEVENTF_EXTENDEDKEY
217
+ if scancode:
218
+ return _key(0, user32.MapVirtualKeyW(vk, MAPVK_VK_TO_VSC), flags | KEYEVENTF_SCANCODE)
219
+ return _key(vk, 0, flags)
220
+
221
+ _send(*[ev(v, False) for v in vks])
222
+ time.sleep(max(hold_ms, 20) / 1000)
223
+ _send(*[ev(v, True) for v in reversed(vks)])
224
+
225
+
226
+ # ---------------------------------------------------------------- background
227
+
228
+
229
+ def _deepest_child(hwnd: int, sx: int, sy: int) -> tuple[int, int, int]:
230
+ """Deepest child window at a screen point, plus the point in its client coords."""
231
+ target = hwnd
232
+ while True:
233
+ p = wintypes.POINT(sx, sy)
234
+ user32.ScreenToClient(target, ctypes.byref(p))
235
+ child = user32.ChildWindowFromPointEx(target, p, CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT)
236
+ if not child or child == target:
237
+ return target, p.x, p.y
238
+ target = child
239
+
240
+
241
+ def _lparam(x: int, y: int) -> int:
242
+ return (y & 0xFFFF) << 16 | (x & 0xFFFF)
243
+
244
+
245
+ def post_click(hwnd: int, sx: int, sy: int, button: str = "left", count: int = 1) -> int:
246
+ _, _, wm_down, wm_up, mk = _BUTTONS[button]
247
+ target, cx, cy = _deepest_child(hwnd, sx, sy)
248
+ lp = _lparam(cx, cy)
249
+ user32.PostMessageW(target, WM_MOUSEMOVE, 0, lp)
250
+ for i in range(count):
251
+ down = WM_LBUTTONDBLCLK if (i == 1 and button == "left") else wm_down
252
+ user32.PostMessageW(target, down, mk, lp)
253
+ user32.PostMessageW(target, wm_up, 0, lp)
254
+ return target
255
+
256
+
257
+ def _focus_child(hwnd: int) -> int:
258
+ tid = user32.GetWindowThreadProcessId(hwnd, None)
259
+ gi = GUITHREADINFO(cbSize=ctypes.sizeof(GUITHREADINFO))
260
+ if user32.GetGUIThreadInfo(tid, ctypes.byref(gi)) and gi.hwndFocus:
261
+ return gi.hwndFocus
262
+ return hwnd
263
+
264
+
265
+ def _send_msg(hwnd: int, msg: int, wp: int, lp: int) -> None:
266
+ # Synchronous, so the text is in place when the call returns; the timeout
267
+ # keeps a hung target from hanging the server.
268
+ res = ctypes.c_size_t()
269
+ user32.SendMessageTimeoutW(hwnd, msg, wp, lp, SMTO_ABORTIFHUNG, 1000, ctypes.byref(res))
270
+
271
+
272
+ def post_text(hwnd: int, text: str) -> int:
273
+ target = _focus_child(hwnd)
274
+ for ch in text.replace("\n", "\r"):
275
+ for unit in ch.encode("utf-16-le").hex(" ", 2).split():
276
+ _send_msg(target, WM_CHAR, int.from_bytes(bytes.fromhex(unit), "little"), 1)
277
+ return target
278
+
279
+
280
+ def post_key(hwnd: int, combo: str) -> int:
281
+ """Background key press. Modifiers are sent as messages too, but most apps check
282
+ the real keyboard state for Ctrl/Alt/Shift, so combinations often arrive as the
283
+ bare key."""
284
+ target = _focus_child(hwnd)
285
+ vks = [_vk(k) for k in combo.replace(" ", "").split("+") if k]
286
+ for v in vks:
287
+ sc = user32.MapVirtualKeyW(v, MAPVK_VK_TO_VSC)
288
+ _send_msg(target, WM_KEYDOWN, v, 1 | sc << 16)
289
+ for v in reversed(vks):
290
+ sc = user32.MapVirtualKeyW(v, MAPVK_VK_TO_VSC)
291
+ _send_msg(target, WM_KEYUP, v, 1 | sc << 16 | 0xC0000000)
292
+ return target
293
+
294
+
295
+ def post_scroll(hwnd: int, sx: int, sy: int, clicks: int) -> int:
296
+ target, _, _ = _deepest_child(hwnd, sx, sy)
297
+ # WM_MOUSEWHEEL takes screen coordinates.
298
+ user32.PostMessageW(target, WM_MOUSEWHEEL, (clicks * WHEEL_DELTA & 0xFFFF) << 16, _lparam(sx, sy))
299
+ return target