omnisight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnisight/__init__.py +1 -0
- omnisight/capture.py +355 -0
- omnisight/inputs.py +299 -0
- omnisight/ocr.py +138 -0
- omnisight/py.typed +0 -0
- omnisight/server.py +432 -0
- omnisight/uia.py +258 -0
- omnisight/winutil.py +266 -0
- omnisight-0.1.0.dist-info/METADATA +204 -0
- omnisight-0.1.0.dist-info/RECORD +13 -0
- omnisight-0.1.0.dist-info/WHEEL +4 -0
- omnisight-0.1.0.dist-info/entry_points.txt +3 -0
- omnisight-0.1.0.dist-info/licenses/LICENSE +202 -0
omnisight/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""OmniSight: computer-use backend that can see and drive any Windows app window."""
|
omnisight/capture.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Window capture with several backends and automatic fallback.
|
|
2
|
+
|
|
3
|
+
Order tried for a window (unless a backend that worked before for the same
|
|
4
|
+
process is cached):
|
|
5
|
+
|
|
6
|
+
printwindow PrintWindow(PW_RENDERFULLCONTENT). Works for most GDI and many
|
|
7
|
+
DirectComposition windows (Chromium, Electron), including covered ones.
|
|
8
|
+
wgc Windows.Graphics.Capture. Sees GPU-rendered content (games,
|
|
9
|
+
UWP) and windows covered by other windows. Cannot see minimized
|
|
10
|
+
windows.
|
|
11
|
+
bitblt Classic window DC copy. Cheap; black for GPU content.
|
|
12
|
+
screen Crop of the desktop at the window rect. Only valid when nothing
|
|
13
|
+
covers the window; checked before use.
|
|
14
|
+
|
|
15
|
+
A frame that is (almost) one flat color is treated as a failed capture and the
|
|
16
|
+
next backend is tried.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import ctypes
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
from ctypes import wintypes
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
|
|
29
|
+
from . import winutil as wu
|
|
30
|
+
|
|
31
|
+
gdi32 = ctypes.WinDLL("gdi32")
|
|
32
|
+
user32 = wu.user32
|
|
33
|
+
|
|
34
|
+
PW_RENDERFULLCONTENT = 0x2
|
|
35
|
+
SRCCOPY = 0x00CC0020
|
|
36
|
+
DIB_RGB_COLORS = 0
|
|
37
|
+
|
|
38
|
+
user32.GetWindowDC.argtypes = [wintypes.HWND]
|
|
39
|
+
user32.GetWindowDC.restype = wintypes.HDC
|
|
40
|
+
user32.ReleaseDC.argtypes = [wintypes.HWND, wintypes.HDC]
|
|
41
|
+
user32.PrintWindow.argtypes = [wintypes.HWND, wintypes.HDC, wintypes.UINT]
|
|
42
|
+
gdi32.CreateCompatibleDC.argtypes = [wintypes.HDC]
|
|
43
|
+
gdi32.CreateCompatibleDC.restype = wintypes.HDC
|
|
44
|
+
gdi32.CreateCompatibleBitmap.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int]
|
|
45
|
+
gdi32.CreateCompatibleBitmap.restype = wintypes.HBITMAP
|
|
46
|
+
gdi32.SelectObject.argtypes = [wintypes.HDC, wintypes.HGDIOBJ]
|
|
47
|
+
gdi32.SelectObject.restype = wintypes.HGDIOBJ
|
|
48
|
+
gdi32.DeleteObject.argtypes = [wintypes.HGDIOBJ]
|
|
49
|
+
gdi32.DeleteDC.argtypes = [wintypes.HDC]
|
|
50
|
+
gdi32.BitBlt.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
|
|
51
|
+
wintypes.HDC, ctypes.c_int, ctypes.c_int, wintypes.DWORD]
|
|
52
|
+
gdi32.GetDIBits.argtypes = [wintypes.HDC, wintypes.HBITMAP, wintypes.UINT, wintypes.UINT,
|
|
53
|
+
ctypes.c_void_p, ctypes.c_void_p, wintypes.UINT]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class BITMAPINFOHEADER(ctypes.Structure):
|
|
57
|
+
_fields_ = [
|
|
58
|
+
("biSize", wintypes.DWORD), ("biWidth", wintypes.LONG), ("biHeight", wintypes.LONG),
|
|
59
|
+
("biPlanes", wintypes.WORD), ("biBitCount", wintypes.WORD),
|
|
60
|
+
("biCompression", wintypes.DWORD), ("biSizeImage", wintypes.DWORD),
|
|
61
|
+
("biXPelsPerMeter", wintypes.LONG), ("biYPelsPerMeter", wintypes.LONG),
|
|
62
|
+
("biClrUsed", wintypes.DWORD), ("biClrImportant", wintypes.DWORD),
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class CaptureError(Exception):
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class Capture:
|
|
72
|
+
image: np.ndarray # H x W x 3, RGB, uint8
|
|
73
|
+
origin: tuple[int, int] # screen position of image pixel (0, 0)
|
|
74
|
+
backend: str
|
|
75
|
+
elapsed_ms: float
|
|
76
|
+
attempts: list[str] = field(default_factory=list) # "backend: why it was rejected"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def is_blank(img: np.ndarray) -> bool:
|
|
80
|
+
if img.size == 0:
|
|
81
|
+
return True
|
|
82
|
+
small = img[:: max(1, img.shape[0] // 64), :: max(1, img.shape[1] // 64)]
|
|
83
|
+
return float(small.reshape(-1, small.shape[-1]).std(axis=0).max()) < 1.5
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
SUSPECT_BLACK = 0.9
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def black_ratio(img: np.ndarray) -> float:
|
|
90
|
+
small = img[:: max(1, img.shape[0] // 128), :: max(1, img.shape[1] // 128)]
|
|
91
|
+
return float((small.max(axis=2) < 8).mean())
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _bgra_to_rgb(buf: np.ndarray) -> np.ndarray:
|
|
95
|
+
return np.ascontiguousarray(buf[:, :, 2::-1])
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# --------------------------------------------------------------------- GDI
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _gdi_grab(hwnd: int, use_printwindow: bool) -> tuple[np.ndarray, tuple[int, int]]:
|
|
102
|
+
wr = wintypes.RECT()
|
|
103
|
+
user32.GetWindowRect(hwnd, ctypes.byref(wr))
|
|
104
|
+
w, h = wr.right - wr.left, wr.bottom - wr.top
|
|
105
|
+
if w <= 0 or h <= 0:
|
|
106
|
+
raise CaptureError("window has no size")
|
|
107
|
+
wdc = user32.GetWindowDC(hwnd)
|
|
108
|
+
mdc = gdi32.CreateCompatibleDC(wdc)
|
|
109
|
+
bmp = gdi32.CreateCompatibleBitmap(wdc, w, h)
|
|
110
|
+
old = gdi32.SelectObject(mdc, bmp)
|
|
111
|
+
try:
|
|
112
|
+
if use_printwindow:
|
|
113
|
+
ok = user32.PrintWindow(hwnd, mdc, PW_RENDERFULLCONTENT)
|
|
114
|
+
else:
|
|
115
|
+
ok = gdi32.BitBlt(mdc, 0, 0, w, h, wdc, 0, 0, SRCCOPY)
|
|
116
|
+
if not ok:
|
|
117
|
+
raise CaptureError("call failed")
|
|
118
|
+
bi = BITMAPINFOHEADER(ctypes.sizeof(BITMAPINFOHEADER), w, -h, 1, 32, 0, 0, 0, 0, 0, 0)
|
|
119
|
+
buf = np.empty((h, w, 4), dtype=np.uint8)
|
|
120
|
+
if not gdi32.GetDIBits(mdc, bmp, 0, h, buf.ctypes.data, ctypes.byref(bi), DIB_RGB_COLORS):
|
|
121
|
+
raise CaptureError("GetDIBits failed")
|
|
122
|
+
finally:
|
|
123
|
+
gdi32.SelectObject(mdc, old)
|
|
124
|
+
gdi32.DeleteObject(bmp)
|
|
125
|
+
gdi32.DeleteDC(mdc)
|
|
126
|
+
user32.ReleaseDC(hwnd, wdc)
|
|
127
|
+
# Trim the invisible DWM resize border so the image matches the visible frame.
|
|
128
|
+
l, t, r, b = wu.window_rect(hwnd)
|
|
129
|
+
ox, oy = max(0, l - wr.left), max(0, t - wr.top)
|
|
130
|
+
buf = buf[oy: oy + (b - t), ox: ox + (r - l)]
|
|
131
|
+
return _bgra_to_rgb(buf), (l, t)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# --------------------------------------------------------------------- WGC
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class _WgcSession:
|
|
138
|
+
"""A live Windows.Graphics.Capture session for one window.
|
|
139
|
+
|
|
140
|
+
WGC only delivers a frame when the window content changes, so the newest
|
|
141
|
+
frame is kept and reused; repeated snapshots of a static window cost
|
|
142
|
+
nothing. Sessions close after IDLE_SECONDS without use.
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
def __init__(self, hwnd: int):
|
|
146
|
+
from windows_capture import WindowsCapture
|
|
147
|
+
|
|
148
|
+
self.hwnd = hwnd
|
|
149
|
+
self.frame: np.ndarray | None = None
|
|
150
|
+
self.frame_time = 0.0
|
|
151
|
+
self.closed = False
|
|
152
|
+
self.last_used = time.monotonic()
|
|
153
|
+
self.ready = threading.Event()
|
|
154
|
+
self._lock = threading.Lock()
|
|
155
|
+
# minimum_update_interval / draw_border are Windows 11 only; leave them unset.
|
|
156
|
+
cap = WindowsCapture(cursor_capture=False, window_hwnd=hwnd)
|
|
157
|
+
|
|
158
|
+
@cap.event
|
|
159
|
+
def on_frame_arrived(frame, control):
|
|
160
|
+
if self.closed:
|
|
161
|
+
control.stop()
|
|
162
|
+
return
|
|
163
|
+
img = frame.frame_buffer.copy() # native buffer is only valid inside the callback
|
|
164
|
+
with self._lock:
|
|
165
|
+
self.frame, self.frame_time = img, time.monotonic()
|
|
166
|
+
self.ready.set()
|
|
167
|
+
|
|
168
|
+
@cap.event
|
|
169
|
+
def on_closed():
|
|
170
|
+
self.closed = True
|
|
171
|
+
self.ready.set()
|
|
172
|
+
|
|
173
|
+
self.control = cap.start_free_threaded()
|
|
174
|
+
|
|
175
|
+
def latest(self, timeout: float) -> tuple[np.ndarray, float]:
|
|
176
|
+
self.last_used = time.monotonic()
|
|
177
|
+
if not self.ready.wait(timeout):
|
|
178
|
+
raise CaptureError("no frame within timeout")
|
|
179
|
+
with self._lock:
|
|
180
|
+
if self.frame is None:
|
|
181
|
+
raise CaptureError("session closed before first frame")
|
|
182
|
+
return self.frame, self.frame_time
|
|
183
|
+
|
|
184
|
+
def stop(self) -> None:
|
|
185
|
+
self.closed = True
|
|
186
|
+
try:
|
|
187
|
+
self.control.stop()
|
|
188
|
+
except Exception:
|
|
189
|
+
pass
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _WgcPool:
|
|
193
|
+
IDLE_SECONDS = 20.0
|
|
194
|
+
|
|
195
|
+
def __init__(self):
|
|
196
|
+
self.sessions: dict[int, _WgcSession] = {}
|
|
197
|
+
self.lock = threading.Lock()
|
|
198
|
+
threading.Thread(target=self._reaper, daemon=True).start()
|
|
199
|
+
|
|
200
|
+
def get(self, hwnd: int) -> _WgcSession:
|
|
201
|
+
with self.lock:
|
|
202
|
+
s = self.sessions.get(hwnd)
|
|
203
|
+
if s is None or s.closed:
|
|
204
|
+
s = self.sessions[hwnd] = _WgcSession(hwnd)
|
|
205
|
+
return s
|
|
206
|
+
|
|
207
|
+
def drop(self, hwnd: int) -> None:
|
|
208
|
+
with self.lock:
|
|
209
|
+
s = self.sessions.pop(hwnd, None)
|
|
210
|
+
if s:
|
|
211
|
+
s.stop()
|
|
212
|
+
|
|
213
|
+
def _reaper(self):
|
|
214
|
+
while True:
|
|
215
|
+
time.sleep(5)
|
|
216
|
+
now = time.monotonic()
|
|
217
|
+
with self.lock:
|
|
218
|
+
dead = [h for h, s in self.sessions.items()
|
|
219
|
+
if s.closed or now - s.last_used > self.IDLE_SECONDS or not wu.is_window(h)]
|
|
220
|
+
stale = [self.sessions.pop(h) for h in dead]
|
|
221
|
+
for s in stale:
|
|
222
|
+
s.stop()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
_wgc_pool = _WgcPool()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _wgc_grab(hwnd: int, timeout: float = 1.5) -> tuple[np.ndarray, tuple[int, int]]:
|
|
229
|
+
session = _wgc_pool.get(hwnd)
|
|
230
|
+
buf, _ = session.latest(timeout)
|
|
231
|
+
l, t, r, b = wu.window_rect(hwnd)
|
|
232
|
+
img = _bgra_to_rgb(buf)
|
|
233
|
+
# Frame may include the invisible border; centre-crop to the visible frame.
|
|
234
|
+
w, h = r - l, b - t
|
|
235
|
+
if img.shape[1] > w or img.shape[0] > h:
|
|
236
|
+
ox = max(0, (img.shape[1] - w) // 2)
|
|
237
|
+
img = img[: h, ox: ox + w]
|
|
238
|
+
return img, (l, t)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# ------------------------------------------------------------------- screen
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def grab_screen_region(left: int, top: int, right: int, bottom: int) -> np.ndarray:
|
|
245
|
+
import mss
|
|
246
|
+
|
|
247
|
+
with mss.mss() as sct:
|
|
248
|
+
shot = sct.grab({"left": left, "top": top, "width": right - left, "height": bottom - top})
|
|
249
|
+
return _bgra_to_rgb(np.asarray(shot))
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def virtual_screen() -> tuple[int, int, int, int]:
|
|
253
|
+
import mss
|
|
254
|
+
|
|
255
|
+
with mss.mss() as sct:
|
|
256
|
+
m = sct.monitors[0]
|
|
257
|
+
return (m["left"], m["top"], m["left"] + m["width"], m["top"] + m["height"])
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def monitors() -> list[dict]:
|
|
261
|
+
import mss
|
|
262
|
+
|
|
263
|
+
with mss.mss() as sct:
|
|
264
|
+
return [dict(index=i, left=m["left"], top=m["top"], width=m["width"], height=m["height"])
|
|
265
|
+
for i, m in enumerate(sct.monitors[1:], start=1)]
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _unoccluded(hwnd: int) -> bool:
|
|
269
|
+
l, t, r, b = wu.window_rect(hwnd)
|
|
270
|
+
pts = [((l + r) // 2, (t + b) // 2)]
|
|
271
|
+
for fx in (0.1, 0.5, 0.9):
|
|
272
|
+
for fy in (0.1, 0.5, 0.9):
|
|
273
|
+
pts.append((int(l + (r - l) * fx), int(t + (b - t) * fy)))
|
|
274
|
+
return all(wu.root_at(x, y) == hwnd for x, y in pts)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _screen_grab(hwnd: int) -> tuple[np.ndarray, tuple[int, int]]:
|
|
278
|
+
if not _unoccluded(hwnd):
|
|
279
|
+
raise CaptureError("window is covered by another window")
|
|
280
|
+
l, t, r, b = wu.window_rect(hwnd)
|
|
281
|
+
return grab_screen_region(l, t, r, b), (l, t)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# ------------------------------------------------------------------- driver
|
|
285
|
+
|
|
286
|
+
BACKENDS = {
|
|
287
|
+
"wgc": _wgc_grab,
|
|
288
|
+
"printwindow": lambda h: _gdi_grab(h, True),
|
|
289
|
+
"bitblt": lambda h: _gdi_grab(h, False),
|
|
290
|
+
"screen": _screen_grab,
|
|
291
|
+
}
|
|
292
|
+
# PrintWindow first: on Windows 10 a WGC session costs 150-600 ms to start and
|
|
293
|
+
# draws a yellow border around the window while it lives.
|
|
294
|
+
DEFAULT_ORDER = ["printwindow", "wgc", "bitblt", "screen"]
|
|
295
|
+
|
|
296
|
+
# exe name -> backend that last produced a usable frame
|
|
297
|
+
_preferred: dict[str, str] = {}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def capture_window(hwnd: int, backend: str = "auto", restore_minimized: bool = False) -> Capture:
|
|
301
|
+
info = wu.get_window(hwnd)
|
|
302
|
+
if info.capture_blocked:
|
|
303
|
+
raise CaptureError(
|
|
304
|
+
"This window has opted out of screen capture (SetWindowDisplayAffinity). "
|
|
305
|
+
"It is a deliberate protection, e.g. a password manager or DRM video; "
|
|
306
|
+
"use the element tree instead, or ask the user to read it.")
|
|
307
|
+
if info.minimized:
|
|
308
|
+
if not restore_minimized:
|
|
309
|
+
raise CaptureError("Window is minimized. Retry with restore_minimized=true to "
|
|
310
|
+
"restore it without stealing focus.")
|
|
311
|
+
wu.restore_no_activate(hwnd)
|
|
312
|
+
time.sleep(0.25)
|
|
313
|
+
if info.cloaked:
|
|
314
|
+
raise CaptureError("Window is cloaked (on another virtual desktop or suspended).")
|
|
315
|
+
|
|
316
|
+
if backend != "auto":
|
|
317
|
+
order = [backend]
|
|
318
|
+
else:
|
|
319
|
+
order = list(DEFAULT_ORDER)
|
|
320
|
+
pref = _preferred.get(info.process)
|
|
321
|
+
if pref in order:
|
|
322
|
+
order.remove(pref)
|
|
323
|
+
order.insert(0, pref)
|
|
324
|
+
|
|
325
|
+
attempts: list[str] = []
|
|
326
|
+
t0 = time.perf_counter()
|
|
327
|
+
suspects: list[tuple[float, Capture]] = []
|
|
328
|
+
for name in order:
|
|
329
|
+
try:
|
|
330
|
+
img, origin = BACKENDS[name](hwnd)
|
|
331
|
+
except Exception as e: # noqa: BLE001 - every backend failure is just "try the next one"
|
|
332
|
+
attempts.append(f"{name}: {e}")
|
|
333
|
+
if name == "wgc":
|
|
334
|
+
_wgc_pool.drop(hwnd)
|
|
335
|
+
continue
|
|
336
|
+
cap = Capture(img, origin, name, (time.perf_counter() - t0) * 1000, attempts)
|
|
337
|
+
black = black_ratio(img)
|
|
338
|
+
if is_blank(img) or black > SUSPECT_BLACK:
|
|
339
|
+
# GPU content read through GDI comes back black, often with only the
|
|
340
|
+
# title bar drawn. Keep it as a fallback; maybe the window is just dark.
|
|
341
|
+
attempts.append(f"{name}: {'blank' if is_blank(img) else f'{black:.0%} black'} frame")
|
|
342
|
+
suspects.append((1.0 if is_blank(img) else black, cap))
|
|
343
|
+
continue
|
|
344
|
+
_preferred[info.process] = name
|
|
345
|
+
return cap
|
|
346
|
+
if suspects:
|
|
347
|
+
return min(suspects, key=lambda s: s[0])[1]
|
|
348
|
+
raise CaptureError("all capture backends failed: " + "; ".join(attempts))
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def capture_screen(region: tuple[int, int, int, int] | None = None) -> Capture:
|
|
352
|
+
t0 = time.perf_counter()
|
|
353
|
+
region = region or virtual_screen()
|
|
354
|
+
img = grab_screen_region(*region)
|
|
355
|
+
return Capture(img, (region[0], region[1]), "screen", (time.perf_counter() - t0) * 1000)
|
omnisight/inputs.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""Mouse and keyboard input.
|
|
2
|
+
|
|
3
|
+
Two delivery paths:
|
|
4
|
+
|
|
5
|
+
foreground SendInput. Works with everything that reads real input (games,
|
|
6
|
+
Chromium, Qt), but the target must be the foreground window and the
|
|
7
|
+
mouse really moves. Keys can be sent as scan codes for games that
|
|
8
|
+
read DirectInput / Raw Input.
|
|
9
|
+
background PostMessage of WM_* messages to the window under the point. Does
|
|
10
|
+
not move the mouse or steal focus. Classic Win32 / WinForms / many
|
|
11
|
+
Qt apps accept it; Chromium and most games ignore it.
|
|
12
|
+
|
|
13
|
+
Windows blocks both paths into an elevated (admin) window when this process is
|
|
14
|
+
not elevated (UIPI). That is reported, not worked around here.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import ctypes
|
|
20
|
+
import time
|
|
21
|
+
from ctypes import wintypes
|
|
22
|
+
|
|
23
|
+
from . import winutil as wu
|
|
24
|
+
|
|
25
|
+
user32 = wu.user32
|
|
26
|
+
|
|
27
|
+
INPUT_MOUSE, INPUT_KEYBOARD = 0, 1
|
|
28
|
+
KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYEVENTF_SCANCODE = 0x1, 0x2, 0x4, 0x8
|
|
29
|
+
MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP = 0x2, 0x4
|
|
30
|
+
MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP = 0x8, 0x10
|
|
31
|
+
MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP = 0x20, 0x40
|
|
32
|
+
MOUSEEVENTF_WHEEL, MOUSEEVENTF_HWHEEL = 0x800, 0x1000
|
|
33
|
+
WHEEL_DELTA = 120
|
|
34
|
+
|
|
35
|
+
WM_MOUSEMOVE, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_LBUTTONDBLCLK = 0x200, 0x201, 0x202, 0x203
|
|
36
|
+
WM_RBUTTONDOWN, WM_RBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP = 0x204, 0x205, 0x207, 0x208
|
|
37
|
+
WM_MOUSEWHEEL = 0x20A
|
|
38
|
+
WM_KEYDOWN, WM_KEYUP, WM_CHAR = 0x100, 0x101, 0x102
|
|
39
|
+
MK_LBUTTON, MK_RBUTTON, MK_MBUTTON = 0x1, 0x2, 0x10
|
|
40
|
+
CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT = 0x1, 0x4
|
|
41
|
+
MAPVK_VK_TO_VSC = 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class MOUSEINPUT(ctypes.Structure):
|
|
45
|
+
_fields_ = [("dx", wintypes.LONG), ("dy", wintypes.LONG), ("mouseData", wintypes.DWORD),
|
|
46
|
+
("dwFlags", wintypes.DWORD), ("time", wintypes.DWORD),
|
|
47
|
+
("dwExtraInfo", ctypes.c_size_t)]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class KEYBDINPUT(ctypes.Structure):
|
|
51
|
+
_fields_ = [("wVk", wintypes.WORD), ("wScan", wintypes.WORD), ("dwFlags", wintypes.DWORD),
|
|
52
|
+
("time", wintypes.DWORD), ("dwExtraInfo", ctypes.c_size_t)]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class HARDWAREINPUT(ctypes.Structure):
|
|
56
|
+
_fields_ = [("uMsg", wintypes.DWORD), ("wParamL", wintypes.WORD), ("wParamH", wintypes.WORD)]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _U(ctypes.Union):
|
|
60
|
+
_fields_ = [("mi", MOUSEINPUT), ("ki", KEYBDINPUT), ("hi", HARDWAREINPUT)]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class INPUT(ctypes.Structure):
|
|
64
|
+
_fields_ = [("type", wintypes.DWORD), ("u", _U)]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class GUITHREADINFO(ctypes.Structure):
|
|
68
|
+
_fields_ = [("cbSize", wintypes.DWORD), ("flags", wintypes.DWORD),
|
|
69
|
+
("hwndActive", wintypes.HWND), ("hwndFocus", wintypes.HWND),
|
|
70
|
+
("hwndCapture", wintypes.HWND), ("hwndMenuOwner", wintypes.HWND),
|
|
71
|
+
("hwndMoveSize", wintypes.HWND), ("hwndCaret", wintypes.HWND),
|
|
72
|
+
("rcCaret", wintypes.RECT)]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
user32.SendInput.argtypes = [wintypes.UINT, ctypes.POINTER(INPUT), ctypes.c_int]
|
|
76
|
+
user32.SetCursorPos.argtypes = [ctypes.c_int, ctypes.c_int]
|
|
77
|
+
user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)]
|
|
78
|
+
user32.PostMessageW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM]
|
|
79
|
+
user32.SendMessageTimeoutW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM,
|
|
80
|
+
wintypes.UINT, wintypes.UINT, ctypes.POINTER(ctypes.c_size_t)]
|
|
81
|
+
SMTO_ABORTIFHUNG = 0x2
|
|
82
|
+
user32.ScreenToClient.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)]
|
|
83
|
+
user32.ChildWindowFromPointEx.argtypes = [wintypes.HWND, wintypes.POINT, wintypes.UINT]
|
|
84
|
+
user32.ChildWindowFromPointEx.restype = wintypes.HWND
|
|
85
|
+
user32.GetGUIThreadInfo.argtypes = [wintypes.DWORD, ctypes.POINTER(GUITHREADINFO)]
|
|
86
|
+
user32.MapVirtualKeyW.argtypes = [wintypes.UINT, wintypes.UINT]
|
|
87
|
+
user32.VkKeyScanW.argtypes = [wintypes.WCHAR]
|
|
88
|
+
user32.VkKeyScanW.restype = ctypes.c_short
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class InputBlocked(Exception):
|
|
92
|
+
pass
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def check_uipi(hwnd: int) -> None:
|
|
96
|
+
info = wu.get_window(hwnd)
|
|
97
|
+
if info.elevated and not wu.self_elevated():
|
|
98
|
+
raise InputBlocked(
|
|
99
|
+
f"{info.process} runs as administrator and this server does not. Windows (UIPI) "
|
|
100
|
+
"drops input sent to it. Restart the MCP server from an elevated terminal to "
|
|
101
|
+
"control this window.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _send(*inputs: INPUT) -> None:
|
|
105
|
+
arr = (INPUT * len(inputs))(*inputs)
|
|
106
|
+
if user32.SendInput(len(inputs), arr, ctypes.sizeof(INPUT)) != len(inputs):
|
|
107
|
+
raise InputBlocked(f"SendInput was blocked (error {ctypes.get_last_error()}); "
|
|
108
|
+
"the secure desktop (UAC, lock screen) may be active.")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _mouse(flags: int, data: int = 0) -> INPUT:
|
|
112
|
+
return INPUT(INPUT_MOUSE, _U(mi=MOUSEINPUT(0, 0, ctypes.c_ulong(data & 0xFFFFFFFF).value, flags, 0, 0)))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _key(vk: int = 0, scan: int = 0, flags: int = 0) -> INPUT:
|
|
116
|
+
return INPUT(INPUT_KEYBOARD, _U(ki=KEYBDINPUT(vk, scan, flags, 0, 0)))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
_BUTTONS = {
|
|
120
|
+
"left": (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),
|
|
121
|
+
"right": (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),
|
|
122
|
+
"middle": (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def cursor_pos() -> tuple[int, int]:
|
|
127
|
+
p = wintypes.POINT()
|
|
128
|
+
user32.GetCursorPos(ctypes.byref(p))
|
|
129
|
+
return p.x, p.y
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def ensure_foreground(hwnd: int) -> None:
|
|
133
|
+
if not wu.focus(hwnd):
|
|
134
|
+
raise InputBlocked("Windows refused to bring the window to the foreground. "
|
|
135
|
+
"Use background=true, or ask the user to click the window once.")
|
|
136
|
+
time.sleep(0.05)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# ---------------------------------------------------------------- foreground
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def click(x: int, y: int, button: str = "left", count: int = 1) -> None:
|
|
143
|
+
down, up = _BUTTONS[button][:2]
|
|
144
|
+
user32.SetCursorPos(x, y)
|
|
145
|
+
time.sleep(0.01)
|
|
146
|
+
for _ in range(count):
|
|
147
|
+
_send(_mouse(down), _mouse(up))
|
|
148
|
+
time.sleep(0.03)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def move(x: int, y: int) -> None:
|
|
152
|
+
user32.SetCursorPos(x, y)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def drag(x1: int, y1: int, x2: int, y2: int, button: str = "left", steps: int = 20) -> None:
|
|
156
|
+
down, up = _BUTTONS[button][:2]
|
|
157
|
+
user32.SetCursorPos(x1, y1)
|
|
158
|
+
_send(_mouse(down))
|
|
159
|
+
time.sleep(0.05)
|
|
160
|
+
for i in range(1, steps + 1):
|
|
161
|
+
user32.SetCursorPos(x1 + (x2 - x1) * i // steps, y1 + (y2 - y1) * i // steps)
|
|
162
|
+
time.sleep(0.01)
|
|
163
|
+
time.sleep(0.05)
|
|
164
|
+
_send(_mouse(up))
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def scroll(x: int, y: int, clicks: int, horizontal: bool = False) -> None:
|
|
168
|
+
user32.SetCursorPos(x, y)
|
|
169
|
+
_send(_mouse(MOUSEEVENTF_HWHEEL if horizontal else MOUSEEVENTF_WHEEL, clicks * WHEEL_DELTA))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def type_text(text: str) -> None:
|
|
173
|
+
"""Unicode injection: layout independent, works for CJK and emoji."""
|
|
174
|
+
events = []
|
|
175
|
+
for ch in text:
|
|
176
|
+
if ch == "\n":
|
|
177
|
+
events += [_key(0x0D), _key(0x0D, flags=KEYEVENTF_KEYUP)]
|
|
178
|
+
continue
|
|
179
|
+
for unit in ch.encode("utf-16-le").hex(" ", 2).split():
|
|
180
|
+
code = int.from_bytes(bytes.fromhex(unit), "little")
|
|
181
|
+
events += [_key(0, code, KEYEVENTF_UNICODE), _key(0, code, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP)]
|
|
182
|
+
for i in range(0, len(events), 64):
|
|
183
|
+
_send(*events[i: i + 64])
|
|
184
|
+
time.sleep(0.005)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
VK = {
|
|
188
|
+
"ctrl": 0x11, "control": 0x11, "shift": 0x10, "alt": 0x12, "win": 0x5B, "cmd": 0x5B,
|
|
189
|
+
"enter": 0x0D, "return": 0x0D, "tab": 0x09, "esc": 0x1B, "escape": 0x1B, "space": 0x20,
|
|
190
|
+
"backspace": 0x08, "delete": 0x2E, "del": 0x2E, "insert": 0x2D, "home": 0x24, "end": 0x23,
|
|
191
|
+
"pageup": 0x21, "pagedown": 0x22, "up": 0x26, "down": 0x28, "left": 0x25, "right": 0x27,
|
|
192
|
+
"capslock": 0x14, "printscreen": 0x2C, "menu": 0x5D, "apps": 0x5D,
|
|
193
|
+
**{f"f{i}": 0x6F + i for i in range(1, 25)},
|
|
194
|
+
}
|
|
195
|
+
_EXTENDED = {0x2E, 0x2D, 0x24, 0x23, 0x21, 0x22, 0x26, 0x28, 0x25, 0x27, 0x5B, 0x5D}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _vk(name: str) -> int:
|
|
199
|
+
n = name.lower()
|
|
200
|
+
if n in VK:
|
|
201
|
+
return VK[n]
|
|
202
|
+
if len(name) == 1:
|
|
203
|
+
r = user32.VkKeyScanW(name)
|
|
204
|
+
if r != -1:
|
|
205
|
+
return r & 0xFF
|
|
206
|
+
raise ValueError(f"unknown key {name!r}")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def key(combo: str, scancode: bool = False, hold_ms: int = 0) -> None:
|
|
210
|
+
"""Press a combination like 'ctrl+shift+s'. scancode=true for games."""
|
|
211
|
+
vks = [_vk(k) for k in combo.replace(" ", "").split("+") if k]
|
|
212
|
+
|
|
213
|
+
def ev(vk, release):
|
|
214
|
+
flags = KEYEVENTF_KEYUP if release else 0
|
|
215
|
+
if vk in _EXTENDED:
|
|
216
|
+
flags |= KEYEVENTF_EXTENDEDKEY
|
|
217
|
+
if scancode:
|
|
218
|
+
return _key(0, user32.MapVirtualKeyW(vk, MAPVK_VK_TO_VSC), flags | KEYEVENTF_SCANCODE)
|
|
219
|
+
return _key(vk, 0, flags)
|
|
220
|
+
|
|
221
|
+
_send(*[ev(v, False) for v in vks])
|
|
222
|
+
time.sleep(max(hold_ms, 20) / 1000)
|
|
223
|
+
_send(*[ev(v, True) for v in reversed(vks)])
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# ---------------------------------------------------------------- background
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _deepest_child(hwnd: int, sx: int, sy: int) -> tuple[int, int, int]:
|
|
230
|
+
"""Deepest child window at a screen point, plus the point in its client coords."""
|
|
231
|
+
target = hwnd
|
|
232
|
+
while True:
|
|
233
|
+
p = wintypes.POINT(sx, sy)
|
|
234
|
+
user32.ScreenToClient(target, ctypes.byref(p))
|
|
235
|
+
child = user32.ChildWindowFromPointEx(target, p, CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT)
|
|
236
|
+
if not child or child == target:
|
|
237
|
+
return target, p.x, p.y
|
|
238
|
+
target = child
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _lparam(x: int, y: int) -> int:
|
|
242
|
+
return (y & 0xFFFF) << 16 | (x & 0xFFFF)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def post_click(hwnd: int, sx: int, sy: int, button: str = "left", count: int = 1) -> int:
|
|
246
|
+
_, _, wm_down, wm_up, mk = _BUTTONS[button]
|
|
247
|
+
target, cx, cy = _deepest_child(hwnd, sx, sy)
|
|
248
|
+
lp = _lparam(cx, cy)
|
|
249
|
+
user32.PostMessageW(target, WM_MOUSEMOVE, 0, lp)
|
|
250
|
+
for i in range(count):
|
|
251
|
+
down = WM_LBUTTONDBLCLK if (i == 1 and button == "left") else wm_down
|
|
252
|
+
user32.PostMessageW(target, down, mk, lp)
|
|
253
|
+
user32.PostMessageW(target, wm_up, 0, lp)
|
|
254
|
+
return target
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _focus_child(hwnd: int) -> int:
|
|
258
|
+
tid = user32.GetWindowThreadProcessId(hwnd, None)
|
|
259
|
+
gi = GUITHREADINFO(cbSize=ctypes.sizeof(GUITHREADINFO))
|
|
260
|
+
if user32.GetGUIThreadInfo(tid, ctypes.byref(gi)) and gi.hwndFocus:
|
|
261
|
+
return gi.hwndFocus
|
|
262
|
+
return hwnd
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _send_msg(hwnd: int, msg: int, wp: int, lp: int) -> None:
|
|
266
|
+
# Synchronous, so the text is in place when the call returns; the timeout
|
|
267
|
+
# keeps a hung target from hanging the server.
|
|
268
|
+
res = ctypes.c_size_t()
|
|
269
|
+
user32.SendMessageTimeoutW(hwnd, msg, wp, lp, SMTO_ABORTIFHUNG, 1000, ctypes.byref(res))
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def post_text(hwnd: int, text: str) -> int:
|
|
273
|
+
target = _focus_child(hwnd)
|
|
274
|
+
for ch in text.replace("\n", "\r"):
|
|
275
|
+
for unit in ch.encode("utf-16-le").hex(" ", 2).split():
|
|
276
|
+
_send_msg(target, WM_CHAR, int.from_bytes(bytes.fromhex(unit), "little"), 1)
|
|
277
|
+
return target
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def post_key(hwnd: int, combo: str) -> int:
|
|
281
|
+
"""Background key press. Modifiers are sent as messages too, but most apps check
|
|
282
|
+
the real keyboard state for Ctrl/Alt/Shift, so combinations often arrive as the
|
|
283
|
+
bare key."""
|
|
284
|
+
target = _focus_child(hwnd)
|
|
285
|
+
vks = [_vk(k) for k in combo.replace(" ", "").split("+") if k]
|
|
286
|
+
for v in vks:
|
|
287
|
+
sc = user32.MapVirtualKeyW(v, MAPVK_VK_TO_VSC)
|
|
288
|
+
_send_msg(target, WM_KEYDOWN, v, 1 | sc << 16)
|
|
289
|
+
for v in reversed(vks):
|
|
290
|
+
sc = user32.MapVirtualKeyW(v, MAPVK_VK_TO_VSC)
|
|
291
|
+
_send_msg(target, WM_KEYUP, v, 1 | sc << 16 | 0xC0000000)
|
|
292
|
+
return target
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def post_scroll(hwnd: int, sx: int, sy: int, clicks: int) -> int:
|
|
296
|
+
target, _, _ = _deepest_child(hwnd, sx, sy)
|
|
297
|
+
# WM_MOUSEWHEEL takes screen coordinates.
|
|
298
|
+
user32.PostMessageW(target, WM_MOUSEWHEEL, (clicks * WHEEL_DELTA & 0xFFFF) << 16, _lparam(sx, sy))
|
|
299
|
+
return target
|