computeruseprotocol 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- computeruseprotocol-0.1.0.dist-info/METADATA +225 -0
- computeruseprotocol-0.1.0.dist-info/RECORD +27 -0
- computeruseprotocol-0.1.0.dist-info/WHEEL +4 -0
- computeruseprotocol-0.1.0.dist-info/entry_points.txt +3 -0
- computeruseprotocol-0.1.0.dist-info/licenses/LICENSE +21 -0
- cup/__init__.py +548 -0
- cup/__main__.py +222 -0
- cup/_base.py +123 -0
- cup/_router.py +63 -0
- cup/actions/__init__.py +9 -0
- cup/actions/_handler.py +62 -0
- cup/actions/_keys.py +56 -0
- cup/actions/_linux.py +1008 -0
- cup/actions/_macos.py +1090 -0
- cup/actions/_web.py +555 -0
- cup/actions/_windows.py +984 -0
- cup/actions/executor.py +162 -0
- cup/format.py +653 -0
- cup/mcp/__init__.py +1 -0
- cup/mcp/__main__.py +11 -0
- cup/mcp/server.py +418 -0
- cup/platforms/__init__.py +0 -0
- cup/platforms/linux.py +1060 -0
- cup/platforms/macos.py +1005 -0
- cup/platforms/web.py +1009 -0
- cup/platforms/windows.py +935 -0
- cup/search.py +583 -0
cup/platforms/macos.py
ADDED
|
@@ -0,0 +1,1005 @@
|
|
|
1
|
+
"""
|
|
2
|
+
macOS AXUIElement platform adapter for CUP.
|
|
3
|
+
|
|
4
|
+
Captures the accessibility tree via pyobjc AXUIElement API and maps it to the
|
|
5
|
+
canonical CUP schema — roles, states, actions, and platform metadata.
|
|
6
|
+
|
|
7
|
+
Requires macOS accessibility permissions:
|
|
8
|
+
System Settings > Privacy & Security > Accessibility > (add Terminal / Python)
|
|
9
|
+
|
|
10
|
+
Dependencies:
|
|
11
|
+
pip install pyobjc-framework-ApplicationServices pyobjc-framework-Cocoa pyobjc-framework-Quartz
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import concurrent.futures
|
|
17
|
+
import itertools
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from AppKit import NSApplicationActivationPolicyRegular, NSArray, NSScreen, NSWorkspace
|
|
21
|
+
from ApplicationServices import (
|
|
22
|
+
AXUIElementCopyActionNames,
|
|
23
|
+
AXUIElementCopyAttributeValue,
|
|
24
|
+
AXUIElementCopyMultipleAttributeValues,
|
|
25
|
+
AXUIElementCreateApplication,
|
|
26
|
+
AXUIElementIsAttributeSettable,
|
|
27
|
+
AXValueGetType,
|
|
28
|
+
AXValueGetValue,
|
|
29
|
+
kAXChildrenAttribute,
|
|
30
|
+
kAXDescriptionAttribute,
|
|
31
|
+
kAXElementBusyAttribute,
|
|
32
|
+
kAXEnabledAttribute,
|
|
33
|
+
kAXErrorSuccess,
|
|
34
|
+
kAXExpandedAttribute,
|
|
35
|
+
kAXFocusedAttribute,
|
|
36
|
+
kAXFocusedWindowAttribute,
|
|
37
|
+
kAXHelpAttribute,
|
|
38
|
+
kAXIdentifierAttribute,
|
|
39
|
+
kAXMainWindowAttribute,
|
|
40
|
+
kAXModalAttribute,
|
|
41
|
+
kAXPositionAttribute,
|
|
42
|
+
kAXRoleAttribute,
|
|
43
|
+
kAXSelectedAttribute,
|
|
44
|
+
kAXSizeAttribute,
|
|
45
|
+
kAXSubroleAttribute,
|
|
46
|
+
kAXTitleAttribute,
|
|
47
|
+
kAXValueAttribute,
|
|
48
|
+
kAXValueCGPointType,
|
|
49
|
+
kAXValueCGSizeType,
|
|
50
|
+
kAXWindowsAttribute,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
from cup._base import PlatformAdapter
|
|
54
|
+
|
|
55
|
+
# ---------------------------------------------------------------------------
|
|
56
|
+
# AXRole -> CUP role mapping
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
# Primary: AXRole string -> CUP role
|
|
60
|
+
CUP_ROLES: dict[str, str] = {
|
|
61
|
+
"AXApplication": "application",
|
|
62
|
+
"AXWindow": "window",
|
|
63
|
+
"AXButton": "button",
|
|
64
|
+
"AXCheckBox": "checkbox",
|
|
65
|
+
"AXRadioButton": "radio",
|
|
66
|
+
"AXComboBox": "combobox",
|
|
67
|
+
"AXPopUpButton": "combobox",
|
|
68
|
+
"AXTextField": "textbox",
|
|
69
|
+
"AXTextArea": "textbox",
|
|
70
|
+
"AXStaticText": "text",
|
|
71
|
+
"AXImage": "img",
|
|
72
|
+
"AXLink": "link",
|
|
73
|
+
"AXList": "list",
|
|
74
|
+
"AXOutline": "tree",
|
|
75
|
+
"AXTable": "table",
|
|
76
|
+
"AXTabGroup": "tablist",
|
|
77
|
+
"AXSlider": "slider",
|
|
78
|
+
"AXProgressIndicator": "progressbar",
|
|
79
|
+
"AXMenu": "menu",
|
|
80
|
+
"AXMenuBar": "menubar",
|
|
81
|
+
"AXMenuBarItem": "menuitem",
|
|
82
|
+
"AXMenuItem": "menuitem",
|
|
83
|
+
"AXToolbar": "toolbar",
|
|
84
|
+
"AXScrollBar": "scrollbar",
|
|
85
|
+
"AXScrollArea": "generic",
|
|
86
|
+
"AXGroup": "group",
|
|
87
|
+
"AXSplitGroup": "group",
|
|
88
|
+
"AXSplitter": "separator",
|
|
89
|
+
"AXHeading": "heading",
|
|
90
|
+
"AXWebArea": "document",
|
|
91
|
+
"AXCell": "cell",
|
|
92
|
+
"AXRow": "row",
|
|
93
|
+
"AXColumn": "columnheader",
|
|
94
|
+
"AXSheet": "alertdialog",
|
|
95
|
+
"AXDrawer": "complementary",
|
|
96
|
+
"AXGrowArea": "generic",
|
|
97
|
+
"AXValueIndicator": "generic",
|
|
98
|
+
"AXIncrementor": "spinbutton",
|
|
99
|
+
"AXHelpTag": "tooltip",
|
|
100
|
+
"AXColorWell": "button",
|
|
101
|
+
"AXDisclosureTriangle": "button",
|
|
102
|
+
"AXDateField": "textbox",
|
|
103
|
+
"AXBrowser": "tree",
|
|
104
|
+
"AXBusyIndicator": "progressbar",
|
|
105
|
+
"AXRuler": "generic",
|
|
106
|
+
"AXRulerMarker": "generic",
|
|
107
|
+
"AXRelevanceIndicator": "progressbar",
|
|
108
|
+
"AXLevelIndicator": "slider",
|
|
109
|
+
"AXLayoutArea": "group",
|
|
110
|
+
"AXLayoutItem": "generic",
|
|
111
|
+
"AXHandle": "generic",
|
|
112
|
+
"AXMatte": "generic",
|
|
113
|
+
"AXUnknown": "generic",
|
|
114
|
+
"AXListMarker": "text",
|
|
115
|
+
"AXMenuButton": "button",
|
|
116
|
+
"AXRadioGroup": "group",
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
# Subrole refinements: (AXRole, AXSubrole) -> CUP role
|
|
120
|
+
CUP_SUBROLE_OVERRIDES: dict[tuple[str, str], str] = {
|
|
121
|
+
# AXGroup subroles
|
|
122
|
+
("AXGroup", "AXApplicationAlert"): "alert",
|
|
123
|
+
("AXGroup", "AXApplicationDialog"): "dialog",
|
|
124
|
+
("AXGroup", "AXApplicationStatus"): "status",
|
|
125
|
+
("AXGroup", "AXLandmarkNavigation"): "navigation",
|
|
126
|
+
("AXGroup", "AXLandmarkSearch"): "search",
|
|
127
|
+
("AXGroup", "AXLandmarkRegion"): "region",
|
|
128
|
+
("AXGroup", "AXLandmarkMain"): "main",
|
|
129
|
+
("AXGroup", "AXLandmarkComplementary"): "complementary",
|
|
130
|
+
("AXGroup", "AXLandmarkContentInfo"): "contentinfo",
|
|
131
|
+
("AXGroup", "AXLandmarkBanner"): "banner",
|
|
132
|
+
("AXGroup", "AXDocument"): "document",
|
|
133
|
+
("AXGroup", "AXWebApplication"): "application",
|
|
134
|
+
("AXGroup", "AXTab"): "tabpanel",
|
|
135
|
+
# AXWindow subroles
|
|
136
|
+
("AXWindow", "AXDialog"): "dialog",
|
|
137
|
+
("AXWindow", "AXFloatingWindow"): "dialog",
|
|
138
|
+
("AXWindow", "AXSystemDialog"): "dialog",
|
|
139
|
+
("AXWindow", "AXSystemFloatingWindow"): "dialog",
|
|
140
|
+
# AXButton subroles
|
|
141
|
+
("AXButton", "AXCloseButton"): "button",
|
|
142
|
+
("AXButton", "AXMinimizeButton"): "button",
|
|
143
|
+
("AXButton", "AXFullScreenButton"): "button",
|
|
144
|
+
# AXRadioButton used as tab
|
|
145
|
+
("AXRadioButton", "AXTabButton"): "tab",
|
|
146
|
+
# AXMenuItem subroles
|
|
147
|
+
("AXMenuItem", "AXMenuItemCheckbox"): "menuitemcheckbox",
|
|
148
|
+
("AXMenuItem", "AXMenuItemRadio"): "menuitemradio",
|
|
149
|
+
# AXTextField subroles
|
|
150
|
+
("AXTextField", "AXSearchField"): "searchbox",
|
|
151
|
+
("AXTextField", "AXSecureTextField"): "textbox",
|
|
152
|
+
# AXStaticText as status
|
|
153
|
+
("AXStaticText", "AXApplicationStatus"): "status",
|
|
154
|
+
# AXRow in outlines -> treeitem (parity with Windows TreeItem)
|
|
155
|
+
("AXRow", "AXOutlineRow"): "treeitem",
|
|
156
|
+
# AXCheckBox as toggle switch
|
|
157
|
+
("AXCheckBox", "AXToggle"): "switch",
|
|
158
|
+
("AXCheckBox", "AXSwitch"): "switch",
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
# Roles that accept text input
|
|
162
|
+
TEXT_INPUT_ROLES = {"textbox", "searchbox", "combobox", "document"}
|
|
163
|
+
|
|
164
|
+
# Roles representing toggle-like elements
|
|
165
|
+
TOGGLE_ROLES = {"checkbox", "switch", "menuitemcheckbox"}
|
|
166
|
+
|
|
167
|
+
# AX roles where AXExpanded is semantically meaningful.
|
|
168
|
+
# Chromium/Electron apps set AXExpanded on nearly every element, so we
|
|
169
|
+
# restrict this to AX roles that genuinely expand/collapse.
|
|
170
|
+
EXPANDABLE_AX_ROLES = {
|
|
171
|
+
"AXComboBox",
|
|
172
|
+
"AXPopUpButton",
|
|
173
|
+
"AXOutline",
|
|
174
|
+
"AXDisclosureTriangle",
|
|
175
|
+
"AXMenu",
|
|
176
|
+
"AXMenuItem",
|
|
177
|
+
"AXMenuBarItem",
|
|
178
|
+
"AXRow",
|
|
179
|
+
"AXBrowser",
|
|
180
|
+
"AXSheet",
|
|
181
|
+
"AXDrawer",
|
|
182
|
+
"AXTabGroup",
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
# AX roles where AXUIElementCopyActionNames is skipped for performance.
|
|
186
|
+
# These roles never produce meaningful CUP actions from their AX action list
|
|
187
|
+
# (their only AX actions are AXScrollToVisible/AXShowMenu which we skip anyway).
|
|
188
|
+
# Actions like "scroll" for AXScrollArea are derived from the role, not from AX.
|
|
189
|
+
_SKIP_ACTIONS_AX_ROLES = {
|
|
190
|
+
"AXStaticText",
|
|
191
|
+
"AXHeading",
|
|
192
|
+
"AXColumn",
|
|
193
|
+
"AXScrollArea",
|
|
194
|
+
"AXSplitGroup",
|
|
195
|
+
"AXSplitter",
|
|
196
|
+
"AXGrowArea",
|
|
197
|
+
"AXValueIndicator",
|
|
198
|
+
"AXRuler",
|
|
199
|
+
"AXRulerMarker",
|
|
200
|
+
"AXLayoutArea",
|
|
201
|
+
"AXLayoutItem",
|
|
202
|
+
"AXHandle",
|
|
203
|
+
"AXMatte",
|
|
204
|
+
"AXUnknown",
|
|
205
|
+
"AXListMarker",
|
|
206
|
+
"AXBusyIndicator",
|
|
207
|
+
"AXRelevanceIndicator",
|
|
208
|
+
"AXLevelIndicator",
|
|
209
|
+
"AXWebArea",
|
|
210
|
+
# Note: AXImage is NOT skipped — clickable images (e.g. avatars) have AXPress.
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# AX attribute helpers
|
|
216
|
+
# ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
# Attributes to batch-read per element via AXUIElementCopyMultipleAttributeValues.
|
|
219
|
+
# Order matters — indices are used to unpack the results array.
|
|
220
|
+
_BATCH_ATTRS_LIST = [
|
|
221
|
+
kAXRoleAttribute, # 0
|
|
222
|
+
kAXSubroleAttribute, # 1
|
|
223
|
+
kAXTitleAttribute, # 2
|
|
224
|
+
kAXDescriptionAttribute, # 3
|
|
225
|
+
kAXHelpAttribute, # 4
|
|
226
|
+
kAXIdentifierAttribute, # 5
|
|
227
|
+
kAXValueAttribute, # 6
|
|
228
|
+
kAXEnabledAttribute, # 7
|
|
229
|
+
kAXFocusedAttribute, # 8
|
|
230
|
+
kAXSelectedAttribute, # 9
|
|
231
|
+
kAXExpandedAttribute, # 10
|
|
232
|
+
kAXElementBusyAttribute, # 11
|
|
233
|
+
kAXModalAttribute, # 12
|
|
234
|
+
kAXPositionAttribute, # 13
|
|
235
|
+
kAXSizeAttribute, # 14
|
|
236
|
+
"AXRequired", # 15
|
|
237
|
+
"AXIsEditable", # 16
|
|
238
|
+
kAXChildrenAttribute, # 17
|
|
239
|
+
]
|
|
240
|
+
_BATCH_ATTRS = NSArray.arrayWithArray_(_BATCH_ATTRS_LIST)
|
|
241
|
+
_BATCH_IDX = {name: i for i, name in enumerate(_BATCH_ATTRS_LIST)}
|
|
242
|
+
|
|
243
|
+
# AXValueGetType returns 5 for error sentinels (kAXValueAXErrorType)
|
|
244
|
+
_AX_VALUE_ERROR_TYPE = 5
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _is_ax_error(val) -> bool:
|
|
248
|
+
"""Check if a batch-read value is an error sentinel."""
|
|
249
|
+
if val is None:
|
|
250
|
+
return True
|
|
251
|
+
try:
|
|
252
|
+
return AXValueGetType(val) == _AX_VALUE_ERROR_TYPE
|
|
253
|
+
except Exception:
|
|
254
|
+
return False
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _batch_read(element) -> list:
|
|
258
|
+
"""Read all standard attributes in one cross-process call.
|
|
259
|
+
|
|
260
|
+
Returns a list of values aligned with _BATCH_ATTRS_LIST.
|
|
261
|
+
Error sentinels are replaced with None.
|
|
262
|
+
"""
|
|
263
|
+
try:
|
|
264
|
+
err, values = AXUIElementCopyMultipleAttributeValues(element, _BATCH_ATTRS, 0, None)
|
|
265
|
+
if err != kAXErrorSuccess or values is None:
|
|
266
|
+
return [None] * len(_BATCH_ATTRS_LIST)
|
|
267
|
+
return [None if _is_ax_error(v) else v for v in values]
|
|
268
|
+
except Exception:
|
|
269
|
+
return [None] * len(_BATCH_ATTRS_LIST)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _get_attr(element, attr: str, default=None):
|
|
273
|
+
"""Safely read a single AX attribute (used for non-batched reads)."""
|
|
274
|
+
try:
|
|
275
|
+
err, value = AXUIElementCopyAttributeValue(element, attr, None)
|
|
276
|
+
if err == kAXErrorSuccess and value is not None:
|
|
277
|
+
return value
|
|
278
|
+
except Exception:
|
|
279
|
+
pass
|
|
280
|
+
return default
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _is_settable(element, attr: str) -> bool:
|
|
284
|
+
"""Check if an attribute is settable on an element."""
|
|
285
|
+
try:
|
|
286
|
+
err, settable = AXUIElementIsAttributeSettable(element, attr, None)
|
|
287
|
+
if err == kAXErrorSuccess:
|
|
288
|
+
return bool(settable)
|
|
289
|
+
except Exception:
|
|
290
|
+
pass
|
|
291
|
+
return False
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _unpack_bounds(pos_ref, size_ref) -> dict | None:
|
|
295
|
+
"""Extract {x, y, w, h} from AXPosition + AXSize value refs."""
|
|
296
|
+
if pos_ref is None or size_ref is None:
|
|
297
|
+
return None
|
|
298
|
+
try:
|
|
299
|
+
_, point = AXValueGetValue(pos_ref, kAXValueCGPointType, None)
|
|
300
|
+
_, size = AXValueGetValue(size_ref, kAXValueCGSizeType, None)
|
|
301
|
+
if point is not None and size is not None:
|
|
302
|
+
return {
|
|
303
|
+
"x": int(point.x),
|
|
304
|
+
"y": int(point.y),
|
|
305
|
+
"w": int(size.width),
|
|
306
|
+
"h": int(size.height),
|
|
307
|
+
}
|
|
308
|
+
except Exception:
|
|
309
|
+
pass
|
|
310
|
+
return None
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
# ---------------------------------------------------------------------------
|
|
314
|
+
# Screen metrics
|
|
315
|
+
# ---------------------------------------------------------------------------
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _macos_screen_info() -> tuple[int, int, float]:
|
|
319
|
+
"""Return (width, height, scale) of the primary display.
|
|
320
|
+
|
|
321
|
+
Width/height are in logical points (macOS coordinates).
|
|
322
|
+
Scale is the backing scale factor (2.0 on Retina, 1.0 on non-Retina).
|
|
323
|
+
"""
|
|
324
|
+
screen = NSScreen.mainScreen()
|
|
325
|
+
if screen is None:
|
|
326
|
+
from Quartz import CGDisplayBounds, CGMainDisplayID
|
|
327
|
+
|
|
328
|
+
bounds = CGDisplayBounds(CGMainDisplayID())
|
|
329
|
+
return int(bounds.size.width), int(bounds.size.height), 1.0
|
|
330
|
+
frame = screen.frame()
|
|
331
|
+
scale = screen.backingScaleFactor()
|
|
332
|
+
return int(frame.size.width), int(frame.size.height), float(scale)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# ---------------------------------------------------------------------------
|
|
336
|
+
# Window enumeration
|
|
337
|
+
# ---------------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
# Process names that are macOS system daemons with on-screen layer-0 windows
|
|
340
|
+
# but should NOT appear in user-facing app lists.
|
|
341
|
+
_SYSTEM_OWNER_NAMES = frozenset(
|
|
342
|
+
{
|
|
343
|
+
"WindowServer",
|
|
344
|
+
"Dock",
|
|
345
|
+
"SystemUIServer",
|
|
346
|
+
"Control Center",
|
|
347
|
+
"Notification Center",
|
|
348
|
+
"loginwindow",
|
|
349
|
+
"Window Manager",
|
|
350
|
+
"Spotlight",
|
|
351
|
+
}
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _cg_window_apps() -> dict[int, str]:
|
|
356
|
+
"""Return {pid: owner_name} for processes with on-screen, normal-layer windows.
|
|
357
|
+
|
|
358
|
+
Uses CGWindowListCopyWindowInfo which always returns fresh data from the
|
|
359
|
+
window server, unlike NSWorkspace.runningApplications() which can be stale
|
|
360
|
+
in a long-running process without an NSRunLoop.
|
|
361
|
+
"""
|
|
362
|
+
try:
|
|
363
|
+
from Quartz import (
|
|
364
|
+
CGWindowListCopyWindowInfo,
|
|
365
|
+
kCGNullWindowID,
|
|
366
|
+
kCGWindowListOptionOnScreenOnly,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
cg_windows = CGWindowListCopyWindowInfo(
|
|
370
|
+
kCGWindowListOptionOnScreenOnly,
|
|
371
|
+
kCGNullWindowID,
|
|
372
|
+
)
|
|
373
|
+
if not cg_windows:
|
|
374
|
+
return {}
|
|
375
|
+
|
|
376
|
+
result: dict[int, str] = {}
|
|
377
|
+
for w in cg_windows:
|
|
378
|
+
# Only normal-level windows (layer 0 = kCGNormalWindowLevel).
|
|
379
|
+
# Filters out menus, tooltips, overlays, screensavers, etc.
|
|
380
|
+
layer = w.get("kCGWindowLayer", -1)
|
|
381
|
+
if layer != 0:
|
|
382
|
+
continue
|
|
383
|
+
|
|
384
|
+
pid = w.get("kCGWindowOwnerPID")
|
|
385
|
+
owner = w.get("kCGWindowOwnerName", "")
|
|
386
|
+
if not pid or not owner:
|
|
387
|
+
continue
|
|
388
|
+
|
|
389
|
+
if owner in _SYSTEM_OWNER_NAMES:
|
|
390
|
+
continue
|
|
391
|
+
|
|
392
|
+
if pid not in result:
|
|
393
|
+
result[pid] = owner
|
|
394
|
+
|
|
395
|
+
return result
|
|
396
|
+
except Exception:
|
|
397
|
+
return {}
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _macos_foreground_app() -> tuple[int, str, str | None]:
|
|
401
|
+
"""Return (pid, app_name, bundle_id) of the frontmost application.
|
|
402
|
+
|
|
403
|
+
Uses CGWindowListCopyWindowInfo to get fresh data from the window server.
|
|
404
|
+
NSWorkspace.frontmostApplication() goes stale in long-running processes
|
|
405
|
+
without an active NSRunLoop (e.g., MCP servers), so we only use it as a
|
|
406
|
+
fallback.
|
|
407
|
+
"""
|
|
408
|
+
# CGWindowList returns windows in front-to-back order. The first
|
|
409
|
+
# layer-0 (normal) window that isn't a system daemon is the frontmost app.
|
|
410
|
+
try:
|
|
411
|
+
from Quartz import (
|
|
412
|
+
CGWindowListCopyWindowInfo,
|
|
413
|
+
kCGNullWindowID,
|
|
414
|
+
kCGWindowListOptionOnScreenOnly,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
cg_windows = CGWindowListCopyWindowInfo(
|
|
418
|
+
kCGWindowListOptionOnScreenOnly,
|
|
419
|
+
kCGNullWindowID,
|
|
420
|
+
)
|
|
421
|
+
if cg_windows:
|
|
422
|
+
for w in cg_windows:
|
|
423
|
+
if w.get("kCGWindowLayer", -1) != 0:
|
|
424
|
+
continue
|
|
425
|
+
pid = w.get("kCGWindowOwnerPID")
|
|
426
|
+
owner = w.get("kCGWindowOwnerName", "")
|
|
427
|
+
if not pid or not owner:
|
|
428
|
+
continue
|
|
429
|
+
if owner in _SYSTEM_OWNER_NAMES:
|
|
430
|
+
continue
|
|
431
|
+
# Found the frontmost app — look up bundle ID via NSRunningApplication
|
|
432
|
+
bundle_id = None
|
|
433
|
+
try:
|
|
434
|
+
from AppKit import NSRunningApplication
|
|
435
|
+
|
|
436
|
+
ns_app = NSRunningApplication.runningApplicationWithProcessIdentifier_(pid)
|
|
437
|
+
if ns_app is not None:
|
|
438
|
+
owner = ns_app.localizedName() or owner
|
|
439
|
+
bundle_id = ns_app.bundleIdentifier()
|
|
440
|
+
except Exception:
|
|
441
|
+
pass
|
|
442
|
+
return (pid, owner, bundle_id)
|
|
443
|
+
except Exception:
|
|
444
|
+
pass
|
|
445
|
+
|
|
446
|
+
# Fallback: NSWorkspace (may be stale without NSRunLoop)
|
|
447
|
+
workspace = NSWorkspace.sharedWorkspace()
|
|
448
|
+
app = workspace.frontmostApplication()
|
|
449
|
+
return (
|
|
450
|
+
app.processIdentifier(),
|
|
451
|
+
app.localizedName() or "",
|
|
452
|
+
app.bundleIdentifier(),
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _macos_visible_apps() -> list[tuple[int, str, str | None]]:
|
|
457
|
+
"""Return [(pid, app_name, bundle_id)] for all visible (regular) apps.
|
|
458
|
+
|
|
459
|
+
Combines NSWorkspace.runningApplications() (provides bundle_id and
|
|
460
|
+
activation policy) with CGWindowListCopyWindowInfo (always fresh from
|
|
461
|
+
the window server) to ensure newly launched apps are not missed due to
|
|
462
|
+
stale NSRunLoop state.
|
|
463
|
+
"""
|
|
464
|
+
workspace = NSWorkspace.sharedWorkspace()
|
|
465
|
+
apps = []
|
|
466
|
+
seen_pids: set[int] = set()
|
|
467
|
+
|
|
468
|
+
for app in workspace.runningApplications():
|
|
469
|
+
if app.activationPolicy() == NSApplicationActivationPolicyRegular:
|
|
470
|
+
pid = app.processIdentifier()
|
|
471
|
+
apps.append(
|
|
472
|
+
(
|
|
473
|
+
pid,
|
|
474
|
+
app.localizedName() or "",
|
|
475
|
+
app.bundleIdentifier(),
|
|
476
|
+
)
|
|
477
|
+
)
|
|
478
|
+
seen_pids.add(pid)
|
|
479
|
+
|
|
480
|
+
# Cross-check: find apps with visible windows that NSWorkspace missed
|
|
481
|
+
for pid, owner_name in _cg_window_apps().items():
|
|
482
|
+
if pid not in seen_pids:
|
|
483
|
+
apps.append((pid, owner_name, None))
|
|
484
|
+
seen_pids.add(pid)
|
|
485
|
+
|
|
486
|
+
return apps
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _macos_windows_for_app(pid: int):
|
|
490
|
+
"""Return list of AXWindow elements for an app, or empty list."""
|
|
491
|
+
app_ref = AXUIElementCreateApplication(pid)
|
|
492
|
+
windows = _get_attr(app_ref, kAXWindowsAttribute)
|
|
493
|
+
if windows is not None:
|
|
494
|
+
return list(windows)
|
|
495
|
+
return []
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _macos_focused_window(pid: int):
|
|
499
|
+
"""Return the focused window AXUIElement for an app, or None."""
|
|
500
|
+
app_ref = AXUIElementCreateApplication(pid)
|
|
501
|
+
win = _get_attr(app_ref, kAXFocusedWindowAttribute)
|
|
502
|
+
if win is not None:
|
|
503
|
+
return win
|
|
504
|
+
return _get_attr(app_ref, kAXMainWindowAttribute)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
# ---------------------------------------------------------------------------
|
|
508
|
+
# CUP node builder
|
|
509
|
+
# ---------------------------------------------------------------------------
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def build_cup_node(element, id_gen, stats: dict) -> tuple[dict, list] | None:
|
|
513
|
+
"""Build a CUP-formatted node from a macOS AXUIElement.
|
|
514
|
+
|
|
515
|
+
Uses batch attribute reading for performance — a single cross-process call
|
|
516
|
+
fetches all 18 standard attributes (including children) instead of
|
|
517
|
+
individual calls per attribute.
|
|
518
|
+
|
|
519
|
+
Returns (node_dict, children_refs) or None if the element has no role.
|
|
520
|
+
"""
|
|
521
|
+
stats["nodes"] += 1
|
|
522
|
+
|
|
523
|
+
# ── Batch-read all standard attributes in one call ──
|
|
524
|
+
vals = _batch_read(element)
|
|
525
|
+
|
|
526
|
+
# ── Core properties ──
|
|
527
|
+
ax_role = vals[0] # kAXRoleAttribute
|
|
528
|
+
if not ax_role or not isinstance(ax_role, str):
|
|
529
|
+
return None
|
|
530
|
+
|
|
531
|
+
ax_subrole = vals[1] # kAXSubroleAttribute
|
|
532
|
+
if ax_subrole is not None and not isinstance(ax_subrole, str):
|
|
533
|
+
ax_subrole = None
|
|
534
|
+
|
|
535
|
+
title = vals[2] # kAXTitleAttribute
|
|
536
|
+
if title is not None and not isinstance(title, str):
|
|
537
|
+
title = None
|
|
538
|
+
|
|
539
|
+
description = vals[3] # kAXDescriptionAttribute
|
|
540
|
+
if description is not None and not isinstance(description, str):
|
|
541
|
+
description = None
|
|
542
|
+
|
|
543
|
+
help_text = vals[4] # kAXHelpAttribute
|
|
544
|
+
if help_text is not None and not isinstance(help_text, str):
|
|
545
|
+
help_text = None
|
|
546
|
+
|
|
547
|
+
ax_identifier = vals[5] # kAXIdentifierAttribute
|
|
548
|
+
if ax_identifier is not None and not isinstance(ax_identifier, str):
|
|
549
|
+
ax_identifier = None
|
|
550
|
+
|
|
551
|
+
raw_value = vals[6] # kAXValueAttribute
|
|
552
|
+
|
|
553
|
+
# Name: prefer title, fall back to description.
|
|
554
|
+
# For AXStaticText, the visible text is often in AXValue (native macOS apps
|
|
555
|
+
# like System Settings), so use that as final fallback for text elements.
|
|
556
|
+
name = title or description or ""
|
|
557
|
+
if not name and ax_role in ("AXStaticText", "AXHeading"):
|
|
558
|
+
if raw_value is not None and isinstance(raw_value, str):
|
|
559
|
+
name = raw_value
|
|
560
|
+
|
|
561
|
+
# Bounds from AXPosition + AXSize
|
|
562
|
+
bounds = _unpack_bounds(vals[13], vals[14])
|
|
563
|
+
|
|
564
|
+
# Stats tracking
|
|
565
|
+
role_key = f"{ax_role}:{ax_subrole}" if ax_subrole else ax_role
|
|
566
|
+
stats["roles"][role_key] = stats["roles"].get(role_key, 0) + 1
|
|
567
|
+
|
|
568
|
+
# ── Role mapping ──
|
|
569
|
+
role = CUP_SUBROLE_OVERRIDES.get((ax_role, ax_subrole))
|
|
570
|
+
if role is None:
|
|
571
|
+
role = CUP_ROLES.get(ax_role, "generic")
|
|
572
|
+
|
|
573
|
+
# ── State properties (from batch values) ──
|
|
574
|
+
is_enabled_val = vals[7] # kAXEnabledAttribute
|
|
575
|
+
is_enabled = bool(is_enabled_val) if is_enabled_val is not None else True
|
|
576
|
+
is_focused = bool(vals[8]) # kAXFocusedAttribute
|
|
577
|
+
is_selected = bool(vals[9]) # kAXSelectedAttribute
|
|
578
|
+
is_busy = bool(vals[11]) # kAXElementBusyAttribute
|
|
579
|
+
is_modal = bool(vals[12]) # kAXModalAttribute
|
|
580
|
+
|
|
581
|
+
# Expanded state — only meaningful for certain AX roles (Chromium/Electron
|
|
582
|
+
# apps set AXExpanded on nearly every element, causing noise)
|
|
583
|
+
expanded_val = vals[10] # kAXExpandedAttribute
|
|
584
|
+
has_expanded = ax_role in EXPANDABLE_AX_ROLES and expanded_val is not None
|
|
585
|
+
is_expanded = bool(expanded_val) if has_expanded else None
|
|
586
|
+
|
|
587
|
+
# Required (from batch)
|
|
588
|
+
is_required = bool(vals[15]) # AXRequired
|
|
589
|
+
|
|
590
|
+
# Value as string
|
|
591
|
+
val_str = ""
|
|
592
|
+
if raw_value is not None:
|
|
593
|
+
try:
|
|
594
|
+
val_str = str(raw_value)
|
|
595
|
+
except Exception:
|
|
596
|
+
pass
|
|
597
|
+
|
|
598
|
+
# Editable (from batch, with settable fallback)
|
|
599
|
+
is_editable = bool(vals[16]) # AXIsEditable
|
|
600
|
+
if not is_editable and role in TEXT_INPUT_ROLES:
|
|
601
|
+
is_editable = _is_settable(element, kAXValueAttribute)
|
|
602
|
+
|
|
603
|
+
# ── Offscreen detection ──
|
|
604
|
+
# macOS has no IsOffscreen property, so we check bounds against screen rect
|
|
605
|
+
is_offscreen = False
|
|
606
|
+
if bounds:
|
|
607
|
+
screen_w = stats.get("screen_w", 99999)
|
|
608
|
+
screen_h = stats.get("screen_h", 99999)
|
|
609
|
+
bx, by, bw, bh = bounds["x"], bounds["y"], bounds["w"], bounds["h"]
|
|
610
|
+
# Element is offscreen if entirely outside screen or has zero size
|
|
611
|
+
if bw <= 0 or bh <= 0 or bx + bw <= 0 or by + bh <= 0 or bx >= screen_w or by >= screen_h:
|
|
612
|
+
is_offscreen = True
|
|
613
|
+
|
|
614
|
+
# ── Build states list ──
|
|
615
|
+
states: list[str] = []
|
|
616
|
+
if not is_enabled:
|
|
617
|
+
states.append("disabled")
|
|
618
|
+
if is_focused:
|
|
619
|
+
states.append("focused")
|
|
620
|
+
if is_offscreen:
|
|
621
|
+
states.append("offscreen")
|
|
622
|
+
if is_selected:
|
|
623
|
+
states.append("selected")
|
|
624
|
+
if is_busy:
|
|
625
|
+
states.append("busy")
|
|
626
|
+
if is_modal:
|
|
627
|
+
states.append("modal")
|
|
628
|
+
if is_required:
|
|
629
|
+
states.append("required")
|
|
630
|
+
if has_expanded:
|
|
631
|
+
if is_expanded:
|
|
632
|
+
states.append("expanded")
|
|
633
|
+
else:
|
|
634
|
+
states.append("collapsed")
|
|
635
|
+
|
|
636
|
+
# Checked/mixed for toggles
|
|
637
|
+
if role in TOGGLE_ROLES and raw_value is not None:
|
|
638
|
+
try:
|
|
639
|
+
int_val = int(raw_value)
|
|
640
|
+
if int_val == 1:
|
|
641
|
+
states.append("checked")
|
|
642
|
+
elif int_val == 2:
|
|
643
|
+
states.append("mixed")
|
|
644
|
+
except (ValueError, TypeError):
|
|
645
|
+
pass
|
|
646
|
+
|
|
647
|
+
if is_editable:
|
|
648
|
+
states.append("editable")
|
|
649
|
+
elif role in TEXT_INPUT_ROLES and not is_editable:
|
|
650
|
+
states.append("readonly")
|
|
651
|
+
|
|
652
|
+
# ── Actions ──
|
|
653
|
+
# Skip the action names cross-process call for roles that never produce
|
|
654
|
+
# meaningful CUP actions (saves ~30-60% of per-node overhead).
|
|
655
|
+
skip_actions = ax_role in _SKIP_ACTIONS_AX_ROLES or (ax_role == "AXGroup" and not name)
|
|
656
|
+
if skip_actions:
|
|
657
|
+
ax_action_list = []
|
|
658
|
+
else:
|
|
659
|
+
try:
|
|
660
|
+
err, ax_actions = AXUIElementCopyActionNames(element, None)
|
|
661
|
+
ax_action_list = list(ax_actions) if err == kAXErrorSuccess and ax_actions else []
|
|
662
|
+
except Exception:
|
|
663
|
+
ax_action_list = []
|
|
664
|
+
|
|
665
|
+
actions: list[str] = []
|
|
666
|
+
for ax_act in ax_action_list:
|
|
667
|
+
if ax_act == "AXPress":
|
|
668
|
+
if role in TOGGLE_ROLES:
|
|
669
|
+
actions.append("toggle")
|
|
670
|
+
elif role in (
|
|
671
|
+
"listitem",
|
|
672
|
+
"option",
|
|
673
|
+
"tab",
|
|
674
|
+
"treeitem",
|
|
675
|
+
"menuitem",
|
|
676
|
+
"menuitemcheckbox",
|
|
677
|
+
"menuitemradio",
|
|
678
|
+
):
|
|
679
|
+
actions.append("select")
|
|
680
|
+
else:
|
|
681
|
+
actions.append("click")
|
|
682
|
+
elif ax_act == "AXIncrement":
|
|
683
|
+
actions.append("increment")
|
|
684
|
+
elif ax_act == "AXDecrement":
|
|
685
|
+
actions.append("decrement")
|
|
686
|
+
elif ax_act == "AXCancel":
|
|
687
|
+
actions.append("dismiss")
|
|
688
|
+
elif ax_act == "AXRaise":
|
|
689
|
+
actions.append("focus")
|
|
690
|
+
elif ax_act == "AXConfirm":
|
|
691
|
+
actions.append("click")
|
|
692
|
+
elif ax_act == "AXPick":
|
|
693
|
+
if "select" not in actions:
|
|
694
|
+
actions.append("select")
|
|
695
|
+
# Note: AXScrollToVisible and AXShowMenu are skipped —
|
|
696
|
+
# Chromium/Electron sets them on ~99% of elements as noise.
|
|
697
|
+
# AXScrollToVisible means "scroll parent to show me" (passive),
|
|
698
|
+
# not "I am scrollable". AXShowMenu opens a context menu.
|
|
699
|
+
|
|
700
|
+
# Text input: add type/setvalue if value is settable
|
|
701
|
+
if role in TEXT_INPUT_ROLES and is_editable:
|
|
702
|
+
if "setvalue" not in actions:
|
|
703
|
+
actions.append("setvalue")
|
|
704
|
+
if "type" not in actions:
|
|
705
|
+
actions.append("type")
|
|
706
|
+
|
|
707
|
+
# Expand/collapse from expanded state
|
|
708
|
+
if has_expanded:
|
|
709
|
+
if "expand" not in actions:
|
|
710
|
+
actions.append("expand")
|
|
711
|
+
if "collapse" not in actions:
|
|
712
|
+
actions.append("collapse")
|
|
713
|
+
|
|
714
|
+
# Scroll areas are scrollable containers
|
|
715
|
+
if ax_role == "AXScrollArea" and "scroll" not in actions:
|
|
716
|
+
actions.append("scroll")
|
|
717
|
+
|
|
718
|
+
# Fallback: focusable
|
|
719
|
+
if not actions and is_enabled:
|
|
720
|
+
actions.append("focus")
|
|
721
|
+
|
|
722
|
+
# ── Attributes (read conditionally per role to avoid overhead on all nodes) ──
|
|
723
|
+
attrs: dict = {}
|
|
724
|
+
|
|
725
|
+
# Tree item nesting depth
|
|
726
|
+
if role == "treeitem":
|
|
727
|
+
dl = _get_attr(element, "AXDisclosureLevel")
|
|
728
|
+
if dl is not None:
|
|
729
|
+
try:
|
|
730
|
+
attrs["level"] = int(dl) + 1 # AX is 0-based, CUP is 1-based
|
|
731
|
+
except (ValueError, TypeError):
|
|
732
|
+
pass
|
|
733
|
+
|
|
734
|
+
# Range widget min/max/current
|
|
735
|
+
if role in ("slider", "progressbar", "spinbutton", "scrollbar"):
|
|
736
|
+
min_val = _get_attr(element, "AXMinValue")
|
|
737
|
+
max_val = _get_attr(element, "AXMaxValue")
|
|
738
|
+
if min_val is not None:
|
|
739
|
+
try:
|
|
740
|
+
attrs["valueMin"] = float(min_val)
|
|
741
|
+
except (ValueError, TypeError):
|
|
742
|
+
pass
|
|
743
|
+
if max_val is not None:
|
|
744
|
+
try:
|
|
745
|
+
attrs["valueMax"] = float(max_val)
|
|
746
|
+
except (ValueError, TypeError):
|
|
747
|
+
pass
|
|
748
|
+
if raw_value is not None:
|
|
749
|
+
try:
|
|
750
|
+
attrs["valueNow"] = float(raw_value)
|
|
751
|
+
except (ValueError, TypeError):
|
|
752
|
+
pass
|
|
753
|
+
|
|
754
|
+
# Placeholder text for inputs
|
|
755
|
+
if role in ("textbox", "searchbox", "combobox"):
|
|
756
|
+
placeholder = _get_attr(element, "AXPlaceholderValue")
|
|
757
|
+
if placeholder is not None and isinstance(placeholder, str) and placeholder:
|
|
758
|
+
attrs["placeholder"] = placeholder[:200]
|
|
759
|
+
|
|
760
|
+
# Link URL
|
|
761
|
+
if role == "link":
|
|
762
|
+
url = _get_attr(element, "AXURL")
|
|
763
|
+
if url is not None:
|
|
764
|
+
url_str = str(url)
|
|
765
|
+
if url_str:
|
|
766
|
+
attrs["url"] = url_str[:500]
|
|
767
|
+
|
|
768
|
+
# Orientation
|
|
769
|
+
if role in ("scrollbar", "slider", "separator", "toolbar", "tablist"):
|
|
770
|
+
orientation = _get_attr(element, "AXOrientation")
|
|
771
|
+
if orientation is not None:
|
|
772
|
+
orient_str = str(orientation)
|
|
773
|
+
if "Vertical" in orient_str:
|
|
774
|
+
attrs["orientation"] = "vertical"
|
|
775
|
+
elif "Horizontal" in orient_str:
|
|
776
|
+
attrs["orientation"] = "horizontal"
|
|
777
|
+
|
|
778
|
+
# ── Assemble CUP node ──
|
|
779
|
+
node: dict = {
|
|
780
|
+
"id": f"e{next(id_gen)}",
|
|
781
|
+
"role": role,
|
|
782
|
+
"name": name[:200],
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
# Description: use help text (or description if title was used as name)
|
|
786
|
+
desc_text = help_text if help_text else (description if title and description else "")
|
|
787
|
+
if desc_text:
|
|
788
|
+
node["description"] = desc_text[:200]
|
|
789
|
+
if val_str and role in (
|
|
790
|
+
"textbox",
|
|
791
|
+
"searchbox",
|
|
792
|
+
"combobox",
|
|
793
|
+
"spinbutton",
|
|
794
|
+
"slider",
|
|
795
|
+
"progressbar",
|
|
796
|
+
"document",
|
|
797
|
+
):
|
|
798
|
+
node["value"] = val_str[:200]
|
|
799
|
+
if bounds:
|
|
800
|
+
node["bounds"] = bounds
|
|
801
|
+
if states:
|
|
802
|
+
node["states"] = states
|
|
803
|
+
if actions:
|
|
804
|
+
node["actions"] = actions
|
|
805
|
+
if attrs:
|
|
806
|
+
node["attributes"] = attrs
|
|
807
|
+
|
|
808
|
+
# ── Platform extension (macOS-specific raw data) ──
|
|
809
|
+
pm: dict = {"axRole": ax_role}
|
|
810
|
+
if ax_subrole:
|
|
811
|
+
pm["axSubrole"] = ax_subrole
|
|
812
|
+
if ax_identifier:
|
|
813
|
+
pm["axIdentifier"] = ax_identifier
|
|
814
|
+
if ax_action_list:
|
|
815
|
+
pm["axActions"] = ax_action_list
|
|
816
|
+
node["platform"] = {"macos": pm}
|
|
817
|
+
|
|
818
|
+
# Children refs from batch (index 17)
|
|
819
|
+
children_refs = vals[17]
|
|
820
|
+
if children_refs is not None:
|
|
821
|
+
children_refs = list(children_refs)
|
|
822
|
+
else:
|
|
823
|
+
children_refs = []
|
|
824
|
+
|
|
825
|
+
return node, children_refs
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
# ---------------------------------------------------------------------------
|
|
829
|
+
# Tree walker
|
|
830
|
+
# ---------------------------------------------------------------------------
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def walk_tree(element, depth: int, max_depth: int, id_gen, stats: dict, refs: dict) -> dict | None:
|
|
834
|
+
"""Recursively walk an AXUIElement tree and build CUP nodes."""
|
|
835
|
+
if depth > max_depth:
|
|
836
|
+
return None
|
|
837
|
+
|
|
838
|
+
result = build_cup_node(element, id_gen, stats)
|
|
839
|
+
if result is None:
|
|
840
|
+
return None
|
|
841
|
+
node, children_refs = result
|
|
842
|
+
|
|
843
|
+
refs[node["id"]] = element
|
|
844
|
+
|
|
845
|
+
stats["max_depth"] = max(stats["max_depth"], depth)
|
|
846
|
+
|
|
847
|
+
if depth < max_depth and children_refs:
|
|
848
|
+
children: list[dict] = []
|
|
849
|
+
for child_ref in children_refs:
|
|
850
|
+
child_node = walk_tree(child_ref, depth + 1, max_depth, id_gen, stats, refs)
|
|
851
|
+
if child_node is not None:
|
|
852
|
+
children.append(child_node)
|
|
853
|
+
if children:
|
|
854
|
+
node["children"] = children
|
|
855
|
+
|
|
856
|
+
return node
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
# ---------------------------------------------------------------------------
|
|
860
|
+
# MacosAdapter — PlatformAdapter implementation
|
|
861
|
+
# ---------------------------------------------------------------------------
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
class MacosAdapter(PlatformAdapter):
|
|
865
|
+
"""CUP adapter for macOS via pyobjc AXUIElement API."""
|
|
866
|
+
|
|
867
|
+
@property
|
|
868
|
+
def platform_name(self) -> str:
|
|
869
|
+
return "macos"
|
|
870
|
+
|
|
871
|
+
def initialize(self) -> None:
|
|
872
|
+
pass # pyobjc has no explicit init step
|
|
873
|
+
|
|
874
|
+
def get_screen_info(self) -> tuple[int, int, float]:
|
|
875
|
+
return _macos_screen_info()
|
|
876
|
+
|
|
877
|
+
def get_foreground_window(self) -> dict[str, Any]:
|
|
878
|
+
pid, app_name, bundle_id = _macos_foreground_app()
|
|
879
|
+
win_ref = _macos_focused_window(pid)
|
|
880
|
+
return {
|
|
881
|
+
"handle": win_ref,
|
|
882
|
+
"title": app_name,
|
|
883
|
+
"pid": pid,
|
|
884
|
+
"bundle_id": bundle_id,
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
def get_all_windows(self) -> list[dict[str, Any]]:
|
|
888
|
+
results: list[dict[str, Any]] = []
|
|
889
|
+
apps = _macos_visible_apps()
|
|
890
|
+
|
|
891
|
+
def _enum(app_info):
|
|
892
|
+
p, n, b = app_info
|
|
893
|
+
return [(p, n, b, w) for w in _macos_windows_for_app(p)]
|
|
894
|
+
|
|
895
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
896
|
+
for batch in pool.map(_enum, apps):
|
|
897
|
+
for pid, name, bid, win_ref in batch:
|
|
898
|
+
results.append(
|
|
899
|
+
{
|
|
900
|
+
"handle": win_ref,
|
|
901
|
+
"title": name,
|
|
902
|
+
"pid": pid,
|
|
903
|
+
"bundle_id": bid,
|
|
904
|
+
}
|
|
905
|
+
)
|
|
906
|
+
return results
|
|
907
|
+
|
|
908
|
+
def get_window_list(self) -> list[dict[str, Any]]:
|
|
909
|
+
fg_pid, _, _ = _macos_foreground_app()
|
|
910
|
+
results: list[dict[str, Any]] = []
|
|
911
|
+
seen_pids: set[int] = set()
|
|
912
|
+
for pid, name, bundle_id in _macos_visible_apps():
|
|
913
|
+
if pid in seen_pids:
|
|
914
|
+
continue
|
|
915
|
+
seen_pids.add(pid)
|
|
916
|
+
results.append(
|
|
917
|
+
{
|
|
918
|
+
"title": name,
|
|
919
|
+
"pid": pid,
|
|
920
|
+
"bundle_id": bundle_id,
|
|
921
|
+
"foreground": pid == fg_pid,
|
|
922
|
+
"bounds": None, # skip AX calls for speed
|
|
923
|
+
}
|
|
924
|
+
)
|
|
925
|
+
return results
|
|
926
|
+
|
|
927
|
+
def get_desktop_window(self) -> dict[str, Any] | None:
|
|
928
|
+
for pid, _name, bundle_id in _macos_visible_apps():
|
|
929
|
+
if bundle_id == "com.apple.finder":
|
|
930
|
+
windows = _macos_windows_for_app(pid)
|
|
931
|
+
for win in windows:
|
|
932
|
+
subrole = _get_attr(win, kAXSubroleAttribute)
|
|
933
|
+
if subrole == "AXDesktop":
|
|
934
|
+
return {
|
|
935
|
+
"handle": win,
|
|
936
|
+
"title": "Desktop",
|
|
937
|
+
"pid": pid,
|
|
938
|
+
"bundle_id": bundle_id,
|
|
939
|
+
}
|
|
940
|
+
# Fallback: first Finder window
|
|
941
|
+
if windows:
|
|
942
|
+
return {
|
|
943
|
+
"handle": windows[0],
|
|
944
|
+
"title": "Desktop",
|
|
945
|
+
"pid": pid,
|
|
946
|
+
"bundle_id": bundle_id,
|
|
947
|
+
}
|
|
948
|
+
return None
|
|
949
|
+
|
|
950
|
+
def capture_tree(
|
|
951
|
+
self,
|
|
952
|
+
windows: list[dict[str, Any]],
|
|
953
|
+
*,
|
|
954
|
+
max_depth: int = 999,
|
|
955
|
+
) -> tuple[list[dict], dict, dict[str, Any]]:
|
|
956
|
+
sw, sh, _ = self.get_screen_info()
|
|
957
|
+
refs: dict[str, Any] = {}
|
|
958
|
+
|
|
959
|
+
if len(windows) <= 1:
|
|
960
|
+
# Single window — walk sequentially (no thread overhead)
|
|
961
|
+
id_gen = itertools.count()
|
|
962
|
+
stats: dict = {"nodes": 0, "max_depth": 0, "roles": {}, "screen_w": sw, "screen_h": sh}
|
|
963
|
+
tree: list[dict] = []
|
|
964
|
+
for win in windows:
|
|
965
|
+
node = walk_tree(win["handle"], 0, max_depth, id_gen, stats, refs)
|
|
966
|
+
if node is not None:
|
|
967
|
+
tree.append(node)
|
|
968
|
+
return tree, stats, refs
|
|
969
|
+
else:
|
|
970
|
+
# Multiple windows — walk in parallel threads.
|
|
971
|
+
# AX API calls release the GIL (C calls via pyobjc), so threads
|
|
972
|
+
# give real parallelism for cross-process attribute reads.
|
|
973
|
+
shared_id_gen = itertools.count()
|
|
974
|
+
merged_stats: dict = {
|
|
975
|
+
"nodes": 0,
|
|
976
|
+
"max_depth": 0,
|
|
977
|
+
"roles": {},
|
|
978
|
+
"screen_w": sw,
|
|
979
|
+
"screen_h": sh,
|
|
980
|
+
}
|
|
981
|
+
tree = []
|
|
982
|
+
|
|
983
|
+
def _walk_one(win):
|
|
984
|
+
local_stats = {
|
|
985
|
+
"nodes": 0,
|
|
986
|
+
"max_depth": 0,
|
|
987
|
+
"roles": {},
|
|
988
|
+
"screen_w": sw,
|
|
989
|
+
"screen_h": sh,
|
|
990
|
+
}
|
|
991
|
+
node = walk_tree(win["handle"], 0, max_depth, shared_id_gen, local_stats, refs)
|
|
992
|
+
return node, local_stats
|
|
993
|
+
|
|
994
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
995
|
+
for node, local_stats in pool.map(_walk_one, windows):
|
|
996
|
+
if node is not None:
|
|
997
|
+
tree.append(node)
|
|
998
|
+
merged_stats["nodes"] += local_stats["nodes"]
|
|
999
|
+
merged_stats["max_depth"] = max(
|
|
1000
|
+
merged_stats["max_depth"], local_stats["max_depth"]
|
|
1001
|
+
)
|
|
1002
|
+
for k, v in local_stats["roles"].items():
|
|
1003
|
+
merged_stats["roles"][k] = merged_stats["roles"].get(k, 0) + v
|
|
1004
|
+
|
|
1005
|
+
return tree, merged_stats, refs
|