computeruseprotocol 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cup/platforms/macos.py ADDED
@@ -0,0 +1,1005 @@
1
+ """
2
+ macOS AXUIElement platform adapter for CUP.
3
+
4
+ Captures the accessibility tree via pyobjc AXUIElement API and maps it to the
5
+ canonical CUP schema — roles, states, actions, and platform metadata.
6
+
7
+ Requires macOS accessibility permissions:
8
+ System Settings > Privacy & Security > Accessibility > (add Terminal / Python)
9
+
10
+ Dependencies:
11
+ pip install pyobjc-framework-ApplicationServices pyobjc-framework-Cocoa pyobjc-framework-Quartz
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import concurrent.futures
17
+ import itertools
18
+ from typing import Any
19
+
20
+ from AppKit import NSApplicationActivationPolicyRegular, NSArray, NSScreen, NSWorkspace
21
+ from ApplicationServices import (
22
+ AXUIElementCopyActionNames,
23
+ AXUIElementCopyAttributeValue,
24
+ AXUIElementCopyMultipleAttributeValues,
25
+ AXUIElementCreateApplication,
26
+ AXUIElementIsAttributeSettable,
27
+ AXValueGetType,
28
+ AXValueGetValue,
29
+ kAXChildrenAttribute,
30
+ kAXDescriptionAttribute,
31
+ kAXElementBusyAttribute,
32
+ kAXEnabledAttribute,
33
+ kAXErrorSuccess,
34
+ kAXExpandedAttribute,
35
+ kAXFocusedAttribute,
36
+ kAXFocusedWindowAttribute,
37
+ kAXHelpAttribute,
38
+ kAXIdentifierAttribute,
39
+ kAXMainWindowAttribute,
40
+ kAXModalAttribute,
41
+ kAXPositionAttribute,
42
+ kAXRoleAttribute,
43
+ kAXSelectedAttribute,
44
+ kAXSizeAttribute,
45
+ kAXSubroleAttribute,
46
+ kAXTitleAttribute,
47
+ kAXValueAttribute,
48
+ kAXValueCGPointType,
49
+ kAXValueCGSizeType,
50
+ kAXWindowsAttribute,
51
+ )
52
+
53
+ from cup._base import PlatformAdapter
54
+
55
+ # ---------------------------------------------------------------------------
56
+ # AXRole -> CUP role mapping
57
+ # ---------------------------------------------------------------------------
58
+
59
+ # Primary: AXRole string -> CUP role
60
+ CUP_ROLES: dict[str, str] = {
61
+ "AXApplication": "application",
62
+ "AXWindow": "window",
63
+ "AXButton": "button",
64
+ "AXCheckBox": "checkbox",
65
+ "AXRadioButton": "radio",
66
+ "AXComboBox": "combobox",
67
+ "AXPopUpButton": "combobox",
68
+ "AXTextField": "textbox",
69
+ "AXTextArea": "textbox",
70
+ "AXStaticText": "text",
71
+ "AXImage": "img",
72
+ "AXLink": "link",
73
+ "AXList": "list",
74
+ "AXOutline": "tree",
75
+ "AXTable": "table",
76
+ "AXTabGroup": "tablist",
77
+ "AXSlider": "slider",
78
+ "AXProgressIndicator": "progressbar",
79
+ "AXMenu": "menu",
80
+ "AXMenuBar": "menubar",
81
+ "AXMenuBarItem": "menuitem",
82
+ "AXMenuItem": "menuitem",
83
+ "AXToolbar": "toolbar",
84
+ "AXScrollBar": "scrollbar",
85
+ "AXScrollArea": "generic",
86
+ "AXGroup": "group",
87
+ "AXSplitGroup": "group",
88
+ "AXSplitter": "separator",
89
+ "AXHeading": "heading",
90
+ "AXWebArea": "document",
91
+ "AXCell": "cell",
92
+ "AXRow": "row",
93
+ "AXColumn": "columnheader",
94
+ "AXSheet": "alertdialog",
95
+ "AXDrawer": "complementary",
96
+ "AXGrowArea": "generic",
97
+ "AXValueIndicator": "generic",
98
+ "AXIncrementor": "spinbutton",
99
+ "AXHelpTag": "tooltip",
100
+ "AXColorWell": "button",
101
+ "AXDisclosureTriangle": "button",
102
+ "AXDateField": "textbox",
103
+ "AXBrowser": "tree",
104
+ "AXBusyIndicator": "progressbar",
105
+ "AXRuler": "generic",
106
+ "AXRulerMarker": "generic",
107
+ "AXRelevanceIndicator": "progressbar",
108
+ "AXLevelIndicator": "slider",
109
+ "AXLayoutArea": "group",
110
+ "AXLayoutItem": "generic",
111
+ "AXHandle": "generic",
112
+ "AXMatte": "generic",
113
+ "AXUnknown": "generic",
114
+ "AXListMarker": "text",
115
+ "AXMenuButton": "button",
116
+ "AXRadioGroup": "group",
117
+ }
118
+
119
+ # Subrole refinements: (AXRole, AXSubrole) -> CUP role
120
+ CUP_SUBROLE_OVERRIDES: dict[tuple[str, str], str] = {
121
+ # AXGroup subroles
122
+ ("AXGroup", "AXApplicationAlert"): "alert",
123
+ ("AXGroup", "AXApplicationDialog"): "dialog",
124
+ ("AXGroup", "AXApplicationStatus"): "status",
125
+ ("AXGroup", "AXLandmarkNavigation"): "navigation",
126
+ ("AXGroup", "AXLandmarkSearch"): "search",
127
+ ("AXGroup", "AXLandmarkRegion"): "region",
128
+ ("AXGroup", "AXLandmarkMain"): "main",
129
+ ("AXGroup", "AXLandmarkComplementary"): "complementary",
130
+ ("AXGroup", "AXLandmarkContentInfo"): "contentinfo",
131
+ ("AXGroup", "AXLandmarkBanner"): "banner",
132
+ ("AXGroup", "AXDocument"): "document",
133
+ ("AXGroup", "AXWebApplication"): "application",
134
+ ("AXGroup", "AXTab"): "tabpanel",
135
+ # AXWindow subroles
136
+ ("AXWindow", "AXDialog"): "dialog",
137
+ ("AXWindow", "AXFloatingWindow"): "dialog",
138
+ ("AXWindow", "AXSystemDialog"): "dialog",
139
+ ("AXWindow", "AXSystemFloatingWindow"): "dialog",
140
+ # AXButton subroles
141
+ ("AXButton", "AXCloseButton"): "button",
142
+ ("AXButton", "AXMinimizeButton"): "button",
143
+ ("AXButton", "AXFullScreenButton"): "button",
144
+ # AXRadioButton used as tab
145
+ ("AXRadioButton", "AXTabButton"): "tab",
146
+ # AXMenuItem subroles
147
+ ("AXMenuItem", "AXMenuItemCheckbox"): "menuitemcheckbox",
148
+ ("AXMenuItem", "AXMenuItemRadio"): "menuitemradio",
149
+ # AXTextField subroles
150
+ ("AXTextField", "AXSearchField"): "searchbox",
151
+ ("AXTextField", "AXSecureTextField"): "textbox",
152
+ # AXStaticText as status
153
+ ("AXStaticText", "AXApplicationStatus"): "status",
154
+ # AXRow in outlines -> treeitem (parity with Windows TreeItem)
155
+ ("AXRow", "AXOutlineRow"): "treeitem",
156
+ # AXCheckBox as toggle switch
157
+ ("AXCheckBox", "AXToggle"): "switch",
158
+ ("AXCheckBox", "AXSwitch"): "switch",
159
+ }
160
+
161
+ # Roles that accept text input
162
+ TEXT_INPUT_ROLES = {"textbox", "searchbox", "combobox", "document"}
163
+
164
+ # Roles representing toggle-like elements
165
+ TOGGLE_ROLES = {"checkbox", "switch", "menuitemcheckbox"}
166
+
167
+ # AX roles where AXExpanded is semantically meaningful.
168
+ # Chromium/Electron apps set AXExpanded on nearly every element, so we
169
+ # restrict this to AX roles that genuinely expand/collapse.
170
+ EXPANDABLE_AX_ROLES = {
171
+ "AXComboBox",
172
+ "AXPopUpButton",
173
+ "AXOutline",
174
+ "AXDisclosureTriangle",
175
+ "AXMenu",
176
+ "AXMenuItem",
177
+ "AXMenuBarItem",
178
+ "AXRow",
179
+ "AXBrowser",
180
+ "AXSheet",
181
+ "AXDrawer",
182
+ "AXTabGroup",
183
+ }
184
+
185
+ # AX roles where AXUIElementCopyActionNames is skipped for performance.
186
+ # These roles never produce meaningful CUP actions from their AX action list
187
+ # (their only AX actions are AXScrollToVisible/AXShowMenu which we skip anyway).
188
+ # Actions like "scroll" for AXScrollArea are derived from the role, not from AX.
189
+ _SKIP_ACTIONS_AX_ROLES = {
190
+ "AXStaticText",
191
+ "AXHeading",
192
+ "AXColumn",
193
+ "AXScrollArea",
194
+ "AXSplitGroup",
195
+ "AXSplitter",
196
+ "AXGrowArea",
197
+ "AXValueIndicator",
198
+ "AXRuler",
199
+ "AXRulerMarker",
200
+ "AXLayoutArea",
201
+ "AXLayoutItem",
202
+ "AXHandle",
203
+ "AXMatte",
204
+ "AXUnknown",
205
+ "AXListMarker",
206
+ "AXBusyIndicator",
207
+ "AXRelevanceIndicator",
208
+ "AXLevelIndicator",
209
+ "AXWebArea",
210
+ # Note: AXImage is NOT skipped — clickable images (e.g. avatars) have AXPress.
211
+ }
212
+
213
+
214
+ # ---------------------------------------------------------------------------
215
+ # AX attribute helpers
216
+ # ---------------------------------------------------------------------------
217
+
218
+ # Attributes to batch-read per element via AXUIElementCopyMultipleAttributeValues.
219
+ # Order matters — indices are used to unpack the results array.
220
+ _BATCH_ATTRS_LIST = [
221
+ kAXRoleAttribute, # 0
222
+ kAXSubroleAttribute, # 1
223
+ kAXTitleAttribute, # 2
224
+ kAXDescriptionAttribute, # 3
225
+ kAXHelpAttribute, # 4
226
+ kAXIdentifierAttribute, # 5
227
+ kAXValueAttribute, # 6
228
+ kAXEnabledAttribute, # 7
229
+ kAXFocusedAttribute, # 8
230
+ kAXSelectedAttribute, # 9
231
+ kAXExpandedAttribute, # 10
232
+ kAXElementBusyAttribute, # 11
233
+ kAXModalAttribute, # 12
234
+ kAXPositionAttribute, # 13
235
+ kAXSizeAttribute, # 14
236
+ "AXRequired", # 15
237
+ "AXIsEditable", # 16
238
+ kAXChildrenAttribute, # 17
239
+ ]
240
+ _BATCH_ATTRS = NSArray.arrayWithArray_(_BATCH_ATTRS_LIST)
241
+ _BATCH_IDX = {name: i for i, name in enumerate(_BATCH_ATTRS_LIST)}
242
+
243
+ # AXValueGetType returns 5 for error sentinels (kAXValueAXErrorType)
244
+ _AX_VALUE_ERROR_TYPE = 5
245
+
246
+
247
+ def _is_ax_error(val) -> bool:
248
+ """Check if a batch-read value is an error sentinel."""
249
+ if val is None:
250
+ return True
251
+ try:
252
+ return AXValueGetType(val) == _AX_VALUE_ERROR_TYPE
253
+ except Exception:
254
+ return False
255
+
256
+
257
+ def _batch_read(element) -> list:
258
+ """Read all standard attributes in one cross-process call.
259
+
260
+ Returns a list of values aligned with _BATCH_ATTRS_LIST.
261
+ Error sentinels are replaced with None.
262
+ """
263
+ try:
264
+ err, values = AXUIElementCopyMultipleAttributeValues(element, _BATCH_ATTRS, 0, None)
265
+ if err != kAXErrorSuccess or values is None:
266
+ return [None] * len(_BATCH_ATTRS_LIST)
267
+ return [None if _is_ax_error(v) else v for v in values]
268
+ except Exception:
269
+ return [None] * len(_BATCH_ATTRS_LIST)
270
+
271
+
272
+ def _get_attr(element, attr: str, default=None):
273
+ """Safely read a single AX attribute (used for non-batched reads)."""
274
+ try:
275
+ err, value = AXUIElementCopyAttributeValue(element, attr, None)
276
+ if err == kAXErrorSuccess and value is not None:
277
+ return value
278
+ except Exception:
279
+ pass
280
+ return default
281
+
282
+
283
+ def _is_settable(element, attr: str) -> bool:
284
+ """Check if an attribute is settable on an element."""
285
+ try:
286
+ err, settable = AXUIElementIsAttributeSettable(element, attr, None)
287
+ if err == kAXErrorSuccess:
288
+ return bool(settable)
289
+ except Exception:
290
+ pass
291
+ return False
292
+
293
+
294
+ def _unpack_bounds(pos_ref, size_ref) -> dict | None:
295
+ """Extract {x, y, w, h} from AXPosition + AXSize value refs."""
296
+ if pos_ref is None or size_ref is None:
297
+ return None
298
+ try:
299
+ _, point = AXValueGetValue(pos_ref, kAXValueCGPointType, None)
300
+ _, size = AXValueGetValue(size_ref, kAXValueCGSizeType, None)
301
+ if point is not None and size is not None:
302
+ return {
303
+ "x": int(point.x),
304
+ "y": int(point.y),
305
+ "w": int(size.width),
306
+ "h": int(size.height),
307
+ }
308
+ except Exception:
309
+ pass
310
+ return None
311
+
312
+
313
+ # ---------------------------------------------------------------------------
314
+ # Screen metrics
315
+ # ---------------------------------------------------------------------------
316
+
317
+
318
+ def _macos_screen_info() -> tuple[int, int, float]:
319
+ """Return (width, height, scale) of the primary display.
320
+
321
+ Width/height are in logical points (macOS coordinates).
322
+ Scale is the backing scale factor (2.0 on Retina, 1.0 on non-Retina).
323
+ """
324
+ screen = NSScreen.mainScreen()
325
+ if screen is None:
326
+ from Quartz import CGDisplayBounds, CGMainDisplayID
327
+
328
+ bounds = CGDisplayBounds(CGMainDisplayID())
329
+ return int(bounds.size.width), int(bounds.size.height), 1.0
330
+ frame = screen.frame()
331
+ scale = screen.backingScaleFactor()
332
+ return int(frame.size.width), int(frame.size.height), float(scale)
333
+
334
+
335
+ # ---------------------------------------------------------------------------
336
+ # Window enumeration
337
+ # ---------------------------------------------------------------------------
338
+
339
+ # Process names that are macOS system daemons with on-screen layer-0 windows
340
+ # but should NOT appear in user-facing app lists.
341
+ _SYSTEM_OWNER_NAMES = frozenset(
342
+ {
343
+ "WindowServer",
344
+ "Dock",
345
+ "SystemUIServer",
346
+ "Control Center",
347
+ "Notification Center",
348
+ "loginwindow",
349
+ "Window Manager",
350
+ "Spotlight",
351
+ }
352
+ )
353
+
354
+
355
+ def _cg_window_apps() -> dict[int, str]:
356
+ """Return {pid: owner_name} for processes with on-screen, normal-layer windows.
357
+
358
+ Uses CGWindowListCopyWindowInfo which always returns fresh data from the
359
+ window server, unlike NSWorkspace.runningApplications() which can be stale
360
+ in a long-running process without an NSRunLoop.
361
+ """
362
+ try:
363
+ from Quartz import (
364
+ CGWindowListCopyWindowInfo,
365
+ kCGNullWindowID,
366
+ kCGWindowListOptionOnScreenOnly,
367
+ )
368
+
369
+ cg_windows = CGWindowListCopyWindowInfo(
370
+ kCGWindowListOptionOnScreenOnly,
371
+ kCGNullWindowID,
372
+ )
373
+ if not cg_windows:
374
+ return {}
375
+
376
+ result: dict[int, str] = {}
377
+ for w in cg_windows:
378
+ # Only normal-level windows (layer 0 = kCGNormalWindowLevel).
379
+ # Filters out menus, tooltips, overlays, screensavers, etc.
380
+ layer = w.get("kCGWindowLayer", -1)
381
+ if layer != 0:
382
+ continue
383
+
384
+ pid = w.get("kCGWindowOwnerPID")
385
+ owner = w.get("kCGWindowOwnerName", "")
386
+ if not pid or not owner:
387
+ continue
388
+
389
+ if owner in _SYSTEM_OWNER_NAMES:
390
+ continue
391
+
392
+ if pid not in result:
393
+ result[pid] = owner
394
+
395
+ return result
396
+ except Exception:
397
+ return {}
398
+
399
+
400
+ def _macos_foreground_app() -> tuple[int, str, str | None]:
401
+ """Return (pid, app_name, bundle_id) of the frontmost application.
402
+
403
+ Uses CGWindowListCopyWindowInfo to get fresh data from the window server.
404
+ NSWorkspace.frontmostApplication() goes stale in long-running processes
405
+ without an active NSRunLoop (e.g., MCP servers), so we only use it as a
406
+ fallback.
407
+ """
408
+ # CGWindowList returns windows in front-to-back order. The first
409
+ # layer-0 (normal) window that isn't a system daemon is the frontmost app.
410
+ try:
411
+ from Quartz import (
412
+ CGWindowListCopyWindowInfo,
413
+ kCGNullWindowID,
414
+ kCGWindowListOptionOnScreenOnly,
415
+ )
416
+
417
+ cg_windows = CGWindowListCopyWindowInfo(
418
+ kCGWindowListOptionOnScreenOnly,
419
+ kCGNullWindowID,
420
+ )
421
+ if cg_windows:
422
+ for w in cg_windows:
423
+ if w.get("kCGWindowLayer", -1) != 0:
424
+ continue
425
+ pid = w.get("kCGWindowOwnerPID")
426
+ owner = w.get("kCGWindowOwnerName", "")
427
+ if not pid or not owner:
428
+ continue
429
+ if owner in _SYSTEM_OWNER_NAMES:
430
+ continue
431
+ # Found the frontmost app — look up bundle ID via NSRunningApplication
432
+ bundle_id = None
433
+ try:
434
+ from AppKit import NSRunningApplication
435
+
436
+ ns_app = NSRunningApplication.runningApplicationWithProcessIdentifier_(pid)
437
+ if ns_app is not None:
438
+ owner = ns_app.localizedName() or owner
439
+ bundle_id = ns_app.bundleIdentifier()
440
+ except Exception:
441
+ pass
442
+ return (pid, owner, bundle_id)
443
+ except Exception:
444
+ pass
445
+
446
+ # Fallback: NSWorkspace (may be stale without NSRunLoop)
447
+ workspace = NSWorkspace.sharedWorkspace()
448
+ app = workspace.frontmostApplication()
449
+ return (
450
+ app.processIdentifier(),
451
+ app.localizedName() or "",
452
+ app.bundleIdentifier(),
453
+ )
454
+
455
+
456
+ def _macos_visible_apps() -> list[tuple[int, str, str | None]]:
457
+ """Return [(pid, app_name, bundle_id)] for all visible (regular) apps.
458
+
459
+ Combines NSWorkspace.runningApplications() (provides bundle_id and
460
+ activation policy) with CGWindowListCopyWindowInfo (always fresh from
461
+ the window server) to ensure newly launched apps are not missed due to
462
+ stale NSRunLoop state.
463
+ """
464
+ workspace = NSWorkspace.sharedWorkspace()
465
+ apps = []
466
+ seen_pids: set[int] = set()
467
+
468
+ for app in workspace.runningApplications():
469
+ if app.activationPolicy() == NSApplicationActivationPolicyRegular:
470
+ pid = app.processIdentifier()
471
+ apps.append(
472
+ (
473
+ pid,
474
+ app.localizedName() or "",
475
+ app.bundleIdentifier(),
476
+ )
477
+ )
478
+ seen_pids.add(pid)
479
+
480
+ # Cross-check: find apps with visible windows that NSWorkspace missed
481
+ for pid, owner_name in _cg_window_apps().items():
482
+ if pid not in seen_pids:
483
+ apps.append((pid, owner_name, None))
484
+ seen_pids.add(pid)
485
+
486
+ return apps
487
+
488
+
489
+ def _macos_windows_for_app(pid: int):
490
+ """Return list of AXWindow elements for an app, or empty list."""
491
+ app_ref = AXUIElementCreateApplication(pid)
492
+ windows = _get_attr(app_ref, kAXWindowsAttribute)
493
+ if windows is not None:
494
+ return list(windows)
495
+ return []
496
+
497
+
498
+ def _macos_focused_window(pid: int):
499
+ """Return the focused window AXUIElement for an app, or None."""
500
+ app_ref = AXUIElementCreateApplication(pid)
501
+ win = _get_attr(app_ref, kAXFocusedWindowAttribute)
502
+ if win is not None:
503
+ return win
504
+ return _get_attr(app_ref, kAXMainWindowAttribute)
505
+
506
+
507
+ # ---------------------------------------------------------------------------
508
+ # CUP node builder
509
+ # ---------------------------------------------------------------------------
510
+
511
+
512
+ def build_cup_node(element, id_gen, stats: dict) -> tuple[dict, list] | None:
513
+ """Build a CUP-formatted node from a macOS AXUIElement.
514
+
515
+ Uses batch attribute reading for performance — a single cross-process call
516
+ fetches all 18 standard attributes (including children) instead of
517
+ individual calls per attribute.
518
+
519
+ Returns (node_dict, children_refs) or None if the element has no role.
520
+ """
521
+ stats["nodes"] += 1
522
+
523
+ # ── Batch-read all standard attributes in one call ──
524
+ vals = _batch_read(element)
525
+
526
+ # ── Core properties ──
527
+ ax_role = vals[0] # kAXRoleAttribute
528
+ if not ax_role or not isinstance(ax_role, str):
529
+ return None
530
+
531
+ ax_subrole = vals[1] # kAXSubroleAttribute
532
+ if ax_subrole is not None and not isinstance(ax_subrole, str):
533
+ ax_subrole = None
534
+
535
+ title = vals[2] # kAXTitleAttribute
536
+ if title is not None and not isinstance(title, str):
537
+ title = None
538
+
539
+ description = vals[3] # kAXDescriptionAttribute
540
+ if description is not None and not isinstance(description, str):
541
+ description = None
542
+
543
+ help_text = vals[4] # kAXHelpAttribute
544
+ if help_text is not None and not isinstance(help_text, str):
545
+ help_text = None
546
+
547
+ ax_identifier = vals[5] # kAXIdentifierAttribute
548
+ if ax_identifier is not None and not isinstance(ax_identifier, str):
549
+ ax_identifier = None
550
+
551
+ raw_value = vals[6] # kAXValueAttribute
552
+
553
+ # Name: prefer title, fall back to description.
554
+ # For AXStaticText, the visible text is often in AXValue (native macOS apps
555
+ # like System Settings), so use that as final fallback for text elements.
556
+ name = title or description or ""
557
+ if not name and ax_role in ("AXStaticText", "AXHeading"):
558
+ if raw_value is not None and isinstance(raw_value, str):
559
+ name = raw_value
560
+
561
+ # Bounds from AXPosition + AXSize
562
+ bounds = _unpack_bounds(vals[13], vals[14])
563
+
564
+ # Stats tracking
565
+ role_key = f"{ax_role}:{ax_subrole}" if ax_subrole else ax_role
566
+ stats["roles"][role_key] = stats["roles"].get(role_key, 0) + 1
567
+
568
+ # ── Role mapping ──
569
+ role = CUP_SUBROLE_OVERRIDES.get((ax_role, ax_subrole))
570
+ if role is None:
571
+ role = CUP_ROLES.get(ax_role, "generic")
572
+
573
+ # ── State properties (from batch values) ──
574
+ is_enabled_val = vals[7] # kAXEnabledAttribute
575
+ is_enabled = bool(is_enabled_val) if is_enabled_val is not None else True
576
+ is_focused = bool(vals[8]) # kAXFocusedAttribute
577
+ is_selected = bool(vals[9]) # kAXSelectedAttribute
578
+ is_busy = bool(vals[11]) # kAXElementBusyAttribute
579
+ is_modal = bool(vals[12]) # kAXModalAttribute
580
+
581
+ # Expanded state — only meaningful for certain AX roles (Chromium/Electron
582
+ # apps set AXExpanded on nearly every element, causing noise)
583
+ expanded_val = vals[10] # kAXExpandedAttribute
584
+ has_expanded = ax_role in EXPANDABLE_AX_ROLES and expanded_val is not None
585
+ is_expanded = bool(expanded_val) if has_expanded else None
586
+
587
+ # Required (from batch)
588
+ is_required = bool(vals[15]) # AXRequired
589
+
590
+ # Value as string
591
+ val_str = ""
592
+ if raw_value is not None:
593
+ try:
594
+ val_str = str(raw_value)
595
+ except Exception:
596
+ pass
597
+
598
+ # Editable (from batch, with settable fallback)
599
+ is_editable = bool(vals[16]) # AXIsEditable
600
+ if not is_editable and role in TEXT_INPUT_ROLES:
601
+ is_editable = _is_settable(element, kAXValueAttribute)
602
+
603
+ # ── Offscreen detection ──
604
+ # macOS has no IsOffscreen property, so we check bounds against screen rect
605
+ is_offscreen = False
606
+ if bounds:
607
+ screen_w = stats.get("screen_w", 99999)
608
+ screen_h = stats.get("screen_h", 99999)
609
+ bx, by, bw, bh = bounds["x"], bounds["y"], bounds["w"], bounds["h"]
610
+ # Element is offscreen if entirely outside screen or has zero size
611
+ if bw <= 0 or bh <= 0 or bx + bw <= 0 or by + bh <= 0 or bx >= screen_w or by >= screen_h:
612
+ is_offscreen = True
613
+
614
+ # ── Build states list ──
615
+ states: list[str] = []
616
+ if not is_enabled:
617
+ states.append("disabled")
618
+ if is_focused:
619
+ states.append("focused")
620
+ if is_offscreen:
621
+ states.append("offscreen")
622
+ if is_selected:
623
+ states.append("selected")
624
+ if is_busy:
625
+ states.append("busy")
626
+ if is_modal:
627
+ states.append("modal")
628
+ if is_required:
629
+ states.append("required")
630
+ if has_expanded:
631
+ if is_expanded:
632
+ states.append("expanded")
633
+ else:
634
+ states.append("collapsed")
635
+
636
+ # Checked/mixed for toggles
637
+ if role in TOGGLE_ROLES and raw_value is not None:
638
+ try:
639
+ int_val = int(raw_value)
640
+ if int_val == 1:
641
+ states.append("checked")
642
+ elif int_val == 2:
643
+ states.append("mixed")
644
+ except (ValueError, TypeError):
645
+ pass
646
+
647
+ if is_editable:
648
+ states.append("editable")
649
+ elif role in TEXT_INPUT_ROLES and not is_editable:
650
+ states.append("readonly")
651
+
652
+ # ── Actions ──
653
+ # Skip the action names cross-process call for roles that never produce
654
+ # meaningful CUP actions (saves ~30-60% of per-node overhead).
655
+ skip_actions = ax_role in _SKIP_ACTIONS_AX_ROLES or (ax_role == "AXGroup" and not name)
656
+ if skip_actions:
657
+ ax_action_list = []
658
+ else:
659
+ try:
660
+ err, ax_actions = AXUIElementCopyActionNames(element, None)
661
+ ax_action_list = list(ax_actions) if err == kAXErrorSuccess and ax_actions else []
662
+ except Exception:
663
+ ax_action_list = []
664
+
665
+ actions: list[str] = []
666
+ for ax_act in ax_action_list:
667
+ if ax_act == "AXPress":
668
+ if role in TOGGLE_ROLES:
669
+ actions.append("toggle")
670
+ elif role in (
671
+ "listitem",
672
+ "option",
673
+ "tab",
674
+ "treeitem",
675
+ "menuitem",
676
+ "menuitemcheckbox",
677
+ "menuitemradio",
678
+ ):
679
+ actions.append("select")
680
+ else:
681
+ actions.append("click")
682
+ elif ax_act == "AXIncrement":
683
+ actions.append("increment")
684
+ elif ax_act == "AXDecrement":
685
+ actions.append("decrement")
686
+ elif ax_act == "AXCancel":
687
+ actions.append("dismiss")
688
+ elif ax_act == "AXRaise":
689
+ actions.append("focus")
690
+ elif ax_act == "AXConfirm":
691
+ actions.append("click")
692
+ elif ax_act == "AXPick":
693
+ if "select" not in actions:
694
+ actions.append("select")
695
+ # Note: AXScrollToVisible and AXShowMenu are skipped —
696
+ # Chromium/Electron sets them on ~99% of elements as noise.
697
+ # AXScrollToVisible means "scroll parent to show me" (passive),
698
+ # not "I am scrollable". AXShowMenu opens a context menu.
699
+
700
+ # Text input: add type/setvalue if value is settable
701
+ if role in TEXT_INPUT_ROLES and is_editable:
702
+ if "setvalue" not in actions:
703
+ actions.append("setvalue")
704
+ if "type" not in actions:
705
+ actions.append("type")
706
+
707
+ # Expand/collapse from expanded state
708
+ if has_expanded:
709
+ if "expand" not in actions:
710
+ actions.append("expand")
711
+ if "collapse" not in actions:
712
+ actions.append("collapse")
713
+
714
+ # Scroll areas are scrollable containers
715
+ if ax_role == "AXScrollArea" and "scroll" not in actions:
716
+ actions.append("scroll")
717
+
718
+ # Fallback: focusable
719
+ if not actions and is_enabled:
720
+ actions.append("focus")
721
+
722
+ # ── Attributes (read conditionally per role to avoid overhead on all nodes) ──
723
+ attrs: dict = {}
724
+
725
+ # Tree item nesting depth
726
+ if role == "treeitem":
727
+ dl = _get_attr(element, "AXDisclosureLevel")
728
+ if dl is not None:
729
+ try:
730
+ attrs["level"] = int(dl) + 1 # AX is 0-based, CUP is 1-based
731
+ except (ValueError, TypeError):
732
+ pass
733
+
734
+ # Range widget min/max/current
735
+ if role in ("slider", "progressbar", "spinbutton", "scrollbar"):
736
+ min_val = _get_attr(element, "AXMinValue")
737
+ max_val = _get_attr(element, "AXMaxValue")
738
+ if min_val is not None:
739
+ try:
740
+ attrs["valueMin"] = float(min_val)
741
+ except (ValueError, TypeError):
742
+ pass
743
+ if max_val is not None:
744
+ try:
745
+ attrs["valueMax"] = float(max_val)
746
+ except (ValueError, TypeError):
747
+ pass
748
+ if raw_value is not None:
749
+ try:
750
+ attrs["valueNow"] = float(raw_value)
751
+ except (ValueError, TypeError):
752
+ pass
753
+
754
+ # Placeholder text for inputs
755
+ if role in ("textbox", "searchbox", "combobox"):
756
+ placeholder = _get_attr(element, "AXPlaceholderValue")
757
+ if placeholder is not None and isinstance(placeholder, str) and placeholder:
758
+ attrs["placeholder"] = placeholder[:200]
759
+
760
+ # Link URL
761
+ if role == "link":
762
+ url = _get_attr(element, "AXURL")
763
+ if url is not None:
764
+ url_str = str(url)
765
+ if url_str:
766
+ attrs["url"] = url_str[:500]
767
+
768
+ # Orientation
769
+ if role in ("scrollbar", "slider", "separator", "toolbar", "tablist"):
770
+ orientation = _get_attr(element, "AXOrientation")
771
+ if orientation is not None:
772
+ orient_str = str(orientation)
773
+ if "Vertical" in orient_str:
774
+ attrs["orientation"] = "vertical"
775
+ elif "Horizontal" in orient_str:
776
+ attrs["orientation"] = "horizontal"
777
+
778
+ # ── Assemble CUP node ──
779
+ node: dict = {
780
+ "id": f"e{next(id_gen)}",
781
+ "role": role,
782
+ "name": name[:200],
783
+ }
784
+
785
+ # Description: use help text (or description if title was used as name)
786
+ desc_text = help_text if help_text else (description if title and description else "")
787
+ if desc_text:
788
+ node["description"] = desc_text[:200]
789
+ if val_str and role in (
790
+ "textbox",
791
+ "searchbox",
792
+ "combobox",
793
+ "spinbutton",
794
+ "slider",
795
+ "progressbar",
796
+ "document",
797
+ ):
798
+ node["value"] = val_str[:200]
799
+ if bounds:
800
+ node["bounds"] = bounds
801
+ if states:
802
+ node["states"] = states
803
+ if actions:
804
+ node["actions"] = actions
805
+ if attrs:
806
+ node["attributes"] = attrs
807
+
808
+ # ── Platform extension (macOS-specific raw data) ──
809
+ pm: dict = {"axRole": ax_role}
810
+ if ax_subrole:
811
+ pm["axSubrole"] = ax_subrole
812
+ if ax_identifier:
813
+ pm["axIdentifier"] = ax_identifier
814
+ if ax_action_list:
815
+ pm["axActions"] = ax_action_list
816
+ node["platform"] = {"macos": pm}
817
+
818
+ # Children refs from batch (index 17)
819
+ children_refs = vals[17]
820
+ if children_refs is not None:
821
+ children_refs = list(children_refs)
822
+ else:
823
+ children_refs = []
824
+
825
+ return node, children_refs
826
+
827
+
828
+ # ---------------------------------------------------------------------------
829
+ # Tree walker
830
+ # ---------------------------------------------------------------------------
831
+
832
+
833
+ def walk_tree(element, depth: int, max_depth: int, id_gen, stats: dict, refs: dict) -> dict | None:
834
+ """Recursively walk an AXUIElement tree and build CUP nodes."""
835
+ if depth > max_depth:
836
+ return None
837
+
838
+ result = build_cup_node(element, id_gen, stats)
839
+ if result is None:
840
+ return None
841
+ node, children_refs = result
842
+
843
+ refs[node["id"]] = element
844
+
845
+ stats["max_depth"] = max(stats["max_depth"], depth)
846
+
847
+ if depth < max_depth and children_refs:
848
+ children: list[dict] = []
849
+ for child_ref in children_refs:
850
+ child_node = walk_tree(child_ref, depth + 1, max_depth, id_gen, stats, refs)
851
+ if child_node is not None:
852
+ children.append(child_node)
853
+ if children:
854
+ node["children"] = children
855
+
856
+ return node
857
+
858
+
859
+ # ---------------------------------------------------------------------------
860
+ # MacosAdapter — PlatformAdapter implementation
861
+ # ---------------------------------------------------------------------------
862
+
863
+
864
+ class MacosAdapter(PlatformAdapter):
865
+ """CUP adapter for macOS via pyobjc AXUIElement API."""
866
+
867
+ @property
868
+ def platform_name(self) -> str:
869
+ return "macos"
870
+
871
+ def initialize(self) -> None:
872
+ pass # pyobjc has no explicit init step
873
+
874
+ def get_screen_info(self) -> tuple[int, int, float]:
875
+ return _macos_screen_info()
876
+
877
+ def get_foreground_window(self) -> dict[str, Any]:
878
+ pid, app_name, bundle_id = _macos_foreground_app()
879
+ win_ref = _macos_focused_window(pid)
880
+ return {
881
+ "handle": win_ref,
882
+ "title": app_name,
883
+ "pid": pid,
884
+ "bundle_id": bundle_id,
885
+ }
886
+
887
+ def get_all_windows(self) -> list[dict[str, Any]]:
888
+ results: list[dict[str, Any]] = []
889
+ apps = _macos_visible_apps()
890
+
891
+ def _enum(app_info):
892
+ p, n, b = app_info
893
+ return [(p, n, b, w) for w in _macos_windows_for_app(p)]
894
+
895
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
896
+ for batch in pool.map(_enum, apps):
897
+ for pid, name, bid, win_ref in batch:
898
+ results.append(
899
+ {
900
+ "handle": win_ref,
901
+ "title": name,
902
+ "pid": pid,
903
+ "bundle_id": bid,
904
+ }
905
+ )
906
+ return results
907
+
908
+ def get_window_list(self) -> list[dict[str, Any]]:
909
+ fg_pid, _, _ = _macos_foreground_app()
910
+ results: list[dict[str, Any]] = []
911
+ seen_pids: set[int] = set()
912
+ for pid, name, bundle_id in _macos_visible_apps():
913
+ if pid in seen_pids:
914
+ continue
915
+ seen_pids.add(pid)
916
+ results.append(
917
+ {
918
+ "title": name,
919
+ "pid": pid,
920
+ "bundle_id": bundle_id,
921
+ "foreground": pid == fg_pid,
922
+ "bounds": None, # skip AX calls for speed
923
+ }
924
+ )
925
+ return results
926
+
927
+ def get_desktop_window(self) -> dict[str, Any] | None:
928
+ for pid, _name, bundle_id in _macos_visible_apps():
929
+ if bundle_id == "com.apple.finder":
930
+ windows = _macos_windows_for_app(pid)
931
+ for win in windows:
932
+ subrole = _get_attr(win, kAXSubroleAttribute)
933
+ if subrole == "AXDesktop":
934
+ return {
935
+ "handle": win,
936
+ "title": "Desktop",
937
+ "pid": pid,
938
+ "bundle_id": bundle_id,
939
+ }
940
+ # Fallback: first Finder window
941
+ if windows:
942
+ return {
943
+ "handle": windows[0],
944
+ "title": "Desktop",
945
+ "pid": pid,
946
+ "bundle_id": bundle_id,
947
+ }
948
+ return None
949
+
950
+ def capture_tree(
951
+ self,
952
+ windows: list[dict[str, Any]],
953
+ *,
954
+ max_depth: int = 999,
955
+ ) -> tuple[list[dict], dict, dict[str, Any]]:
956
+ sw, sh, _ = self.get_screen_info()
957
+ refs: dict[str, Any] = {}
958
+
959
+ if len(windows) <= 1:
960
+ # Single window — walk sequentially (no thread overhead)
961
+ id_gen = itertools.count()
962
+ stats: dict = {"nodes": 0, "max_depth": 0, "roles": {}, "screen_w": sw, "screen_h": sh}
963
+ tree: list[dict] = []
964
+ for win in windows:
965
+ node = walk_tree(win["handle"], 0, max_depth, id_gen, stats, refs)
966
+ if node is not None:
967
+ tree.append(node)
968
+ return tree, stats, refs
969
+ else:
970
+ # Multiple windows — walk in parallel threads.
971
+ # AX API calls release the GIL (C calls via pyobjc), so threads
972
+ # give real parallelism for cross-process attribute reads.
973
+ shared_id_gen = itertools.count()
974
+ merged_stats: dict = {
975
+ "nodes": 0,
976
+ "max_depth": 0,
977
+ "roles": {},
978
+ "screen_w": sw,
979
+ "screen_h": sh,
980
+ }
981
+ tree = []
982
+
983
+ def _walk_one(win):
984
+ local_stats = {
985
+ "nodes": 0,
986
+ "max_depth": 0,
987
+ "roles": {},
988
+ "screen_w": sw,
989
+ "screen_h": sh,
990
+ }
991
+ node = walk_tree(win["handle"], 0, max_depth, shared_id_gen, local_stats, refs)
992
+ return node, local_stats
993
+
994
+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
995
+ for node, local_stats in pool.map(_walk_one, windows):
996
+ if node is not None:
997
+ tree.append(node)
998
+ merged_stats["nodes"] += local_stats["nodes"]
999
+ merged_stats["max_depth"] = max(
1000
+ merged_stats["max_depth"], local_stats["max_depth"]
1001
+ )
1002
+ for k, v in local_stats["roles"].items():
1003
+ merged_stats["roles"][k] = merged_stats["roles"].get(k, 0) + v
1004
+
1005
+ return tree, merged_stats, refs