pi-ui-extend 1.0.41 → 1.0.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/README.md +50 -13
  2. package/dist/app/app.d.ts +13 -0
  3. package/dist/app/app.js +158 -8
  4. package/dist/app/cli/install.js +2 -2
  5. package/dist/app/commands/command-controller.d.ts +4 -0
  6. package/dist/app/commands/command-controller.js +11 -0
  7. package/dist/app/commands/command-git-actions.d.ts +25 -0
  8. package/dist/app/commands/command-git-actions.js +381 -0
  9. package/dist/app/commands/command-host.d.ts +4 -0
  10. package/dist/app/commands/command-host.js +5 -2
  11. package/dist/app/commands/command-model-actions.d.ts +1 -0
  12. package/dist/app/commands/command-model-actions.js +57 -35
  13. package/dist/app/commands/command-navigation-actions.d.ts +1 -0
  14. package/dist/app/commands/command-navigation-actions.js +16 -3
  15. package/dist/app/commands/command-registry.d.ts +2 -0
  16. package/dist/app/commands/command-registry.js +15 -1
  17. package/dist/app/commands/command-session-actions.js +9 -3
  18. package/dist/app/commands/reload-context-inventory.d.ts +11 -0
  19. package/dist/app/commands/reload-context-inventory.js +70 -0
  20. package/dist/app/extensions/extension-actions-controller.d.ts +1 -0
  21. package/dist/app/extensions/extension-actions-controller.js +6 -0
  22. package/dist/app/extensions/subagent-catalog-state.d.ts +9 -0
  23. package/dist/app/extensions/subagent-catalog-state.js +23 -0
  24. package/dist/app/input/autocomplete-controller.js +61 -34
  25. package/dist/app/input/input-action-controller.d.ts +2 -0
  26. package/dist/app/input/input-action-controller.js +8 -1
  27. package/dist/app/input/input-controller.d.ts +2 -1
  28. package/dist/app/input/input-controller.js +7 -2
  29. package/dist/app/input/prompt-enhancer-controller.js +53 -35
  30. package/dist/app/input/voice-controller.d.ts +41 -45
  31. package/dist/app/input/voice-controller.js +351 -419
  32. package/dist/app/model/model-usage-controller.js +17 -7
  33. package/dist/app/model/model-usage-status.d.ts +4 -1
  34. package/dist/app/model/model-usage-status.js +270 -46
  35. package/dist/app/popup/menu-items-controller.d.ts +13 -3
  36. package/dist/app/popup/menu-items-controller.js +37 -21
  37. package/dist/app/popup/popup-action-controller.d.ts +12 -2
  38. package/dist/app/popup/popup-action-controller.js +77 -24
  39. package/dist/app/popup/popup-menu-controller.d.ts +36 -14
  40. package/dist/app/popup/popup-menu-controller.js +239 -69
  41. package/dist/app/rendering/dcp-stats.d.ts +9 -4
  42. package/dist/app/rendering/dcp-stats.js +40 -425
  43. package/dist/app/rendering/editor-panels.js +10 -4
  44. package/dist/app/rendering/popup-menu-renderer.d.ts +3 -5
  45. package/dist/app/rendering/popup-menu-renderer.js +42 -37
  46. package/dist/app/rendering/render-controller.js +23 -2
  47. package/dist/app/rendering/status-line-renderer.d.ts +5 -0
  48. package/dist/app/rendering/status-line-renderer.js +41 -5
  49. package/dist/app/rendering/tab-line-renderer.js +26 -20
  50. package/dist/app/runtime.d.ts +12 -1
  51. package/dist/app/runtime.js +120 -12
  52. package/dist/app/screen/mouse-controller.d.ts +2 -0
  53. package/dist/app/screen/mouse-controller.js +17 -7
  54. package/dist/app/screen/status-controller.d.ts +4 -0
  55. package/dist/app/screen/status-controller.js +5 -0
  56. package/dist/app/session/lazy-session-manager.js +34 -0
  57. package/dist/app/session/session-event-controller.d.ts +1 -0
  58. package/dist/app/session/session-event-controller.js +10 -1
  59. package/dist/app/session/session-history.d.ts +1 -0
  60. package/dist/app/session/session-history.js +12 -1
  61. package/dist/app/session/session-lifecycle-controller.d.ts +7 -1
  62. package/dist/app/session/session-lifecycle-controller.js +16 -1
  63. package/dist/app/session/tabs-controller.d.ts +26 -1
  64. package/dist/app/session/tabs-controller.js +454 -147
  65. package/dist/app/subagents/subagents-files.js +60 -1
  66. package/dist/app/subagents/subagents-model.d.ts +1 -0
  67. package/dist/app/subagents/subagents-model.js +18 -2
  68. package/dist/app/subagents/subagents-widget-controller.d.ts +1 -0
  69. package/dist/app/subagents/subagents-widget-controller.js +6 -0
  70. package/dist/app/types.d.ts +16 -1
  71. package/dist/app/workspace/workspace-actions-controller.js +10 -2
  72. package/dist/app/workspace/workspace-undo.d.ts +1 -0
  73. package/dist/app/workspace/workspace-undo.js +1 -0
  74. package/dist/bundled-extensions/question/index.js +9 -1
  75. package/dist/bundled-extensions/question/remote.d.ts +4 -0
  76. package/dist/bundled-extensions/question/remote.js +33 -0
  77. package/dist/bundled-extensions/telegram-connector/bot.d.ts +43 -0
  78. package/dist/bundled-extensions/telegram-connector/bot.js +166 -0
  79. package/dist/bundled-extensions/telegram-connector/config.d.ts +8 -0
  80. package/dist/bundled-extensions/telegram-connector/config.js +87 -0
  81. package/dist/bundled-extensions/telegram-connector/coordinator.d.ts +66 -0
  82. package/dist/bundled-extensions/telegram-connector/coordinator.js +413 -0
  83. package/dist/bundled-extensions/telegram-connector/index.d.ts +3 -0
  84. package/dist/bundled-extensions/telegram-connector/index.js +195 -0
  85. package/dist/bundled-extensions/terminal-bell/index.d.ts +0 -8
  86. package/dist/bundled-extensions/terminal-bell/index.js +0 -76
  87. package/dist/bundled-extensions/workspace-undo/index.d.ts +26 -0
  88. package/dist/bundled-extensions/workspace-undo/index.js +191 -0
  89. package/dist/config.d.ts +21 -2
  90. package/dist/config.js +159 -32
  91. package/dist/default-pix-config.js +25 -5
  92. package/dist/schemas/index.d.ts +1 -0
  93. package/dist/schemas/index.js +1 -0
  94. package/dist/schemas/pi-tools-suite-schema.d.ts +88 -62
  95. package/dist/schemas/pi-tools-suite-schema.js +54 -83
  96. package/dist/schemas/pix-schema.d.ts +19 -2
  97. package/dist/schemas/pix-schema.js +47 -5
  98. package/dist/schemas/tasks-schema.d.ts +18 -0
  99. package/dist/schemas/tasks-schema.js +41 -0
  100. package/docs/concurrency.md +9 -1
  101. package/docs/desktop-mvp.md +22 -6
  102. package/docs/desktop-task-manager.md +148 -84
  103. package/docs/release.md +30 -6
  104. package/external/pi-tools-suite/README.md +197 -84
  105. package/external/pi-tools-suite/docs/evals.md +1 -1
  106. package/external/pi-tools-suite/docs/session-recovery.md +47 -14
  107. package/external/pi-tools-suite/docs/subagent-model-pools.md +48 -30
  108. package/external/pi-tools-suite/docs/ui-qa-subagent.md +440 -0
  109. package/external/pi-tools-suite/package.json +6 -1
  110. package/external/pi-tools-suite/src/antigravity-auth/auth-store.ts +2 -1
  111. package/external/pi-tools-suite/src/antigravity-auth/constants.ts +9 -3
  112. package/external/pi-tools-suite/src/antigravity-auth/headers.ts +41 -3
  113. package/external/pi-tools-suite/src/antigravity-auth/models.ts +94 -21
  114. package/external/pi-tools-suite/src/antigravity-auth/oauth.ts +5 -17
  115. package/external/pi-tools-suite/src/antigravity-auth/payload.ts +72 -7
  116. package/external/pi-tools-suite/src/antigravity-auth/stream.ts +13 -1
  117. package/external/pi-tools-suite/src/async-subagents/agents/frontier-review.md +23 -0
  118. package/external/pi-tools-suite/src/async-subagents/agents/implement.md +1 -1
  119. package/external/pi-tools-suite/src/async-subagents/agents/presets.jsonc +16 -0
  120. package/external/pi-tools-suite/src/async-subagents/agents/research.md +5 -3
  121. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/browser.mjs +346 -0
  122. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/desktop.mjs +1038 -0
  123. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/tui.mjs +759 -0
  124. package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/scripts/browser-qa-runner.mjs +32 -23
  125. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/chrome-devtools/chrome-devtools-provider.mjs +895 -0
  126. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/linux/linux-atspi.py +494 -0
  127. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/macos/macos-accessibility.swift +1085 -0
  128. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/bridge-client.mjs +50 -0
  129. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/native-terminal-host.mjs +501 -0
  130. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/windows/windows-uia.ps1 +454 -0
  131. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-auth.md +114 -0
  132. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-chrome-devtools.md +91 -0
  133. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-playwright.md +145 -0
  134. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser.md +82 -0
  135. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-linux.md +26 -0
  136. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-macos.md +29 -0
  137. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-windows.md +23 -0
  138. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop.md +54 -0
  139. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-native-terminal.md +59 -0
  140. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-pty.md +37 -0
  141. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui.md +60 -0
  142. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/scripts/ui-qa-runner.mjs +535 -0
  143. package/external/pi-tools-suite/src/async-subagents/agents/ui-qa.md +75 -0
  144. package/external/pi-tools-suite/src/async-subagents/commands.ts +15 -71
  145. package/external/pi-tools-suite/src/async-subagents/core/activity.ts +33 -0
  146. package/external/pi-tools-suite/src/async-subagents/core/agent-catalog.ts +5 -4
  147. package/external/pi-tools-suite/src/async-subagents/core/agent-strategy.ts +1 -1
  148. package/external/pi-tools-suite/src/async-subagents/core/agents-dir.ts +8 -5
  149. package/external/pi-tools-suite/src/async-subagents/core/browser-qa.ts +16 -2
  150. package/external/pi-tools-suite/src/async-subagents/core/config.ts +160 -327
  151. package/external/pi-tools-suite/src/async-subagents/core/model-selection.ts +2 -1
  152. package/external/pi-tools-suite/src/async-subagents/core/registry.ts +158 -33
  153. package/external/pi-tools-suite/src/async-subagents/core/routing.ts +21 -5
  154. package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +40 -26
  155. package/external/pi-tools-suite/src/async-subagents/core/state.ts +40 -0
  156. package/external/pi-tools-suite/src/async-subagents/core/types.ts +7 -0
  157. package/external/pi-tools-suite/src/async-subagents/core/ultrawork-auto.ts +51 -45
  158. package/external/pi-tools-suite/src/async-subagents/index.ts +69 -5
  159. package/external/pi-tools-suite/src/async-subagents/lib.ts +16 -9
  160. package/external/pi-tools-suite/src/async-subagents/tools/spawn.ts +8 -5
  161. package/external/pi-tools-suite/src/async-subagents/tools/subagents.ts +5 -4
  162. package/external/pi-tools-suite/src/async-subagents/types.ts +1 -0
  163. package/external/pi-tools-suite/src/coding-discipline/index.ts +90 -59
  164. package/external/pi-tools-suite/src/config.ts +55 -1
  165. package/external/pi-tools-suite/src/context-gateway/accounting-log.ts +282 -0
  166. package/external/pi-tools-suite/src/context-gateway/config.ts +77 -3
  167. package/external/pi-tools-suite/src/context-gateway/efficiency.ts +414 -0
  168. package/external/pi-tools-suite/src/context-gateway/enforcement.ts +222 -0
  169. package/external/pi-tools-suite/src/context-gateway/index.ts +206 -19
  170. package/external/pi-tools-suite/src/context-gateway/storeless-capabilities.ts +7 -7
  171. package/external/pi-tools-suite/src/context-gateway/telemetry.ts +101 -28
  172. package/external/pi-tools-suite/src/context-gateway/types.ts +16 -3
  173. package/external/pi-tools-suite/src/context-inventory.ts +99 -0
  174. package/external/pi-tools-suite/src/dcp/auto-compress-budget.ts +39 -4
  175. package/external/pi-tools-suite/src/dcp/auto-compress.ts +128 -49
  176. package/external/pi-tools-suite/src/dcp/commands.ts +32 -156
  177. package/external/pi-tools-suite/src/dcp/compress-tool.ts +177 -46
  178. package/external/pi-tools-suite/src/dcp/compression-blocks.ts +6 -58
  179. package/external/pi-tools-suite/src/dcp/compression-preview.ts +9 -0
  180. package/external/pi-tools-suite/src/dcp/compression-progress.ts +22 -0
  181. package/external/pi-tools-suite/src/dcp/config.ts +114 -17
  182. package/external/pi-tools-suite/src/dcp/conversation-index.ts +36 -9
  183. package/external/pi-tools-suite/src/dcp/diagnostics.ts +41 -0
  184. package/external/pi-tools-suite/src/dcp/fresh-tool-results.ts +38 -0
  185. package/external/pi-tools-suite/src/dcp/index.ts +338 -66
  186. package/external/pi-tools-suite/src/dcp/journal.ts +126 -4
  187. package/external/pi-tools-suite/src/dcp/progress-controller.ts +2 -1
  188. package/external/pi-tools-suite/src/dcp/prompts.ts +83 -192
  189. package/external/pi-tools-suite/src/dcp/protected-continuity.ts +175 -0
  190. package/external/pi-tools-suite/src/dcp/pruner-candidates.ts +93 -3
  191. package/external/pi-tools-suite/src/dcp/pruner-message-ids.ts +6 -10
  192. package/external/pi-tools-suite/src/dcp/pruner-nudge.ts +36 -33
  193. package/external/pi-tools-suite/src/dcp/pruner-tools.ts +6 -3
  194. package/external/pi-tools-suite/src/dcp/pruner.ts +1 -0
  195. package/external/pi-tools-suite/src/dcp/routine-pressure.ts +86 -0
  196. package/external/pi-tools-suite/src/dcp/state.ts +9 -0
  197. package/external/pi-tools-suite/src/dcp/statistics.d.ts +8 -0
  198. package/external/pi-tools-suite/src/dcp/statistics.js +156 -0
  199. package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +78 -22
  200. package/external/pi-tools-suite/src/index.ts +7 -1
  201. package/external/pi-tools-suite/src/lib/project.ts +36 -1
  202. package/external/pi-tools-suite/src/model-tools/index.ts +10 -7
  203. package/external/pi-tools-suite/src/repo-discovery/index.ts +304 -4
  204. package/external/pi-tools-suite/src/resource-registry/index.ts +2551 -0
  205. package/external/pi-tools-suite/src/session-recovery/index.ts +17 -0
  206. package/external/pi-tools-suite/src/shell-command-policy.ts +219 -0
  207. package/external/pi-tools-suite/src/todo/index.ts +21 -0
  208. package/external/pi-tools-suite/src/todo/todo.ts +19 -1
  209. package/external/pi-tools-suite/src/tool-descriptions.ts +47 -19
  210. package/package.json +10 -9
  211. package/schemas/pi-tools-suite.json +466 -287
  212. package/schemas/pix.json +129 -11
  213. package/schemas/tasks.json +131 -0
  214. package/skills/simplify/SKILL.md +33 -5
  215. package/docs/desktop-markdown-media.md +0 -77
  216. package/external/pi-tools-suite/docs/browser-qa-subagent.md +0 -177
  217. package/external/pi-tools-suite/docs/dcp-emergency-current-turn.md +0 -102
  218. package/external/pi-tools-suite/src/async-subagents/agents/browser-qa.md +0 -598
  219. package/external/pi-tools-suite/src/async-subagents/async-subagents.sample.jsonc +0 -54
  220. package/external/pi-tools-suite/src/skill-installer/index.ts +0 -333
  221. package/skills/playwright-cli/SKILL.md +0 -420
  222. package/skills/playwright-cli/references/element-attributes.md +0 -23
  223. package/skills/playwright-cli/references/playwright-tests.md +0 -50
  224. package/skills/playwright-cli/references/request-mocking.md +0 -87
  225. package/skills/playwright-cli/references/running-code.md +0 -241
  226. package/skills/playwright-cli/references/session-management.md +0 -273
  227. package/skills/playwright-cli/references/spec-driven-testing.md +0 -311
  228. package/skills/playwright-cli/references/storage-state.md +0 -290
  229. package/skills/playwright-cli/references/test-generation.md +0 -142
  230. package/skills/playwright-cli/references/tracing.md +0 -154
  231. package/skills/playwright-cli/references/video-recording.md +0 -147
  232. package/skills/spec-lite/SKILL.md +0 -140
  233. /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-auth.example.jsonc +0 -0
  234. /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-flow.example.jsonc +0 -0
  235. /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.LICENSE +0 -0
  236. /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.mjs +0 -0
@@ -0,0 +1,1085 @@
1
+ import AppKit
2
+ import ApplicationServices
3
+ import AVFoundation
4
+ import CoreMedia
5
+ import Darwin
6
+ import Foundation
7
+ import ScreenCaptureKit
8
+
9
+ struct Failure: Error, CustomStringConvertible {
10
+ let description: String
11
+
12
+ init(_ description: String) {
13
+ self.description = description
14
+ }
15
+ }
16
+
17
+ struct Arguments {
18
+ let command: String
19
+ private let options: [String: String]
20
+ private let flags: Set<String>
21
+
22
+ init(_ values: [String]) throws {
23
+ guard let command = values.first else {
24
+ throw Failure(Self.usage)
25
+ }
26
+ self.command = command
27
+
28
+ let valueOptions: Set<String> = [
29
+ "--app", "--bundle-id", "--pid", "--pgid", "--depth", "--limit", "--match",
30
+ "--role", "--occurrence", "--path", "--value", "--text",
31
+ "--key", "--modifiers", "--x", "--y", "--out", "--timeout", "--duration",
32
+ ]
33
+ var parsedOptions: [String: String] = [:]
34
+ var parsedFlags = Set<String>()
35
+ var index = 1
36
+ while index < values.count {
37
+ let argument = values[index]
38
+ guard argument.hasPrefix("--") else {
39
+ throw Failure("unexpected argument: \(argument)")
40
+ }
41
+ if valueOptions.contains(argument) {
42
+ guard index + 1 < values.count else {
43
+ throw Failure("missing value for \(argument)")
44
+ }
45
+ parsedOptions[argument] = values[index + 1]
46
+ index += 2
47
+ } else {
48
+ parsedFlags.insert(argument)
49
+ index += 1
50
+ }
51
+ }
52
+ options = parsedOptions
53
+ flags = parsedFlags
54
+ }
55
+
56
+ func value(_ name: String) -> String? {
57
+ options["--\(name)"]
58
+ }
59
+
60
+ func required(_ name: String) throws -> String {
61
+ guard let value = value(name), !value.isEmpty else {
62
+ throw Failure("missing --\(name)")
63
+ }
64
+ return value
65
+ }
66
+
67
+ func integer(_ name: String, default fallback: Int) throws -> Int {
68
+ guard let raw = value(name) else { return fallback }
69
+ guard let parsed = Int(raw) else {
70
+ throw Failure("invalid integer for --\(name): \(raw)")
71
+ }
72
+ return parsed
73
+ }
74
+
75
+ func double(_ name: String, default fallback: Double? = nil) throws -> Double {
76
+ if let raw = value(name), let parsed = Double(raw) {
77
+ return parsed
78
+ }
79
+ if let fallback { return fallback }
80
+ throw Failure("missing or invalid --\(name)")
81
+ }
82
+
83
+ func has(_ name: String) -> Bool {
84
+ flags.contains("--\(name)")
85
+ }
86
+
87
+ static let usage = """
88
+ usage: macos-window.sh COMMAND [options]
89
+ doctor [--prompt]
90
+ list
91
+ App selectors: --pid PID (preferred), --pgid PGID, --app NAME, --bundle-id ID
92
+ wait-window SELECTOR [--timeout 60] [--print-pid]
93
+ focus SELECTOR
94
+ inspect SELECTOR [--depth 10] [--limit 80] [--all]
95
+ describe SELECTOR (--path PATH | --match TEXT [--role ROLE] [--occurrence N])
96
+ screenshot SELECTOR --out FILE
97
+ window-id SELECTOR
98
+ record-window SELECTOR --out FILE [--duration SECONDS]
99
+ click SELECTOR (--path PATH | --match TEXT [--role ROLE] [--occurrence N]) [--frame-fallback]
100
+ set-value SELECTOR (--path PATH | --match TEXT) --value TEXT
101
+ type SELECTOR --text TEXT
102
+ key SELECTOR --key KEY [--modifiers cmd,shift,ctrl,opt]
103
+ click-point SELECTOR --x X --y Y
104
+ quit SELECTOR [--force]
105
+ """
106
+ }
107
+
108
+ func copyAttribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
109
+ var value: CFTypeRef?
110
+ guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else {
111
+ return nil
112
+ }
113
+ return value
114
+ }
115
+
116
+ func stringAttribute(_ element: AXUIElement, _ name: String) -> String? {
117
+ copyAttribute(element, name) as? String
118
+ }
119
+
120
+ func scalarAttribute(_ element: AXUIElement, _ name: String) -> String? {
121
+ guard let value = copyAttribute(element, name) else { return nil }
122
+ if let string = value as? String { return string }
123
+ if CFGetTypeID(value) == CFBooleanGetTypeID(), let boolean = value as? NSNumber {
124
+ return boolean.boolValue ? "true" : "false"
125
+ }
126
+ if let number = value as? NSNumber { return number.stringValue }
127
+ return nil
128
+ }
129
+
130
+ func children(_ element: AXUIElement) -> [AXUIElement] {
131
+ copyAttribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? []
132
+ }
133
+
134
+ func pointAttribute(_ element: AXUIElement, _ name: String) -> CGPoint? {
135
+ guard let raw = copyAttribute(element, name), CFGetTypeID(raw) == AXValueGetTypeID() else {
136
+ return nil
137
+ }
138
+ var point = CGPoint.zero
139
+ guard AXValueGetValue(raw as! AXValue, .cgPoint, &point) else { return nil }
140
+ return point
141
+ }
142
+
143
+ func sizeAttribute(_ element: AXUIElement, _ name: String) -> CGSize? {
144
+ guard let raw = copyAttribute(element, name), CFGetTypeID(raw) == AXValueGetTypeID() else {
145
+ return nil
146
+ }
147
+ var size = CGSize.zero
148
+ guard AXValueGetValue(raw as! AXValue, .cgSize, &size) else { return nil }
149
+ return size
150
+ }
151
+
152
+ func frame(_ element: AXUIElement) -> CGRect? {
153
+ guard let position = pointAttribute(element, kAXPositionAttribute),
154
+ let size = sizeAttribute(element, kAXSizeAttribute)
155
+ else { return nil }
156
+ return CGRect(origin: position, size: size)
157
+ }
158
+
159
+ func compact(_ value: String?, limit: Int = 72) -> String? {
160
+ guard let value else { return nil }
161
+ let singleLine = value
162
+ .replacingOccurrences(of: "\n", with: " ")
163
+ .replacingOccurrences(of: "\r", with: " ")
164
+ .trimmingCharacters(in: .whitespacesAndNewlines)
165
+ guard !singleLine.isEmpty else { return nil }
166
+ if singleLine.count <= limit { return singleLine }
167
+ return String(singleLine.prefix(limit - 1)) + "…"
168
+ }
169
+
170
+ func role(_ element: AXUIElement) -> String {
171
+ stringAttribute(element, kAXRoleAttribute) ?? "AXUnknown"
172
+ }
173
+
174
+ func fields(_ element: AXUIElement) -> [(String, String)] {
175
+ let elementRole = role(element)
176
+ var result: [(String, String)] = []
177
+ let pairs = [
178
+ ("title", kAXTitleAttribute),
179
+ ("description", kAXDescriptionAttribute),
180
+ ("identifier", kAXIdentifierAttribute),
181
+ ("help", kAXHelpAttribute),
182
+ ]
183
+ for (label, attribute) in pairs {
184
+ if let value = compact(stringAttribute(element, attribute)) {
185
+ result.append((label, value))
186
+ }
187
+ }
188
+ if elementRole == "AXSecureTextField" {
189
+ result.append(("value", "<redacted>"))
190
+ } else if let value = compact(scalarAttribute(element, kAXValueAttribute)) {
191
+ result.append(("value", value))
192
+ }
193
+ for (label, attribute) in [
194
+ ("enabled", kAXEnabledAttribute),
195
+ ("focused", kAXFocusedAttribute),
196
+ ("selected", kAXSelectedAttribute),
197
+ ] {
198
+ if let value = scalarAttribute(element, attribute) {
199
+ result.append((label, value))
200
+ }
201
+ }
202
+ return result
203
+ }
204
+
205
+ func quoted(_ value: String) -> String {
206
+ "\"" + value.replacingOccurrences(of: "\"", with: "\\\"") + "\""
207
+ }
208
+
209
+ func summary(_ element: AXUIElement, path: String) -> String {
210
+ var parts = [path, role(element)]
211
+ parts.append(contentsOf: fields(element).map { "\($0.0)=\(quoted($0.1))" })
212
+ if let rectangle = frame(element) {
213
+ parts.append(
214
+ "frame=\(Int(rectangle.origin.x)),\(Int(rectangle.origin.y))," +
215
+ "\(Int(rectangle.width))x\(Int(rectangle.height))"
216
+ )
217
+ }
218
+ return parts.joined(separator: " ")
219
+ }
220
+
221
+ func findApplication(_ arguments: Arguments) throws -> NSRunningApplication {
222
+ let requestedName = arguments.value("app")
223
+ let requestedBundle = arguments.value("bundle-id")
224
+ let requestedPID = try arguments.value("pid").map {
225
+ guard let pid = Int32($0), pid > 0 else { throw Failure("invalid --pid: \($0)") }
226
+ return pid
227
+ }
228
+ let requestedPGID = try arguments.value("pgid").map {
229
+ guard let pgid = Int32($0), pgid > 0 else { throw Failure("invalid --pgid: \($0)") }
230
+ return pgid
231
+ }
232
+ guard requestedName != nil || requestedBundle != nil || requestedPID != nil || requestedPGID != nil else {
233
+ throw Failure("missing app selector (--pid, --pgid, --app, or --bundle-id)")
234
+ }
235
+
236
+ // `NSWorkspace.runningApplications` can remain stale when this helper starts
237
+ // before a newly spawned GUI process has registered with LaunchServices.
238
+ // Resolve an explicit PID directly so wait-window can observe that process
239
+ // once AppKit finishes launching instead of polling a cached empty snapshot.
240
+ var candidates: [NSRunningApplication]
241
+ if let requestedPID, let application = NSRunningApplication(processIdentifier: requestedPID),
242
+ !application.isTerminated
243
+ {
244
+ candidates = [application]
245
+ } else {
246
+ candidates = NSWorkspace.shared.runningApplications.filter { !$0.isTerminated }
247
+ if requestedPID != nil { candidates = [] }
248
+ }
249
+ if let requestedPGID {
250
+ candidates = candidates.filter { getpgid($0.processIdentifier) == requestedPGID }
251
+ }
252
+ if let requestedBundle {
253
+ candidates = candidates.filter { $0.bundleIdentifier == requestedBundle }
254
+ }
255
+ if let requestedName {
256
+ let exact = candidates.filter {
257
+ $0.localizedName?.caseInsensitiveCompare(requestedName) == .orderedSame
258
+ }
259
+ if !exact.isEmpty {
260
+ candidates = exact
261
+ } else {
262
+ let partial = candidates.filter {
263
+ $0.localizedName?.localizedCaseInsensitiveContains(requestedName) == true
264
+ }
265
+ if !partial.isEmpty {
266
+ candidates = partial
267
+ } else if AXIsProcessTrusted() {
268
+ candidates = candidates.filter { app in
269
+ let application = axApplication(app)
270
+ let windows = copyAttribute(application, kAXWindowsAttribute)
271
+ as? [AXUIElement] ?? []
272
+ return windows.contains { window in
273
+ stringAttribute(window, kAXTitleAttribute)?
274
+ .localizedCaseInsensitiveContains(requestedName) == true
275
+ }
276
+ }
277
+ }
278
+ }
279
+ }
280
+ guard !candidates.isEmpty else {
281
+ throw Failure("no running app matches the supplied selector")
282
+ }
283
+ guard candidates.count == 1 else {
284
+ let preview = candidates.prefix(10).map {
285
+ "\($0.localizedName ?? "?") pid=\($0.processIdentifier)"
286
+ }.joined(separator: ", ")
287
+ throw Failure("ambiguous app selector (\(candidates.count) matches): \(preview); use --pid")
288
+ }
289
+ return candidates[0]
290
+ }
291
+
292
+ func axApplication(_ app: NSRunningApplication) -> AXUIElement {
293
+ AXUIElementCreateApplication(app.processIdentifier)
294
+ }
295
+
296
+ func frontWindow(_ app: NSRunningApplication) throws -> AXUIElement {
297
+ let application = axApplication(app)
298
+ if let focused = copyAttribute(application, kAXFocusedWindowAttribute) {
299
+ return unsafeBitCast(focused, to: AXUIElement.self)
300
+ }
301
+ guard let window = (copyAttribute(application, kAXWindowsAttribute) as? [AXUIElement])?.first else {
302
+ throw Failure("no accessible window for \(app.localizedName ?? "app")")
303
+ }
304
+ return window
305
+ }
306
+
307
+ func focusedElement(_ app: NSRunningApplication) throws -> AXUIElement {
308
+ let application = axApplication(app)
309
+ guard let focused = copyAttribute(application, kAXFocusedUIElementAttribute) else {
310
+ throw Failure("app has no focused accessibility element")
311
+ }
312
+ return unsafeBitCast(focused, to: AXUIElement.self)
313
+ }
314
+
315
+ func requireAccessibility() throws {
316
+ guard AXIsProcessTrusted() else {
317
+ throw Failure("Accessibility permission is disabled; run doctor --prompt")
318
+ }
319
+ }
320
+
321
+ struct LocatedElement {
322
+ let element: AXUIElement
323
+ let path: String
324
+ }
325
+
326
+ func printJSON(_ value: Any) throws {
327
+ let data = try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
328
+ guard let encoded = String(data: data, encoding: .utf8) else {
329
+ throw Failure("could not encode accessibility result")
330
+ }
331
+ print(encoded)
332
+ }
333
+
334
+ func record(_ located: LocatedElement) -> [String: Any] {
335
+ var attributes: [String: String] = [:]
336
+ for (name, value) in fields(located.element) { attributes[name] = value }
337
+ var result: [String: Any] = [
338
+ "path": located.path,
339
+ "role": role(located.element),
340
+ "attributes": attributes,
341
+ ]
342
+ if let rectangle = frame(located.element) {
343
+ result["frame"] = [
344
+ "x": rectangle.origin.x,
345
+ "y": rectangle.origin.y,
346
+ "width": rectangle.width,
347
+ "height": rectangle.height,
348
+ ]
349
+ }
350
+ return result
351
+ }
352
+
353
+ func walk(
354
+ _ root: AXUIElement,
355
+ maxDepth: Int,
356
+ maxVisited: Int = 2_000
357
+ ) -> [LocatedElement] {
358
+ var found: [LocatedElement] = []
359
+ var visited = 0
360
+
361
+ func visit(_ element: AXUIElement, path: String, depth: Int) {
362
+ guard visited < maxVisited, depth <= maxDepth else { return }
363
+ visited += 1
364
+ found.append(LocatedElement(element: element, path: path))
365
+ for (index, child) in children(element).enumerated() {
366
+ visit(child, path: "\(path).\(index)", depth: depth + 1)
367
+ }
368
+ }
369
+
370
+ visit(root, path: "0", depth: 0)
371
+ return found
372
+ }
373
+
374
+ func resolvePath(_ root: AXUIElement, path: String) throws -> LocatedElement {
375
+ let components = path.split(separator: ".").compactMap { Int($0) }
376
+ guard components.first == 0, components.count == path.split(separator: ".").count else {
377
+ throw Failure("invalid AX path: \(path)")
378
+ }
379
+ var element = root
380
+ var traversed = "0"
381
+ for index in components.dropFirst() {
382
+ let descendants = children(element)
383
+ guard descendants.indices.contains(index) else {
384
+ throw Failure("stale AX path at \(traversed).\(index); inspect again")
385
+ }
386
+ element = descendants[index]
387
+ traversed += ".\(index)"
388
+ }
389
+ return LocatedElement(element: element, path: traversed)
390
+ }
391
+
392
+ func normalizedRole(_ value: String) -> String {
393
+ value.hasPrefix("AX") ? value : "AX" + value.prefix(1).uppercased() + value.dropFirst()
394
+ }
395
+
396
+ func locate(_ root: AXUIElement, arguments: Arguments) throws -> LocatedElement {
397
+ if let path = arguments.value("path") {
398
+ return try resolvePath(root, path: path)
399
+ }
400
+ let query = try arguments.required("match").lowercased()
401
+ let requestedRole = arguments.value("role").map(normalizedRole)
402
+ let candidates = walk(root, maxDepth: 12).filter { candidate in
403
+ if let requestedRole, role(candidate.element) != requestedRole { return false }
404
+ return fields(candidate.element).contains {
405
+ $0.1.lowercased().contains(query)
406
+ }
407
+ }
408
+ guard !candidates.isEmpty else {
409
+ throw Failure("no accessibility element matches \(quoted(query))")
410
+ }
411
+ if let rawOccurrence = arguments.value("occurrence") {
412
+ guard let occurrence = Int(rawOccurrence), occurrence > 0,
413
+ candidates.indices.contains(occurrence - 1)
414
+ else {
415
+ throw Failure("invalid --occurrence \(rawOccurrence); matches=\(candidates.count)")
416
+ }
417
+ return candidates[occurrence - 1]
418
+ }
419
+ guard candidates.count == 1 else {
420
+ let preview = candidates.prefix(10).map {
421
+ " " + summary($0.element, path: $0.path)
422
+ }.joined(separator: "\n")
423
+ throw Failure("ambiguous match (\(candidates.count)); use --path or --occurrence:\n\(preview)")
424
+ }
425
+ return candidates[0]
426
+ }
427
+
428
+ func activate(_ app: NSRunningApplication) throws {
429
+ let activated = app.activate(options: [.activateAllWindows])
430
+ let application = axApplication(app)
431
+ _ = AXUIElementSetAttributeValue(
432
+ application,
433
+ kAXFrontmostAttribute as CFString,
434
+ kCFBooleanTrue
435
+ )
436
+ if let window = try? frontWindow(app) {
437
+ _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
438
+ }
439
+ guard activated || app.isActive else {
440
+ throw Failure("could not activate \(app.localizedName ?? "app")")
441
+ }
442
+ usleep(150_000)
443
+ }
444
+
445
+ func clickGlobal(_ point: CGPoint) throws {
446
+ guard let move = CGEvent(
447
+ mouseEventSource: nil,
448
+ mouseType: .mouseMoved,
449
+ mouseCursorPosition: point,
450
+ mouseButton: .left
451
+ ),
452
+ let down = CGEvent(
453
+ mouseEventSource: nil,
454
+ mouseType: .leftMouseDown,
455
+ mouseCursorPosition: point,
456
+ mouseButton: .left
457
+ ),
458
+ let up = CGEvent(
459
+ mouseEventSource: nil,
460
+ mouseType: .leftMouseUp,
461
+ mouseCursorPosition: point,
462
+ mouseButton: .left
463
+ )
464
+ else { throw Failure("could not create mouse event") }
465
+ move.post(tap: .cghidEventTap)
466
+ down.post(tap: .cghidEventTap)
467
+ usleep(40_000)
468
+ up.post(tap: .cghidEventTap)
469
+ }
470
+
471
+ func press(_ located: LocatedElement, allowFrameFallback: Bool) throws -> String {
472
+ let error = AXUIElementPerformAction(located.element, kAXPressAction as CFString)
473
+ if error == .success { return "accessibility" }
474
+ guard allowFrameFallback else {
475
+ throw Failure(
476
+ "AXPress failed (\(error.rawValue)) for path=\(located.path) role=\(role(located.element)); " +
477
+ "inspect again or explicitly pass --frame-fallback"
478
+ )
479
+ }
480
+ guard let rectangle = frame(located.element), !rectangle.isEmpty else {
481
+ throw Failure("AXPress failed (\(error.rawValue)) and element has no frame")
482
+ }
483
+ try clickGlobal(CGPoint(x: rectangle.midX, y: rectangle.midY))
484
+ return "frame-center-fallback"
485
+ }
486
+
487
+ func sendText(_ text: String) throws {
488
+ guard let source = CGEventSource(stateID: .combinedSessionState) else {
489
+ throw Failure("could not create keyboard event source")
490
+ }
491
+ let utf16 = Array(text.utf16)
492
+ for start in stride(from: 0, to: utf16.count, by: 32) {
493
+ let chunk = Array(utf16[start ..< min(start + 32, utf16.count)])
494
+ guard let down = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true),
495
+ let up = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false)
496
+ else { throw Failure("could not create keyboard event") }
497
+ chunk.withUnsafeBufferPointer { buffer in
498
+ down.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress)
499
+ up.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress)
500
+ }
501
+ down.post(tap: .cghidEventTap)
502
+ up.post(tap: .cghidEventTap)
503
+ usleep(10_000)
504
+ }
505
+ }
506
+
507
+ func keyCode(_ name: String) throws -> CGKeyCode {
508
+ let codes: [String: CGKeyCode] = [
509
+ "return": 36, "enter": 36, "tab": 48, "space": 49, "delete": 51,
510
+ "escape": 53, "esc": 53, "home": 115, "end": 119, "pageup": 116,
511
+ "pagedown": 121, "left": 123, "right": 124, "down": 125, "up": 126,
512
+ "a": 0, "c": 8, "v": 9, "x": 7, "z": 6,
513
+ ]
514
+ guard let code = codes[name.lowercased()] else {
515
+ throw Failure("unsupported key: \(name)")
516
+ }
517
+ return code
518
+ }
519
+
520
+ func modifierFlags(_ raw: String?) throws -> CGEventFlags {
521
+ var flags: CGEventFlags = []
522
+ for modifier in raw?.split(separator: ",").map(String.init) ?? [] {
523
+ switch modifier.lowercased() {
524
+ case "cmd", "command": flags.insert(.maskCommand)
525
+ case "shift": flags.insert(.maskShift)
526
+ case "ctrl", "control": flags.insert(.maskControl)
527
+ case "opt", "option", "alt": flags.insert(.maskAlternate)
528
+ case "fn": flags.insert(.maskSecondaryFn)
529
+ default: throw Failure("unsupported modifier: \(modifier)")
530
+ }
531
+ }
532
+ return flags
533
+ }
534
+
535
+ func sendKey(name: String, modifiers: String?) throws {
536
+ let code = try keyCode(name)
537
+ let flags = try modifierFlags(modifiers)
538
+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
539
+ let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
540
+ else { throw Failure("could not create keyboard event") }
541
+ down.flags = flags
542
+ up.flags = flags
543
+ down.post(tap: .cghidEventTap)
544
+ up.post(tap: .cghidEventTap)
545
+ }
546
+
547
+ // Correlates the app's focused AX window with the on-screen CGWindow that owns
548
+ // it so callers can target exactly that window (never a region or a full
549
+ // display) without shell involvement.
550
+ func correlatedWindowID(_ app: NSRunningApplication, window: AXUIElement) throws -> CGWindowID {
551
+ let targetFrame = try frame(window).unwrap("focused window has no frame")
552
+ let raw = CGWindowListCopyWindowInfo(
553
+ [.optionOnScreenOnly, .excludeDesktopElements],
554
+ kCGNullWindowID
555
+ ) as? [[String: Any]] ?? []
556
+ let candidates = raw.compactMap { info -> (CGWindowID, CGRect)? in
557
+ guard (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
558
+ (info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
559
+ let number = (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value,
560
+ let bounds = info[kCGWindowBounds as String] as? NSDictionary,
561
+ let rectangle = CGRect(dictionaryRepresentation: bounds),
562
+ rectangle.width >= 40,
563
+ rectangle.height >= 40
564
+ else { return nil }
565
+ return (number, rectangle)
566
+ }
567
+ guard !candidates.isEmpty else {
568
+ throw Failure("no on-screen window found for screenshot")
569
+ }
570
+ let scored = candidates.map { candidate -> (CGWindowID, CGFloat) in
571
+ let rectangle = candidate.1
572
+ let score = abs(rectangle.minX - targetFrame.minX) + abs(rectangle.minY - targetFrame.minY) +
573
+ abs(rectangle.width - targetFrame.width) + abs(rectangle.height - targetFrame.height)
574
+ return (candidate.0, score)
575
+ }.sorted { $0.1 < $1.1 }
576
+ guard let selected = scored.first, selected.1 <= 12 else {
577
+ throw Failure("could not correlate the focused AX window with an on-screen window")
578
+ }
579
+ return selected.0
580
+ }
581
+
582
+ func screenshotWindow(_ app: NSRunningApplication, window: AXUIElement, output: String) throws {
583
+ let selected = try correlatedWindowID(app, window: window)
584
+
585
+ let destination = URL(fileURLWithPath: output).standardizedFileURL
586
+ try FileManager.default.createDirectory(
587
+ at: destination.deletingLastPathComponent(),
588
+ withIntermediateDirectories: true
589
+ )
590
+ let task = Process()
591
+ task.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
592
+ task.arguments = ["-x", "-o", "-l", String(selected), destination.path]
593
+ try task.run()
594
+ task.waitUntilExit()
595
+ guard task.terminationStatus == 0, FileManager.default.fileExists(atPath: destination.path) else {
596
+ throw Failure("screencapture failed with status \(task.terminationStatus)")
597
+ }
598
+ print(destination.path)
599
+ }
600
+
601
+ // Swift screen-recording producer: captures exactly the on-screen window that
602
+ // backs the app's focused AX window through ScreenCaptureKit's
603
+ // desktop-independent window filter and encodes it to a silent H.264 MP4 with
604
+ // AVAssetWriter. There is deliberately no display/region fallback, no audio,
605
+ // and no screencapture(1) involvement for video.
606
+ @available(macOS 12.3, *)
607
+ final class WindowVideoRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
608
+ let queue = DispatchQueue(label: "ui-qa.window-video-recorder")
609
+ private let writer: AVAssetWriter
610
+ private let input: AVAssetWriterInput
611
+ private let adaptor: AVAssetWriterInputPixelBufferAdaptor
612
+ private let completed = DispatchSemaphore(value: 0)
613
+ private var stream: SCStream?
614
+ private var duration: Double
615
+ private var sessionStarted = false
616
+ private var stopping = false
617
+ private var failure: String?
618
+ private var firstSourceTime: CMTime?
619
+ private var lastSourceTime: CMTime?
620
+ private var lastPixelBuffer: CVPixelBuffer?
621
+ private var firstFrameUptime: UInt64?
622
+
623
+ init(output: URL, width: Int, height: Int, duration: Double) throws {
624
+ writer = try AVAssetWriter(outputURL: output, fileType: .mp4)
625
+ input = AVAssetWriterInput(mediaType: .video, outputSettings: [
626
+ AVVideoCodecKey: AVVideoCodecType.h264,
627
+ AVVideoWidthKey: width,
628
+ AVVideoHeightKey: height,
629
+ ])
630
+ input.expectsMediaDataInRealTime = true
631
+ adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: nil)
632
+ writer.add(input)
633
+ self.duration = duration
634
+ super.init()
635
+ }
636
+
637
+ func start(filter: SCContentFilter, configuration: SCStreamConfiguration) throws {
638
+ let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
639
+ try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
640
+ self.stream = stream
641
+ guard writer.startWriting() else {
642
+ throw Failure("could not start the video asset writer: \(writer.error.map { String(describing: $0) } ?? "unknown error")")
643
+ }
644
+ let started = DispatchSemaphore(value: 0)
645
+ var startError: Error?
646
+ stream.startCapture { error in
647
+ startError = error
648
+ started.signal()
649
+ }
650
+ guard started.wait(timeout: .now() + 10) == .success else {
651
+ throw Failure("timed out starting the window capture stream")
652
+ }
653
+ if let startError {
654
+ throw Failure("could not start the window capture stream: \(startError.localizedDescription)")
655
+ }
656
+ // Hard cap enforced inside the helper regardless of caller behavior.
657
+ queue.asyncAfter(deadline: .now() + duration) { self.stopNow() }
658
+ }
659
+
660
+ func requestStop() {
661
+ queue.async { self.stopNow() }
662
+ }
663
+
664
+ // Blocks until the recording is finalized; returns the failure reason, if any.
665
+ func waitAndFinish() -> String? {
666
+ let margin: Double = 15
667
+ if completed.wait(timeout: .now() + duration + margin) == .timedOut {
668
+ requestStop()
669
+ _ = completed.wait(timeout: .now() + margin)
670
+ }
671
+ return failure
672
+ }
673
+
674
+ private func stopNow() {
675
+ guard !stopping else { return }
676
+ stopping = true
677
+ let group = DispatchGroup()
678
+ if let stream {
679
+ group.enter()
680
+ stream.stopCapture { _ in group.leave() }
681
+ }
682
+ group.notify(queue: queue) {
683
+ guard self.sessionStarted else {
684
+ if self.failure == nil { self.failure = "no frames were captured from the target window" }
685
+ self.completed.signal()
686
+ return
687
+ }
688
+ self.appendFinalFrame()
689
+ self.input.markAsFinished()
690
+ self.writer.finishWriting {
691
+ if self.writer.status != .completed, self.failure == nil {
692
+ self.failure = "video asset writer finished with status \(self.writer.status.rawValue)"
693
+ }
694
+ self.completed.signal()
695
+ }
696
+ }
697
+ }
698
+
699
+ private func appendFinalFrame() {
700
+ guard let firstSourceTime, let lastSourceTime, let lastPixelBuffer,
701
+ let firstFrameUptime, input.isReadyForMoreMediaData
702
+ else { return }
703
+ let elapsedNanoseconds = DispatchTime.now().uptimeNanoseconds - firstFrameUptime
704
+ let elapsedSeconds = min(Double(elapsedNanoseconds) / 1_000_000_000, duration)
705
+ var finalTime = CMTimeAdd(
706
+ firstSourceTime,
707
+ CMTime(seconds: elapsedSeconds, preferredTimescale: 600)
708
+ )
709
+ if CMTimeCompare(finalTime, lastSourceTime) <= 0 {
710
+ finalTime = CMTimeAdd(lastSourceTime, CMTime(value: 1, timescale: 15))
711
+ }
712
+ if !adaptor.append(lastPixelBuffer, withPresentationTime: finalTime), failure == nil {
713
+ failure = "video final-frame append failed: \(writer.error.map { String(describing: $0) } ?? "unknown error")"
714
+ }
715
+ }
716
+
717
+ private func frameStatus(_ sampleBuffer: CMSampleBuffer) -> SCFrameStatus {
718
+ let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false)
719
+ as? [[SCStreamFrameInfo: Any]]
720
+ guard let raw = attachments?.first?[.status] else { return .complete }
721
+ if let status = raw as? SCFrameStatus { return status }
722
+ if let number = raw as? NSNumber, let status = SCFrameStatus(rawValue: number.intValue) { return status }
723
+ return .complete
724
+ }
725
+
726
+ func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
727
+ guard outputType == .screen, !stopping else { return }
728
+ let status = frameStatus(sampleBuffer)
729
+ // ScreenCaptureKit marks unchanged-window frames as `idle`; they still
730
+ // carry the current window surface and must be encoded so a static UI
731
+ // produces a truthful non-zero-duration timeline.
732
+ guard status == .complete || status == .started || status == .idle else { return }
733
+ let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
734
+ if !sessionStarted {
735
+ writer.startSession(atSourceTime: timestamp)
736
+ sessionStarted = true
737
+ firstSourceTime = timestamp
738
+ firstFrameUptime = DispatchTime.now().uptimeNanoseconds
739
+ }
740
+ guard writer.status == .writing, input.isReadyForMoreMediaData,
741
+ let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
742
+ else { return }
743
+ if adaptor.append(pixelBuffer, withPresentationTime: timestamp) {
744
+ lastSourceTime = timestamp
745
+ lastPixelBuffer = pixelBuffer
746
+ return
747
+ }
748
+ if failure == nil {
749
+ failure = "video pixel buffer append failed: \(writer.error.map { String(describing: $0) } ?? "unknown error")"
750
+ }
751
+ stopNow()
752
+ }
753
+
754
+ func stream(_ stream: SCStream, didStopWithError error: Error) {
755
+ // The captured window or its app died: finalize best-effort. A clean
756
+ // programmatic stopCapture never reaches this callback.
757
+ queue.async {
758
+ if !self.stopping, self.failure == nil {
759
+ self.failure = "capture stream stopped early: \(error.localizedDescription)"
760
+ }
761
+ self.stopNow()
762
+ }
763
+ }
764
+ }
765
+
766
+ private var recorderSignalSources: [DispatchSourceSignal] = []
767
+ private let recorderSignalLock = NSLock()
768
+ private var activeRecorder: WindowVideoRecorder?
769
+ private var recorderStopRequestedStorage = false
770
+
771
+ private var recorderStopRequested: Bool {
772
+ get {
773
+ recorderSignalLock.lock()
774
+ defer { recorderSignalLock.unlock() }
775
+ return recorderStopRequestedStorage
776
+ }
777
+ set {
778
+ recorderSignalLock.lock()
779
+ recorderStopRequestedStorage = newValue
780
+ recorderSignalLock.unlock()
781
+ }
782
+ }
783
+
784
+ // Installs graceful-stop handling for the record-window command. This must run
785
+ // before any slow startup work (app activation, shareable-content
786
+ // enumeration), because the desktop backend may legitimately SIGTERM the
787
+ // recorder early to stop and finalize.
788
+ func installRecorderSignalHandlers() {
789
+ signal(SIGTERM, SIG_IGN)
790
+ signal(SIGINT, SIG_IGN)
791
+ for number in [SIGTERM, SIGINT] {
792
+ let source = DispatchSource.makeSignalSource(signal: number, queue: DispatchQueue.global())
793
+ source.setEventHandler {
794
+ recorderStopRequested = true
795
+ activeRecorder?.requestStop()
796
+ }
797
+ source.resume()
798
+ recorderSignalSources.append(source)
799
+ }
800
+ }
801
+
802
+ @available(macOS 12.3, *)
803
+ func recordWindowVideo(app: NSRunningApplication, window: AXUIElement, arguments: Arguments) throws {
804
+ let windowID = try correlatedWindowID(app, window: window)
805
+ let output = URL(fileURLWithPath: try arguments.required("out")).standardizedFileURL
806
+ try FileManager.default.createDirectory(
807
+ at: output.deletingLastPathComponent(),
808
+ withIntermediateDirectories: true
809
+ )
810
+ let temporary = URL(fileURLWithPath: "\(output.path).\(ProcessInfo.processInfo.processIdentifier).tmp")
811
+
812
+ let semaphore = DispatchSemaphore(value: 0)
813
+ var content: SCShareableContent?
814
+ var enumerationError: Error?
815
+ SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: true) { result, error in
816
+ content = result
817
+ enumerationError = error
818
+ semaphore.signal()
819
+ }
820
+ guard semaphore.wait(timeout: .now() + 15) == .success else {
821
+ throw Failure("timed out enumerating capturable windows")
822
+ }
823
+ if let enumerationError {
824
+ throw Failure("could not enumerate capturable windows: \(enumerationError.localizedDescription)")
825
+ }
826
+ guard let content, let target = content.windows.first(where: { $0.windowID == windowID }) else {
827
+ throw Failure("window \(windowID) is not present among capturable on-screen windows")
828
+ }
829
+
830
+ let requested = try arguments.double("duration", default: 30)
831
+ let duration = min(max(requested, 0.5), 30)
832
+ let windowFrame = try frame(window).unwrap("focused window has no frame")
833
+ let scale = (NSScreen.screens.first { $0.frame.intersects(windowFrame) } ?? NSScreen.main)?.backingScaleFactor ?? 2
834
+ let pixelWidth = evenPixel(Double(windowFrame.width) * scale)
835
+ let pixelHeight = evenPixel(Double(windowFrame.height) * scale)
836
+ let configuration = SCStreamConfiguration()
837
+ configuration.width = pixelWidth
838
+ configuration.height = pixelHeight
839
+ // Independent-window streams do not scale up by default. Our encoder
840
+ // surface is Retina-sized, so without scalesToFit the 1x window surface is
841
+ // left in the top-left corner with unused space around it. Fill the exact
842
+ // window output surface while preserving the source aspect ratio.
843
+ configuration.scalesToFit = true
844
+ configuration.minimumFrameInterval = CMTime(value: 1, timescale: 15)
845
+ configuration.showsCursor = false
846
+
847
+ let recorder = try WindowVideoRecorder(
848
+ output: temporary,
849
+ width: pixelWidth,
850
+ height: pixelHeight,
851
+ duration: duration
852
+ )
853
+ activeRecorder = recorder
854
+ try recorder.start(filter: SCContentFilter(desktopIndependentWindow: target), configuration: configuration)
855
+ print("recording-started")
856
+ fflush(stdout)
857
+ if recorderStopRequested { recorder.requestStop() }
858
+ let failure = recorder.waitAndFinish()
859
+ if let failure {
860
+ try? FileManager.default.removeItem(at: temporary)
861
+ throw Failure(failure)
862
+ }
863
+ // Publish atomically: a killed helper can never leave a partial artifact.
864
+ if FileManager.default.fileExists(atPath: output.path) {
865
+ try FileManager.default.removeItem(at: output)
866
+ }
867
+ try FileManager.default.moveItem(at: temporary, to: output)
868
+ try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: output.path)
869
+ print(output.path)
870
+ }
871
+
872
+ private func evenPixel(_ value: Double) -> Int {
873
+ max(2, (Int(value.rounded(.down)) / 2) * 2)
874
+ }
875
+
876
+ func isUseful(_ element: AXUIElement) -> Bool {
877
+ let usefulRoles: Set<String> = [
878
+ kAXButtonRole, kAXCheckBoxRole, kAXComboBoxRole, "AXLink",
879
+ kAXMenuItemRole, kAXPopUpButtonRole, kAXRadioButtonRole,
880
+ "AXSecureTextField", kAXSliderRole, kAXStaticTextRole,
881
+ kAXTextAreaRole, kAXTextFieldRole, kAXWindowRole,
882
+ ]
883
+ return usefulRoles.contains(role(element)) || !fields(element).isEmpty
884
+ }
885
+
886
+ func run(_ arguments: Arguments) throws {
887
+ switch arguments.command {
888
+ case "doctor":
889
+ let accessibility: Bool
890
+ if arguments.has("prompt") {
891
+ let key = kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String
892
+ accessibility = AXIsProcessTrustedWithOptions([key: true] as CFDictionary)
893
+ } else {
894
+ accessibility = AXIsProcessTrusted()
895
+ }
896
+ let screenRecording = arguments.has("prompt")
897
+ ? CGRequestScreenCaptureAccess()
898
+ : CGPreflightScreenCaptureAccess()
899
+ let screenCaptureKit = ProcessInfo.processInfo.isOperatingSystemAtLeast(
900
+ OperatingSystemVersion(majorVersion: 12, minorVersion: 3, patchVersion: 0)
901
+ )
902
+ print("macos=\(ProcessInfo.processInfo.operatingSystemVersionString)")
903
+ print("accessibility=\(accessibility ? "granted" : "missing")")
904
+ print("screen_recording=\(screenRecording ? "granted" : "missing")")
905
+ print("sck=\(screenCaptureKit ? "available" : "unsupported")")
906
+
907
+ case "list":
908
+ for app in NSWorkspace.shared.runningApplications
909
+ .filter({ !$0.isTerminated && $0.activationPolicy == .regular })
910
+ .sorted(by: { ($0.localizedName ?? "") < ($1.localizedName ?? "") })
911
+ {
912
+ print(
913
+ "app=\(quoted(app.localizedName ?? "?")) " +
914
+ "bundle=\(quoted(app.bundleIdentifier ?? "?")) pid=\(app.processIdentifier)"
915
+ )
916
+ }
917
+
918
+ case "wait-window":
919
+ try requireAccessibility()
920
+ let timeout = try arguments.double("timeout", default: 60)
921
+ let deadline = Date().addingTimeInterval(timeout)
922
+ while Date() < deadline {
923
+ if let app = try? findApplication(arguments), (try? frontWindow(app)) != nil {
924
+ if arguments.has("print-pid") {
925
+ print(app.processIdentifier)
926
+ } else {
927
+ print("ready app=\(quoted(app.localizedName ?? "?")) pid=\(app.processIdentifier)")
928
+ }
929
+ return
930
+ }
931
+ usleep(200_000)
932
+ }
933
+ throw Failure("window did not appear within \(timeout)s")
934
+
935
+ case "focus":
936
+ try requireAccessibility()
937
+ let app = try findApplication(arguments)
938
+ try activate(app)
939
+ print("focused app=\(quoted(app.localizedName ?? "?")) pid=\(app.processIdentifier)")
940
+
941
+ case "inspect":
942
+ try requireAccessibility()
943
+ let app = try findApplication(arguments)
944
+ let window = try frontWindow(app)
945
+ let depth = try arguments.integer("depth", default: 10)
946
+ let limit = try arguments.integer("limit", default: 80)
947
+ var shown = 0
948
+ for located in walk(window, maxDepth: depth) {
949
+ if !arguments.has("all"), !isUseful(located.element) { continue }
950
+ print(summary(located.element, path: located.path))
951
+ shown += 1
952
+ if shown >= limit { break }
953
+ }
954
+ print("shown=\(shown) limit=\(limit) depth=\(depth)")
955
+
956
+ case "describe":
957
+ try requireAccessibility()
958
+ let app = try findApplication(arguments)
959
+ let located = try locate(frontWindow(app), arguments: arguments)
960
+ try printJSON(record(located))
961
+
962
+ case "screenshot":
963
+ let app = try findApplication(arguments)
964
+ try activate(app)
965
+ let window = try frontWindow(app)
966
+ try screenshotWindow(app, window: window, output: arguments.required("out"))
967
+
968
+ case "window-id":
969
+ let app = try findApplication(arguments)
970
+ let window = try frontWindow(app)
971
+ print(try correlatedWindowID(app, window: window))
972
+
973
+ case "record-window":
974
+ // Graceful-stop handlers go in before any slow startup work so the
975
+ // desktop backend can always stop and finalize the recorder.
976
+ installRecorderSignalHandlers()
977
+ try requireAccessibility()
978
+ guard #available(macOS 12.3, *) else {
979
+ throw Failure("ScreenCaptureKit window recording requires macOS 12.3 or newer")
980
+ }
981
+ guard CGPreflightScreenCaptureAccess() else {
982
+ throw Failure("Screen Recording permission is required to record a window; run doctor --prompt")
983
+ }
984
+ let app = try findApplication(arguments)
985
+ try activate(app)
986
+ let window = try frontWindow(app)
987
+ try recordWindowVideo(app: app, window: window, arguments: arguments)
988
+
989
+ case "click":
990
+ try requireAccessibility()
991
+ let app = try findApplication(arguments)
992
+ try activate(app)
993
+ let located = try locate(frontWindow(app), arguments: arguments)
994
+ let method = try press(located, allowFrameFallback: arguments.has("frame-fallback"))
995
+ print("clicked method=\(method) path=\(located.path) role=\(role(located.element))")
996
+
997
+ case "set-value":
998
+ try requireAccessibility()
999
+ let app = try findApplication(arguments)
1000
+ try activate(app)
1001
+ let located = try locate(frontWindow(app), arguments: arguments)
1002
+ var settable = DarwinBoolean(false)
1003
+ let check = AXUIElementIsAttributeSettable(
1004
+ located.element,
1005
+ kAXValueAttribute as CFString,
1006
+ &settable
1007
+ )
1008
+ guard check == .success, settable.boolValue else {
1009
+ throw Failure("AX value is not settable for path=\(located.path) role=\(role(located.element))")
1010
+ }
1011
+ let value = try arguments.required("value")
1012
+ let result = AXUIElementSetAttributeValue(
1013
+ located.element,
1014
+ kAXValueAttribute as CFString,
1015
+ value as CFString
1016
+ )
1017
+ guard result == .success else {
1018
+ throw Failure("setting AX value failed: \(result.rawValue)")
1019
+ }
1020
+ print("set-value path=\(located.path) role=\(role(located.element))")
1021
+
1022
+ case "type":
1023
+ try requireAccessibility()
1024
+ let app = try findApplication(arguments)
1025
+ try activate(app)
1026
+ let focused = try focusedElement(app)
1027
+ let focusedRole = role(focused)
1028
+ let textRoles: Set<String> = [
1029
+ kAXComboBoxRole, "AXSecureTextField", kAXTextAreaRole, kAXTextFieldRole,
1030
+ ]
1031
+ guard textRoles.contains(focusedRole) else {
1032
+ throw Failure("refusing to type: focused role is \(focusedRole), not a text input")
1033
+ }
1034
+ let text = try arguments.required("text")
1035
+ try sendText(text)
1036
+ print("typed characters=\(text.count) focused_role=\(focusedRole)")
1037
+
1038
+ case "key":
1039
+ try requireAccessibility()
1040
+ let app = try findApplication(arguments)
1041
+ try activate(app)
1042
+ let key = try arguments.required("key")
1043
+ try sendKey(name: key, modifiers: arguments.value("modifiers"))
1044
+ print("key=\(key)")
1045
+
1046
+ case "click-point":
1047
+ try requireAccessibility()
1048
+ let app = try findApplication(arguments)
1049
+ try activate(app)
1050
+ let windowFrame = try frame(frontWindow(app)).unwrap("front window has no frame")
1051
+ let x = try arguments.double("x")
1052
+ let y = try arguments.double("y")
1053
+ guard x >= 0, y >= 0, x <= windowFrame.width, y <= windowFrame.height else {
1054
+ throw Failure("relative point is outside the front window")
1055
+ }
1056
+ try clickGlobal(CGPoint(x: windowFrame.minX + x, y: windowFrame.minY + y))
1057
+ print("clicked relative=\(Int(x)),\(Int(y))")
1058
+
1059
+ case "quit":
1060
+ let app = try findApplication(arguments)
1061
+ let requested = arguments.has("force") ? app.forceTerminate() : app.terminate()
1062
+ guard requested else { throw Failure("quit request was rejected") }
1063
+ print("quit requested pid=\(app.processIdentifier)")
1064
+
1065
+ default:
1066
+ throw Failure(Arguments.usage)
1067
+ }
1068
+ }
1069
+
1070
+ extension Optional {
1071
+ func unwrap(_ message: String) throws -> Wrapped {
1072
+ guard let self else { throw Failure(message) }
1073
+ return self
1074
+ }
1075
+ }
1076
+
1077
+ do {
1078
+ try run(Arguments(Array(CommandLine.arguments.dropFirst())))
1079
+ } catch let error as Failure {
1080
+ FileHandle.standardError.write(Data("error: \(error.description)\n".utf8))
1081
+ exit(2)
1082
+ } catch {
1083
+ FileHandle.standardError.write(Data("error: \(error)\n".utf8))
1084
+ exit(2)
1085
+ }