pi-ui-extend 1.0.41 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -13
- package/dist/app/app.d.ts +13 -0
- package/dist/app/app.js +158 -8
- package/dist/app/cli/install.js +2 -2
- package/dist/app/commands/command-controller.d.ts +4 -0
- package/dist/app/commands/command-controller.js +11 -0
- package/dist/app/commands/command-git-actions.d.ts +25 -0
- package/dist/app/commands/command-git-actions.js +381 -0
- package/dist/app/commands/command-host.d.ts +4 -0
- package/dist/app/commands/command-host.js +5 -2
- package/dist/app/commands/command-model-actions.d.ts +1 -0
- package/dist/app/commands/command-model-actions.js +57 -35
- package/dist/app/commands/command-navigation-actions.d.ts +1 -0
- package/dist/app/commands/command-navigation-actions.js +16 -3
- package/dist/app/commands/command-registry.d.ts +2 -0
- package/dist/app/commands/command-registry.js +15 -1
- package/dist/app/commands/command-session-actions.js +9 -3
- package/dist/app/commands/reload-context-inventory.d.ts +11 -0
- package/dist/app/commands/reload-context-inventory.js +70 -0
- package/dist/app/extensions/extension-actions-controller.d.ts +1 -0
- package/dist/app/extensions/extension-actions-controller.js +6 -0
- package/dist/app/extensions/subagent-catalog-state.d.ts +9 -0
- package/dist/app/extensions/subagent-catalog-state.js +23 -0
- package/dist/app/input/autocomplete-controller.js +61 -34
- package/dist/app/input/input-action-controller.d.ts +2 -0
- package/dist/app/input/input-action-controller.js +8 -1
- package/dist/app/input/input-controller.d.ts +2 -1
- package/dist/app/input/input-controller.js +7 -2
- package/dist/app/input/prompt-enhancer-controller.js +53 -35
- package/dist/app/input/voice-controller.d.ts +41 -45
- package/dist/app/input/voice-controller.js +351 -419
- package/dist/app/model/model-usage-controller.js +17 -7
- package/dist/app/model/model-usage-status.d.ts +4 -1
- package/dist/app/model/model-usage-status.js +270 -46
- package/dist/app/popup/menu-items-controller.d.ts +13 -3
- package/dist/app/popup/menu-items-controller.js +37 -21
- package/dist/app/popup/popup-action-controller.d.ts +12 -2
- package/dist/app/popup/popup-action-controller.js +77 -24
- package/dist/app/popup/popup-menu-controller.d.ts +36 -14
- package/dist/app/popup/popup-menu-controller.js +239 -69
- package/dist/app/rendering/dcp-stats.d.ts +9 -4
- package/dist/app/rendering/dcp-stats.js +40 -425
- package/dist/app/rendering/editor-panels.js +10 -4
- package/dist/app/rendering/popup-menu-renderer.d.ts +3 -5
- package/dist/app/rendering/popup-menu-renderer.js +42 -37
- package/dist/app/rendering/render-controller.js +23 -2
- package/dist/app/rendering/status-line-renderer.d.ts +5 -0
- package/dist/app/rendering/status-line-renderer.js +41 -5
- package/dist/app/rendering/tab-line-renderer.js +26 -20
- package/dist/app/runtime.d.ts +12 -1
- package/dist/app/runtime.js +120 -12
- package/dist/app/screen/mouse-controller.d.ts +2 -0
- package/dist/app/screen/mouse-controller.js +17 -7
- package/dist/app/screen/status-controller.d.ts +4 -0
- package/dist/app/screen/status-controller.js +5 -0
- package/dist/app/session/lazy-session-manager.js +34 -0
- package/dist/app/session/session-event-controller.d.ts +1 -0
- package/dist/app/session/session-event-controller.js +10 -1
- package/dist/app/session/session-history.d.ts +1 -0
- package/dist/app/session/session-history.js +12 -1
- package/dist/app/session/session-lifecycle-controller.d.ts +7 -1
- package/dist/app/session/session-lifecycle-controller.js +16 -1
- package/dist/app/session/tabs-controller.d.ts +26 -1
- package/dist/app/session/tabs-controller.js +454 -147
- package/dist/app/subagents/subagents-files.js +60 -1
- package/dist/app/subagents/subagents-model.d.ts +1 -0
- package/dist/app/subagents/subagents-model.js +18 -2
- package/dist/app/subagents/subagents-widget-controller.d.ts +1 -0
- package/dist/app/subagents/subagents-widget-controller.js +6 -0
- package/dist/app/types.d.ts +16 -1
- package/dist/app/workspace/workspace-actions-controller.js +10 -2
- package/dist/app/workspace/workspace-undo.d.ts +1 -0
- package/dist/app/workspace/workspace-undo.js +1 -0
- package/dist/bundled-extensions/question/index.js +9 -1
- package/dist/bundled-extensions/question/remote.d.ts +4 -0
- package/dist/bundled-extensions/question/remote.js +33 -0
- package/dist/bundled-extensions/telegram-connector/bot.d.ts +43 -0
- package/dist/bundled-extensions/telegram-connector/bot.js +166 -0
- package/dist/bundled-extensions/telegram-connector/config.d.ts +8 -0
- package/dist/bundled-extensions/telegram-connector/config.js +87 -0
- package/dist/bundled-extensions/telegram-connector/coordinator.d.ts +66 -0
- package/dist/bundled-extensions/telegram-connector/coordinator.js +413 -0
- package/dist/bundled-extensions/telegram-connector/index.d.ts +3 -0
- package/dist/bundled-extensions/telegram-connector/index.js +195 -0
- package/dist/bundled-extensions/terminal-bell/index.d.ts +0 -8
- package/dist/bundled-extensions/terminal-bell/index.js +0 -76
- package/dist/bundled-extensions/workspace-undo/index.d.ts +26 -0
- package/dist/bundled-extensions/workspace-undo/index.js +191 -0
- package/dist/config.d.ts +21 -2
- package/dist/config.js +159 -32
- package/dist/default-pix-config.js +25 -5
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/pi-tools-suite-schema.d.ts +88 -62
- package/dist/schemas/pi-tools-suite-schema.js +54 -83
- package/dist/schemas/pix-schema.d.ts +19 -2
- package/dist/schemas/pix-schema.js +47 -5
- package/dist/schemas/tasks-schema.d.ts +18 -0
- package/dist/schemas/tasks-schema.js +41 -0
- package/docs/concurrency.md +9 -1
- package/docs/desktop-mvp.md +22 -6
- package/docs/desktop-task-manager.md +148 -84
- package/docs/release.md +30 -6
- package/external/pi-tools-suite/README.md +197 -84
- package/external/pi-tools-suite/docs/evals.md +1 -1
- package/external/pi-tools-suite/docs/session-recovery.md +47 -14
- package/external/pi-tools-suite/docs/subagent-model-pools.md +48 -30
- package/external/pi-tools-suite/docs/ui-qa-subagent.md +440 -0
- package/external/pi-tools-suite/package.json +6 -1
- package/external/pi-tools-suite/src/antigravity-auth/auth-store.ts +2 -1
- package/external/pi-tools-suite/src/antigravity-auth/constants.ts +9 -3
- package/external/pi-tools-suite/src/antigravity-auth/headers.ts +41 -3
- package/external/pi-tools-suite/src/antigravity-auth/models.ts +94 -21
- package/external/pi-tools-suite/src/antigravity-auth/oauth.ts +5 -17
- package/external/pi-tools-suite/src/antigravity-auth/payload.ts +72 -7
- package/external/pi-tools-suite/src/antigravity-auth/stream.ts +13 -1
- package/external/pi-tools-suite/src/async-subagents/agents/frontier-review.md +23 -0
- package/external/pi-tools-suite/src/async-subagents/agents/implement.md +1 -1
- package/external/pi-tools-suite/src/async-subagents/agents/presets.jsonc +16 -0
- package/external/pi-tools-suite/src/async-subagents/agents/research.md +5 -3
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/browser.mjs +346 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/desktop.mjs +1038 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/backends/tui.mjs +759 -0
- package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/scripts/browser-qa-runner.mjs +32 -23
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/chrome-devtools/chrome-devtools-provider.mjs +895 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/linux/linux-atspi.py +494 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/macos/macos-accessibility.swift +1085 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/bridge-client.mjs +50 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/native-terminal/native-terminal-host.mjs +501 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/drivers/windows/windows-uia.ps1 +454 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-auth.md +114 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-chrome-devtools.md +91 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser-playwright.md +145 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/browser.md +82 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-linux.md +26 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-macos.md +29 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop-windows.md +23 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/desktop.md +54 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-native-terminal.md +59 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui-pty.md +37 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/guides/tui.md +60 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa/scripts/ui-qa-runner.mjs +535 -0
- package/external/pi-tools-suite/src/async-subagents/agents/ui-qa.md +75 -0
- package/external/pi-tools-suite/src/async-subagents/commands.ts +15 -71
- package/external/pi-tools-suite/src/async-subagents/core/activity.ts +33 -0
- package/external/pi-tools-suite/src/async-subagents/core/agent-catalog.ts +5 -4
- package/external/pi-tools-suite/src/async-subagents/core/agent-strategy.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/agents-dir.ts +8 -5
- package/external/pi-tools-suite/src/async-subagents/core/browser-qa.ts +16 -2
- package/external/pi-tools-suite/src/async-subagents/core/config.ts +160 -327
- package/external/pi-tools-suite/src/async-subagents/core/model-selection.ts +2 -1
- package/external/pi-tools-suite/src/async-subagents/core/registry.ts +158 -33
- package/external/pi-tools-suite/src/async-subagents/core/routing.ts +21 -5
- package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +40 -26
- package/external/pi-tools-suite/src/async-subagents/core/state.ts +40 -0
- package/external/pi-tools-suite/src/async-subagents/core/types.ts +7 -0
- package/external/pi-tools-suite/src/async-subagents/core/ultrawork-auto.ts +51 -45
- package/external/pi-tools-suite/src/async-subagents/index.ts +69 -5
- package/external/pi-tools-suite/src/async-subagents/lib.ts +16 -9
- package/external/pi-tools-suite/src/async-subagents/tools/spawn.ts +8 -5
- package/external/pi-tools-suite/src/async-subagents/tools/subagents.ts +5 -4
- package/external/pi-tools-suite/src/async-subagents/types.ts +1 -0
- package/external/pi-tools-suite/src/coding-discipline/index.ts +90 -59
- package/external/pi-tools-suite/src/config.ts +55 -1
- package/external/pi-tools-suite/src/context-gateway/accounting-log.ts +282 -0
- package/external/pi-tools-suite/src/context-gateway/config.ts +77 -3
- package/external/pi-tools-suite/src/context-gateway/efficiency.ts +414 -0
- package/external/pi-tools-suite/src/context-gateway/enforcement.ts +222 -0
- package/external/pi-tools-suite/src/context-gateway/index.ts +206 -19
- package/external/pi-tools-suite/src/context-gateway/storeless-capabilities.ts +7 -7
- package/external/pi-tools-suite/src/context-gateway/telemetry.ts +101 -28
- package/external/pi-tools-suite/src/context-gateway/types.ts +16 -3
- package/external/pi-tools-suite/src/context-inventory.ts +99 -0
- package/external/pi-tools-suite/src/dcp/auto-compress-budget.ts +39 -4
- package/external/pi-tools-suite/src/dcp/auto-compress.ts +128 -49
- package/external/pi-tools-suite/src/dcp/commands.ts +32 -156
- package/external/pi-tools-suite/src/dcp/compress-tool.ts +177 -46
- package/external/pi-tools-suite/src/dcp/compression-blocks.ts +6 -58
- package/external/pi-tools-suite/src/dcp/compression-preview.ts +9 -0
- package/external/pi-tools-suite/src/dcp/compression-progress.ts +22 -0
- package/external/pi-tools-suite/src/dcp/config.ts +114 -17
- package/external/pi-tools-suite/src/dcp/conversation-index.ts +36 -9
- package/external/pi-tools-suite/src/dcp/diagnostics.ts +41 -0
- package/external/pi-tools-suite/src/dcp/fresh-tool-results.ts +38 -0
- package/external/pi-tools-suite/src/dcp/index.ts +338 -66
- package/external/pi-tools-suite/src/dcp/journal.ts +126 -4
- package/external/pi-tools-suite/src/dcp/progress-controller.ts +2 -1
- package/external/pi-tools-suite/src/dcp/prompts.ts +83 -192
- package/external/pi-tools-suite/src/dcp/protected-continuity.ts +175 -0
- package/external/pi-tools-suite/src/dcp/pruner-candidates.ts +93 -3
- package/external/pi-tools-suite/src/dcp/pruner-message-ids.ts +6 -10
- package/external/pi-tools-suite/src/dcp/pruner-nudge.ts +36 -33
- package/external/pi-tools-suite/src/dcp/pruner-tools.ts +6 -3
- package/external/pi-tools-suite/src/dcp/pruner.ts +1 -0
- package/external/pi-tools-suite/src/dcp/routine-pressure.ts +86 -0
- package/external/pi-tools-suite/src/dcp/state.ts +9 -0
- package/external/pi-tools-suite/src/dcp/statistics.d.ts +8 -0
- package/external/pi-tools-suite/src/dcp/statistics.js +156 -0
- package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +78 -22
- package/external/pi-tools-suite/src/index.ts +7 -1
- package/external/pi-tools-suite/src/lib/project.ts +36 -1
- package/external/pi-tools-suite/src/model-tools/index.ts +10 -7
- package/external/pi-tools-suite/src/repo-discovery/index.ts +304 -4
- package/external/pi-tools-suite/src/resource-registry/index.ts +2551 -0
- package/external/pi-tools-suite/src/session-recovery/index.ts +17 -0
- package/external/pi-tools-suite/src/shell-command-policy.ts +219 -0
- package/external/pi-tools-suite/src/todo/index.ts +21 -0
- package/external/pi-tools-suite/src/todo/todo.ts +19 -1
- package/external/pi-tools-suite/src/tool-descriptions.ts +47 -19
- package/package.json +10 -9
- package/schemas/pi-tools-suite.json +466 -287
- package/schemas/pix.json +129 -11
- package/schemas/tasks.json +131 -0
- package/skills/simplify/SKILL.md +33 -5
- package/docs/desktop-markdown-media.md +0 -77
- package/external/pi-tools-suite/docs/browser-qa-subagent.md +0 -177
- package/external/pi-tools-suite/docs/dcp-emergency-current-turn.md +0 -102
- package/external/pi-tools-suite/src/async-subagents/agents/browser-qa.md +0 -598
- package/external/pi-tools-suite/src/async-subagents/async-subagents.sample.jsonc +0 -54
- package/external/pi-tools-suite/src/skill-installer/index.ts +0 -333
- package/skills/playwright-cli/SKILL.md +0 -420
- package/skills/playwright-cli/references/element-attributes.md +0 -23
- package/skills/playwright-cli/references/playwright-tests.md +0 -50
- package/skills/playwright-cli/references/request-mocking.md +0 -87
- package/skills/playwright-cli/references/running-code.md +0 -241
- package/skills/playwright-cli/references/session-management.md +0 -273
- package/skills/playwright-cli/references/spec-driven-testing.md +0 -311
- package/skills/playwright-cli/references/storage-state.md +0 -290
- package/skills/playwright-cli/references/test-generation.md +0 -142
- package/skills/playwright-cli/references/tracing.md +0 -154
- package/skills/playwright-cli/references/video-recording.md +0 -147
- package/skills/spec-lite/SKILL.md +0 -140
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-auth.example.jsonc +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/examples/qa-flow.example.jsonc +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.LICENSE +0 -0
- /package/external/pi-tools-suite/src/async-subagents/agents/{browser-qa → ui-qa/browser}/vendor/fflate.mjs +0 -0
|
@@ -0,0 +1,1085 @@
|
|
|
1
|
+
import AppKit
|
|
2
|
+
import ApplicationServices
|
|
3
|
+
import AVFoundation
|
|
4
|
+
import CoreMedia
|
|
5
|
+
import Darwin
|
|
6
|
+
import Foundation
|
|
7
|
+
import ScreenCaptureKit
|
|
8
|
+
|
|
9
|
+
struct Failure: Error, CustomStringConvertible {
|
|
10
|
+
let description: String
|
|
11
|
+
|
|
12
|
+
init(_ description: String) {
|
|
13
|
+
self.description = description
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
struct Arguments {
|
|
18
|
+
let command: String
|
|
19
|
+
private let options: [String: String]
|
|
20
|
+
private let flags: Set<String>
|
|
21
|
+
|
|
22
|
+
init(_ values: [String]) throws {
|
|
23
|
+
guard let command = values.first else {
|
|
24
|
+
throw Failure(Self.usage)
|
|
25
|
+
}
|
|
26
|
+
self.command = command
|
|
27
|
+
|
|
28
|
+
let valueOptions: Set<String> = [
|
|
29
|
+
"--app", "--bundle-id", "--pid", "--pgid", "--depth", "--limit", "--match",
|
|
30
|
+
"--role", "--occurrence", "--path", "--value", "--text",
|
|
31
|
+
"--key", "--modifiers", "--x", "--y", "--out", "--timeout", "--duration",
|
|
32
|
+
]
|
|
33
|
+
var parsedOptions: [String: String] = [:]
|
|
34
|
+
var parsedFlags = Set<String>()
|
|
35
|
+
var index = 1
|
|
36
|
+
while index < values.count {
|
|
37
|
+
let argument = values[index]
|
|
38
|
+
guard argument.hasPrefix("--") else {
|
|
39
|
+
throw Failure("unexpected argument: \(argument)")
|
|
40
|
+
}
|
|
41
|
+
if valueOptions.contains(argument) {
|
|
42
|
+
guard index + 1 < values.count else {
|
|
43
|
+
throw Failure("missing value for \(argument)")
|
|
44
|
+
}
|
|
45
|
+
parsedOptions[argument] = values[index + 1]
|
|
46
|
+
index += 2
|
|
47
|
+
} else {
|
|
48
|
+
parsedFlags.insert(argument)
|
|
49
|
+
index += 1
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
options = parsedOptions
|
|
53
|
+
flags = parsedFlags
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
func value(_ name: String) -> String? {
|
|
57
|
+
options["--\(name)"]
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
func required(_ name: String) throws -> String {
|
|
61
|
+
guard let value = value(name), !value.isEmpty else {
|
|
62
|
+
throw Failure("missing --\(name)")
|
|
63
|
+
}
|
|
64
|
+
return value
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
func integer(_ name: String, default fallback: Int) throws -> Int {
|
|
68
|
+
guard let raw = value(name) else { return fallback }
|
|
69
|
+
guard let parsed = Int(raw) else {
|
|
70
|
+
throw Failure("invalid integer for --\(name): \(raw)")
|
|
71
|
+
}
|
|
72
|
+
return parsed
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
func double(_ name: String, default fallback: Double? = nil) throws -> Double {
|
|
76
|
+
if let raw = value(name), let parsed = Double(raw) {
|
|
77
|
+
return parsed
|
|
78
|
+
}
|
|
79
|
+
if let fallback { return fallback }
|
|
80
|
+
throw Failure("missing or invalid --\(name)")
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
func has(_ name: String) -> Bool {
|
|
84
|
+
flags.contains("--\(name)")
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
static let usage = """
|
|
88
|
+
usage: macos-window.sh COMMAND [options]
|
|
89
|
+
doctor [--prompt]
|
|
90
|
+
list
|
|
91
|
+
App selectors: --pid PID (preferred), --pgid PGID, --app NAME, --bundle-id ID
|
|
92
|
+
wait-window SELECTOR [--timeout 60] [--print-pid]
|
|
93
|
+
focus SELECTOR
|
|
94
|
+
inspect SELECTOR [--depth 10] [--limit 80] [--all]
|
|
95
|
+
describe SELECTOR (--path PATH | --match TEXT [--role ROLE] [--occurrence N])
|
|
96
|
+
screenshot SELECTOR --out FILE
|
|
97
|
+
window-id SELECTOR
|
|
98
|
+
record-window SELECTOR --out FILE [--duration SECONDS]
|
|
99
|
+
click SELECTOR (--path PATH | --match TEXT [--role ROLE] [--occurrence N]) [--frame-fallback]
|
|
100
|
+
set-value SELECTOR (--path PATH | --match TEXT) --value TEXT
|
|
101
|
+
type SELECTOR --text TEXT
|
|
102
|
+
key SELECTOR --key KEY [--modifiers cmd,shift,ctrl,opt]
|
|
103
|
+
click-point SELECTOR --x X --y Y
|
|
104
|
+
quit SELECTOR [--force]
|
|
105
|
+
"""
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
func copyAttribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
|
|
109
|
+
var value: CFTypeRef?
|
|
110
|
+
guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else {
|
|
111
|
+
return nil
|
|
112
|
+
}
|
|
113
|
+
return value
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
func stringAttribute(_ element: AXUIElement, _ name: String) -> String? {
|
|
117
|
+
copyAttribute(element, name) as? String
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
func scalarAttribute(_ element: AXUIElement, _ name: String) -> String? {
|
|
121
|
+
guard let value = copyAttribute(element, name) else { return nil }
|
|
122
|
+
if let string = value as? String { return string }
|
|
123
|
+
if CFGetTypeID(value) == CFBooleanGetTypeID(), let boolean = value as? NSNumber {
|
|
124
|
+
return boolean.boolValue ? "true" : "false"
|
|
125
|
+
}
|
|
126
|
+
if let number = value as? NSNumber { return number.stringValue }
|
|
127
|
+
return nil
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
func children(_ element: AXUIElement) -> [AXUIElement] {
|
|
131
|
+
copyAttribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? []
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
func pointAttribute(_ element: AXUIElement, _ name: String) -> CGPoint? {
|
|
135
|
+
guard let raw = copyAttribute(element, name), CFGetTypeID(raw) == AXValueGetTypeID() else {
|
|
136
|
+
return nil
|
|
137
|
+
}
|
|
138
|
+
var point = CGPoint.zero
|
|
139
|
+
guard AXValueGetValue(raw as! AXValue, .cgPoint, &point) else { return nil }
|
|
140
|
+
return point
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
func sizeAttribute(_ element: AXUIElement, _ name: String) -> CGSize? {
|
|
144
|
+
guard let raw = copyAttribute(element, name), CFGetTypeID(raw) == AXValueGetTypeID() else {
|
|
145
|
+
return nil
|
|
146
|
+
}
|
|
147
|
+
var size = CGSize.zero
|
|
148
|
+
guard AXValueGetValue(raw as! AXValue, .cgSize, &size) else { return nil }
|
|
149
|
+
return size
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
func frame(_ element: AXUIElement) -> CGRect? {
|
|
153
|
+
guard let position = pointAttribute(element, kAXPositionAttribute),
|
|
154
|
+
let size = sizeAttribute(element, kAXSizeAttribute)
|
|
155
|
+
else { return nil }
|
|
156
|
+
return CGRect(origin: position, size: size)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
func compact(_ value: String?, limit: Int = 72) -> String? {
|
|
160
|
+
guard let value else { return nil }
|
|
161
|
+
let singleLine = value
|
|
162
|
+
.replacingOccurrences(of: "\n", with: " ")
|
|
163
|
+
.replacingOccurrences(of: "\r", with: " ")
|
|
164
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
165
|
+
guard !singleLine.isEmpty else { return nil }
|
|
166
|
+
if singleLine.count <= limit { return singleLine }
|
|
167
|
+
return String(singleLine.prefix(limit - 1)) + "…"
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
func role(_ element: AXUIElement) -> String {
|
|
171
|
+
stringAttribute(element, kAXRoleAttribute) ?? "AXUnknown"
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
func fields(_ element: AXUIElement) -> [(String, String)] {
|
|
175
|
+
let elementRole = role(element)
|
|
176
|
+
var result: [(String, String)] = []
|
|
177
|
+
let pairs = [
|
|
178
|
+
("title", kAXTitleAttribute),
|
|
179
|
+
("description", kAXDescriptionAttribute),
|
|
180
|
+
("identifier", kAXIdentifierAttribute),
|
|
181
|
+
("help", kAXHelpAttribute),
|
|
182
|
+
]
|
|
183
|
+
for (label, attribute) in pairs {
|
|
184
|
+
if let value = compact(stringAttribute(element, attribute)) {
|
|
185
|
+
result.append((label, value))
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if elementRole == "AXSecureTextField" {
|
|
189
|
+
result.append(("value", "<redacted>"))
|
|
190
|
+
} else if let value = compact(scalarAttribute(element, kAXValueAttribute)) {
|
|
191
|
+
result.append(("value", value))
|
|
192
|
+
}
|
|
193
|
+
for (label, attribute) in [
|
|
194
|
+
("enabled", kAXEnabledAttribute),
|
|
195
|
+
("focused", kAXFocusedAttribute),
|
|
196
|
+
("selected", kAXSelectedAttribute),
|
|
197
|
+
] {
|
|
198
|
+
if let value = scalarAttribute(element, attribute) {
|
|
199
|
+
result.append((label, value))
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return result
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
func quoted(_ value: String) -> String {
|
|
206
|
+
"\"" + value.replacingOccurrences(of: "\"", with: "\\\"") + "\""
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
func summary(_ element: AXUIElement, path: String) -> String {
|
|
210
|
+
var parts = [path, role(element)]
|
|
211
|
+
parts.append(contentsOf: fields(element).map { "\($0.0)=\(quoted($0.1))" })
|
|
212
|
+
if let rectangle = frame(element) {
|
|
213
|
+
parts.append(
|
|
214
|
+
"frame=\(Int(rectangle.origin.x)),\(Int(rectangle.origin.y))," +
|
|
215
|
+
"\(Int(rectangle.width))x\(Int(rectangle.height))"
|
|
216
|
+
)
|
|
217
|
+
}
|
|
218
|
+
return parts.joined(separator: " ")
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
func findApplication(_ arguments: Arguments) throws -> NSRunningApplication {
|
|
222
|
+
let requestedName = arguments.value("app")
|
|
223
|
+
let requestedBundle = arguments.value("bundle-id")
|
|
224
|
+
let requestedPID = try arguments.value("pid").map {
|
|
225
|
+
guard let pid = Int32($0), pid > 0 else { throw Failure("invalid --pid: \($0)") }
|
|
226
|
+
return pid
|
|
227
|
+
}
|
|
228
|
+
let requestedPGID = try arguments.value("pgid").map {
|
|
229
|
+
guard let pgid = Int32($0), pgid > 0 else { throw Failure("invalid --pgid: \($0)") }
|
|
230
|
+
return pgid
|
|
231
|
+
}
|
|
232
|
+
guard requestedName != nil || requestedBundle != nil || requestedPID != nil || requestedPGID != nil else {
|
|
233
|
+
throw Failure("missing app selector (--pid, --pgid, --app, or --bundle-id)")
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// `NSWorkspace.runningApplications` can remain stale when this helper starts
|
|
237
|
+
// before a newly spawned GUI process has registered with LaunchServices.
|
|
238
|
+
// Resolve an explicit PID directly so wait-window can observe that process
|
|
239
|
+
// once AppKit finishes launching instead of polling a cached empty snapshot.
|
|
240
|
+
var candidates: [NSRunningApplication]
|
|
241
|
+
if let requestedPID, let application = NSRunningApplication(processIdentifier: requestedPID),
|
|
242
|
+
!application.isTerminated
|
|
243
|
+
{
|
|
244
|
+
candidates = [application]
|
|
245
|
+
} else {
|
|
246
|
+
candidates = NSWorkspace.shared.runningApplications.filter { !$0.isTerminated }
|
|
247
|
+
if requestedPID != nil { candidates = [] }
|
|
248
|
+
}
|
|
249
|
+
if let requestedPGID {
|
|
250
|
+
candidates = candidates.filter { getpgid($0.processIdentifier) == requestedPGID }
|
|
251
|
+
}
|
|
252
|
+
if let requestedBundle {
|
|
253
|
+
candidates = candidates.filter { $0.bundleIdentifier == requestedBundle }
|
|
254
|
+
}
|
|
255
|
+
if let requestedName {
|
|
256
|
+
let exact = candidates.filter {
|
|
257
|
+
$0.localizedName?.caseInsensitiveCompare(requestedName) == .orderedSame
|
|
258
|
+
}
|
|
259
|
+
if !exact.isEmpty {
|
|
260
|
+
candidates = exact
|
|
261
|
+
} else {
|
|
262
|
+
let partial = candidates.filter {
|
|
263
|
+
$0.localizedName?.localizedCaseInsensitiveContains(requestedName) == true
|
|
264
|
+
}
|
|
265
|
+
if !partial.isEmpty {
|
|
266
|
+
candidates = partial
|
|
267
|
+
} else if AXIsProcessTrusted() {
|
|
268
|
+
candidates = candidates.filter { app in
|
|
269
|
+
let application = axApplication(app)
|
|
270
|
+
let windows = copyAttribute(application, kAXWindowsAttribute)
|
|
271
|
+
as? [AXUIElement] ?? []
|
|
272
|
+
return windows.contains { window in
|
|
273
|
+
stringAttribute(window, kAXTitleAttribute)?
|
|
274
|
+
.localizedCaseInsensitiveContains(requestedName) == true
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
guard !candidates.isEmpty else {
|
|
281
|
+
throw Failure("no running app matches the supplied selector")
|
|
282
|
+
}
|
|
283
|
+
guard candidates.count == 1 else {
|
|
284
|
+
let preview = candidates.prefix(10).map {
|
|
285
|
+
"\($0.localizedName ?? "?") pid=\($0.processIdentifier)"
|
|
286
|
+
}.joined(separator: ", ")
|
|
287
|
+
throw Failure("ambiguous app selector (\(candidates.count) matches): \(preview); use --pid")
|
|
288
|
+
}
|
|
289
|
+
return candidates[0]
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
func axApplication(_ app: NSRunningApplication) -> AXUIElement {
|
|
293
|
+
AXUIElementCreateApplication(app.processIdentifier)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
func frontWindow(_ app: NSRunningApplication) throws -> AXUIElement {
|
|
297
|
+
let application = axApplication(app)
|
|
298
|
+
if let focused = copyAttribute(application, kAXFocusedWindowAttribute) {
|
|
299
|
+
return unsafeBitCast(focused, to: AXUIElement.self)
|
|
300
|
+
}
|
|
301
|
+
guard let window = (copyAttribute(application, kAXWindowsAttribute) as? [AXUIElement])?.first else {
|
|
302
|
+
throw Failure("no accessible window for \(app.localizedName ?? "app")")
|
|
303
|
+
}
|
|
304
|
+
return window
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
func focusedElement(_ app: NSRunningApplication) throws -> AXUIElement {
|
|
308
|
+
let application = axApplication(app)
|
|
309
|
+
guard let focused = copyAttribute(application, kAXFocusedUIElementAttribute) else {
|
|
310
|
+
throw Failure("app has no focused accessibility element")
|
|
311
|
+
}
|
|
312
|
+
return unsafeBitCast(focused, to: AXUIElement.self)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
func requireAccessibility() throws {
|
|
316
|
+
guard AXIsProcessTrusted() else {
|
|
317
|
+
throw Failure("Accessibility permission is disabled; run doctor --prompt")
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
struct LocatedElement {
|
|
322
|
+
let element: AXUIElement
|
|
323
|
+
let path: String
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
func printJSON(_ value: Any) throws {
|
|
327
|
+
let data = try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
|
|
328
|
+
guard let encoded = String(data: data, encoding: .utf8) else {
|
|
329
|
+
throw Failure("could not encode accessibility result")
|
|
330
|
+
}
|
|
331
|
+
print(encoded)
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
func record(_ located: LocatedElement) -> [String: Any] {
|
|
335
|
+
var attributes: [String: String] = [:]
|
|
336
|
+
for (name, value) in fields(located.element) { attributes[name] = value }
|
|
337
|
+
var result: [String: Any] = [
|
|
338
|
+
"path": located.path,
|
|
339
|
+
"role": role(located.element),
|
|
340
|
+
"attributes": attributes,
|
|
341
|
+
]
|
|
342
|
+
if let rectangle = frame(located.element) {
|
|
343
|
+
result["frame"] = [
|
|
344
|
+
"x": rectangle.origin.x,
|
|
345
|
+
"y": rectangle.origin.y,
|
|
346
|
+
"width": rectangle.width,
|
|
347
|
+
"height": rectangle.height,
|
|
348
|
+
]
|
|
349
|
+
}
|
|
350
|
+
return result
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
func walk(
|
|
354
|
+
_ root: AXUIElement,
|
|
355
|
+
maxDepth: Int,
|
|
356
|
+
maxVisited: Int = 2_000
|
|
357
|
+
) -> [LocatedElement] {
|
|
358
|
+
var found: [LocatedElement] = []
|
|
359
|
+
var visited = 0
|
|
360
|
+
|
|
361
|
+
func visit(_ element: AXUIElement, path: String, depth: Int) {
|
|
362
|
+
guard visited < maxVisited, depth <= maxDepth else { return }
|
|
363
|
+
visited += 1
|
|
364
|
+
found.append(LocatedElement(element: element, path: path))
|
|
365
|
+
for (index, child) in children(element).enumerated() {
|
|
366
|
+
visit(child, path: "\(path).\(index)", depth: depth + 1)
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
visit(root, path: "0", depth: 0)
|
|
371
|
+
return found
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
func resolvePath(_ root: AXUIElement, path: String) throws -> LocatedElement {
|
|
375
|
+
let components = path.split(separator: ".").compactMap { Int($0) }
|
|
376
|
+
guard components.first == 0, components.count == path.split(separator: ".").count else {
|
|
377
|
+
throw Failure("invalid AX path: \(path)")
|
|
378
|
+
}
|
|
379
|
+
var element = root
|
|
380
|
+
var traversed = "0"
|
|
381
|
+
for index in components.dropFirst() {
|
|
382
|
+
let descendants = children(element)
|
|
383
|
+
guard descendants.indices.contains(index) else {
|
|
384
|
+
throw Failure("stale AX path at \(traversed).\(index); inspect again")
|
|
385
|
+
}
|
|
386
|
+
element = descendants[index]
|
|
387
|
+
traversed += ".\(index)"
|
|
388
|
+
}
|
|
389
|
+
return LocatedElement(element: element, path: traversed)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
func normalizedRole(_ value: String) -> String {
|
|
393
|
+
value.hasPrefix("AX") ? value : "AX" + value.prefix(1).uppercased() + value.dropFirst()
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
func locate(_ root: AXUIElement, arguments: Arguments) throws -> LocatedElement {
|
|
397
|
+
if let path = arguments.value("path") {
|
|
398
|
+
return try resolvePath(root, path: path)
|
|
399
|
+
}
|
|
400
|
+
let query = try arguments.required("match").lowercased()
|
|
401
|
+
let requestedRole = arguments.value("role").map(normalizedRole)
|
|
402
|
+
let candidates = walk(root, maxDepth: 12).filter { candidate in
|
|
403
|
+
if let requestedRole, role(candidate.element) != requestedRole { return false }
|
|
404
|
+
return fields(candidate.element).contains {
|
|
405
|
+
$0.1.lowercased().contains(query)
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
guard !candidates.isEmpty else {
|
|
409
|
+
throw Failure("no accessibility element matches \(quoted(query))")
|
|
410
|
+
}
|
|
411
|
+
if let rawOccurrence = arguments.value("occurrence") {
|
|
412
|
+
guard let occurrence = Int(rawOccurrence), occurrence > 0,
|
|
413
|
+
candidates.indices.contains(occurrence - 1)
|
|
414
|
+
else {
|
|
415
|
+
throw Failure("invalid --occurrence \(rawOccurrence); matches=\(candidates.count)")
|
|
416
|
+
}
|
|
417
|
+
return candidates[occurrence - 1]
|
|
418
|
+
}
|
|
419
|
+
guard candidates.count == 1 else {
|
|
420
|
+
let preview = candidates.prefix(10).map {
|
|
421
|
+
" " + summary($0.element, path: $0.path)
|
|
422
|
+
}.joined(separator: "\n")
|
|
423
|
+
throw Failure("ambiguous match (\(candidates.count)); use --path or --occurrence:\n\(preview)")
|
|
424
|
+
}
|
|
425
|
+
return candidates[0]
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
func activate(_ app: NSRunningApplication) throws {
|
|
429
|
+
let activated = app.activate(options: [.activateAllWindows])
|
|
430
|
+
let application = axApplication(app)
|
|
431
|
+
_ = AXUIElementSetAttributeValue(
|
|
432
|
+
application,
|
|
433
|
+
kAXFrontmostAttribute as CFString,
|
|
434
|
+
kCFBooleanTrue
|
|
435
|
+
)
|
|
436
|
+
if let window = try? frontWindow(app) {
|
|
437
|
+
_ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
|
|
438
|
+
}
|
|
439
|
+
guard activated || app.isActive else {
|
|
440
|
+
throw Failure("could not activate \(app.localizedName ?? "app")")
|
|
441
|
+
}
|
|
442
|
+
usleep(150_000)
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
func clickGlobal(_ point: CGPoint) throws {
|
|
446
|
+
guard let move = CGEvent(
|
|
447
|
+
mouseEventSource: nil,
|
|
448
|
+
mouseType: .mouseMoved,
|
|
449
|
+
mouseCursorPosition: point,
|
|
450
|
+
mouseButton: .left
|
|
451
|
+
),
|
|
452
|
+
let down = CGEvent(
|
|
453
|
+
mouseEventSource: nil,
|
|
454
|
+
mouseType: .leftMouseDown,
|
|
455
|
+
mouseCursorPosition: point,
|
|
456
|
+
mouseButton: .left
|
|
457
|
+
),
|
|
458
|
+
let up = CGEvent(
|
|
459
|
+
mouseEventSource: nil,
|
|
460
|
+
mouseType: .leftMouseUp,
|
|
461
|
+
mouseCursorPosition: point,
|
|
462
|
+
mouseButton: .left
|
|
463
|
+
)
|
|
464
|
+
else { throw Failure("could not create mouse event") }
|
|
465
|
+
move.post(tap: .cghidEventTap)
|
|
466
|
+
down.post(tap: .cghidEventTap)
|
|
467
|
+
usleep(40_000)
|
|
468
|
+
up.post(tap: .cghidEventTap)
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
func press(_ located: LocatedElement, allowFrameFallback: Bool) throws -> String {
|
|
472
|
+
let error = AXUIElementPerformAction(located.element, kAXPressAction as CFString)
|
|
473
|
+
if error == .success { return "accessibility" }
|
|
474
|
+
guard allowFrameFallback else {
|
|
475
|
+
throw Failure(
|
|
476
|
+
"AXPress failed (\(error.rawValue)) for path=\(located.path) role=\(role(located.element)); " +
|
|
477
|
+
"inspect again or explicitly pass --frame-fallback"
|
|
478
|
+
)
|
|
479
|
+
}
|
|
480
|
+
guard let rectangle = frame(located.element), !rectangle.isEmpty else {
|
|
481
|
+
throw Failure("AXPress failed (\(error.rawValue)) and element has no frame")
|
|
482
|
+
}
|
|
483
|
+
try clickGlobal(CGPoint(x: rectangle.midX, y: rectangle.midY))
|
|
484
|
+
return "frame-center-fallback"
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
func sendText(_ text: String) throws {
|
|
488
|
+
guard let source = CGEventSource(stateID: .combinedSessionState) else {
|
|
489
|
+
throw Failure("could not create keyboard event source")
|
|
490
|
+
}
|
|
491
|
+
let utf16 = Array(text.utf16)
|
|
492
|
+
for start in stride(from: 0, to: utf16.count, by: 32) {
|
|
493
|
+
let chunk = Array(utf16[start ..< min(start + 32, utf16.count)])
|
|
494
|
+
guard let down = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true),
|
|
495
|
+
let up = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false)
|
|
496
|
+
else { throw Failure("could not create keyboard event") }
|
|
497
|
+
chunk.withUnsafeBufferPointer { buffer in
|
|
498
|
+
down.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress)
|
|
499
|
+
up.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: buffer.baseAddress)
|
|
500
|
+
}
|
|
501
|
+
down.post(tap: .cghidEventTap)
|
|
502
|
+
up.post(tap: .cghidEventTap)
|
|
503
|
+
usleep(10_000)
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
func keyCode(_ name: String) throws -> CGKeyCode {
|
|
508
|
+
let codes: [String: CGKeyCode] = [
|
|
509
|
+
"return": 36, "enter": 36, "tab": 48, "space": 49, "delete": 51,
|
|
510
|
+
"escape": 53, "esc": 53, "home": 115, "end": 119, "pageup": 116,
|
|
511
|
+
"pagedown": 121, "left": 123, "right": 124, "down": 125, "up": 126,
|
|
512
|
+
"a": 0, "c": 8, "v": 9, "x": 7, "z": 6,
|
|
513
|
+
]
|
|
514
|
+
guard let code = codes[name.lowercased()] else {
|
|
515
|
+
throw Failure("unsupported key: \(name)")
|
|
516
|
+
}
|
|
517
|
+
return code
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
func modifierFlags(_ raw: String?) throws -> CGEventFlags {
|
|
521
|
+
var flags: CGEventFlags = []
|
|
522
|
+
for modifier in raw?.split(separator: ",").map(String.init) ?? [] {
|
|
523
|
+
switch modifier.lowercased() {
|
|
524
|
+
case "cmd", "command": flags.insert(.maskCommand)
|
|
525
|
+
case "shift": flags.insert(.maskShift)
|
|
526
|
+
case "ctrl", "control": flags.insert(.maskControl)
|
|
527
|
+
case "opt", "option", "alt": flags.insert(.maskAlternate)
|
|
528
|
+
case "fn": flags.insert(.maskSecondaryFn)
|
|
529
|
+
default: throw Failure("unsupported modifier: \(modifier)")
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
return flags
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
func sendKey(name: String, modifiers: String?) throws {
|
|
536
|
+
let code = try keyCode(name)
|
|
537
|
+
let flags = try modifierFlags(modifiers)
|
|
538
|
+
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
|
|
539
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
|
|
540
|
+
else { throw Failure("could not create keyboard event") }
|
|
541
|
+
down.flags = flags
|
|
542
|
+
up.flags = flags
|
|
543
|
+
down.post(tap: .cghidEventTap)
|
|
544
|
+
up.post(tap: .cghidEventTap)
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// Correlates the app's focused AX window with the on-screen CGWindow that owns
|
|
548
|
+
// it so callers can target exactly that window (never a region or a full
|
|
549
|
+
// display) without shell involvement.
|
|
550
|
+
func correlatedWindowID(_ app: NSRunningApplication, window: AXUIElement) throws -> CGWindowID {
|
|
551
|
+
let targetFrame = try frame(window).unwrap("focused window has no frame")
|
|
552
|
+
let raw = CGWindowListCopyWindowInfo(
|
|
553
|
+
[.optionOnScreenOnly, .excludeDesktopElements],
|
|
554
|
+
kCGNullWindowID
|
|
555
|
+
) as? [[String: Any]] ?? []
|
|
556
|
+
let candidates = raw.compactMap { info -> (CGWindowID, CGRect)? in
|
|
557
|
+
guard (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
|
|
558
|
+
(info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
|
|
559
|
+
let number = (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value,
|
|
560
|
+
let bounds = info[kCGWindowBounds as String] as? NSDictionary,
|
|
561
|
+
let rectangle = CGRect(dictionaryRepresentation: bounds),
|
|
562
|
+
rectangle.width >= 40,
|
|
563
|
+
rectangle.height >= 40
|
|
564
|
+
else { return nil }
|
|
565
|
+
return (number, rectangle)
|
|
566
|
+
}
|
|
567
|
+
guard !candidates.isEmpty else {
|
|
568
|
+
throw Failure("no on-screen window found for screenshot")
|
|
569
|
+
}
|
|
570
|
+
let scored = candidates.map { candidate -> (CGWindowID, CGFloat) in
|
|
571
|
+
let rectangle = candidate.1
|
|
572
|
+
let score = abs(rectangle.minX - targetFrame.minX) + abs(rectangle.minY - targetFrame.minY) +
|
|
573
|
+
abs(rectangle.width - targetFrame.width) + abs(rectangle.height - targetFrame.height)
|
|
574
|
+
return (candidate.0, score)
|
|
575
|
+
}.sorted { $0.1 < $1.1 }
|
|
576
|
+
guard let selected = scored.first, selected.1 <= 12 else {
|
|
577
|
+
throw Failure("could not correlate the focused AX window with an on-screen window")
|
|
578
|
+
}
|
|
579
|
+
return selected.0
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
func screenshotWindow(_ app: NSRunningApplication, window: AXUIElement, output: String) throws {
|
|
583
|
+
let selected = try correlatedWindowID(app, window: window)
|
|
584
|
+
|
|
585
|
+
let destination = URL(fileURLWithPath: output).standardizedFileURL
|
|
586
|
+
try FileManager.default.createDirectory(
|
|
587
|
+
at: destination.deletingLastPathComponent(),
|
|
588
|
+
withIntermediateDirectories: true
|
|
589
|
+
)
|
|
590
|
+
let task = Process()
|
|
591
|
+
task.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
|
|
592
|
+
task.arguments = ["-x", "-o", "-l", String(selected), destination.path]
|
|
593
|
+
try task.run()
|
|
594
|
+
task.waitUntilExit()
|
|
595
|
+
guard task.terminationStatus == 0, FileManager.default.fileExists(atPath: destination.path) else {
|
|
596
|
+
throw Failure("screencapture failed with status \(task.terminationStatus)")
|
|
597
|
+
}
|
|
598
|
+
print(destination.path)
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
// Swift screen-recording producer: captures exactly the on-screen window that
|
|
602
|
+
// backs the app's focused AX window through ScreenCaptureKit's
|
|
603
|
+
// desktop-independent window filter and encodes it to a silent H.264 MP4 with
|
|
604
|
+
// AVAssetWriter. There is deliberately no display/region fallback, no audio,
|
|
605
|
+
// and no screencapture(1) involvement for video.
|
|
606
|
+
@available(macOS 12.3, *)
|
|
607
|
+
final class WindowVideoRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
|
608
|
+
let queue = DispatchQueue(label: "ui-qa.window-video-recorder")
|
|
609
|
+
private let writer: AVAssetWriter
|
|
610
|
+
private let input: AVAssetWriterInput
|
|
611
|
+
private let adaptor: AVAssetWriterInputPixelBufferAdaptor
|
|
612
|
+
private let completed = DispatchSemaphore(value: 0)
|
|
613
|
+
private var stream: SCStream?
|
|
614
|
+
private var duration: Double
|
|
615
|
+
private var sessionStarted = false
|
|
616
|
+
private var stopping = false
|
|
617
|
+
private var failure: String?
|
|
618
|
+
private var firstSourceTime: CMTime?
|
|
619
|
+
private var lastSourceTime: CMTime?
|
|
620
|
+
private var lastPixelBuffer: CVPixelBuffer?
|
|
621
|
+
private var firstFrameUptime: UInt64?
|
|
622
|
+
|
|
623
|
+
init(output: URL, width: Int, height: Int, duration: Double) throws {
|
|
624
|
+
writer = try AVAssetWriter(outputURL: output, fileType: .mp4)
|
|
625
|
+
input = AVAssetWriterInput(mediaType: .video, outputSettings: [
|
|
626
|
+
AVVideoCodecKey: AVVideoCodecType.h264,
|
|
627
|
+
AVVideoWidthKey: width,
|
|
628
|
+
AVVideoHeightKey: height,
|
|
629
|
+
])
|
|
630
|
+
input.expectsMediaDataInRealTime = true
|
|
631
|
+
adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: nil)
|
|
632
|
+
writer.add(input)
|
|
633
|
+
self.duration = duration
|
|
634
|
+
super.init()
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
func start(filter: SCContentFilter, configuration: SCStreamConfiguration) throws {
|
|
638
|
+
let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
|
|
639
|
+
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
|
640
|
+
self.stream = stream
|
|
641
|
+
guard writer.startWriting() else {
|
|
642
|
+
throw Failure("could not start the video asset writer: \(writer.error.map { String(describing: $0) } ?? "unknown error")")
|
|
643
|
+
}
|
|
644
|
+
let started = DispatchSemaphore(value: 0)
|
|
645
|
+
var startError: Error?
|
|
646
|
+
stream.startCapture { error in
|
|
647
|
+
startError = error
|
|
648
|
+
started.signal()
|
|
649
|
+
}
|
|
650
|
+
guard started.wait(timeout: .now() + 10) == .success else {
|
|
651
|
+
throw Failure("timed out starting the window capture stream")
|
|
652
|
+
}
|
|
653
|
+
if let startError {
|
|
654
|
+
throw Failure("could not start the window capture stream: \(startError.localizedDescription)")
|
|
655
|
+
}
|
|
656
|
+
// Hard cap enforced inside the helper regardless of caller behavior.
|
|
657
|
+
queue.asyncAfter(deadline: .now() + duration) { self.stopNow() }
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
func requestStop() {
|
|
661
|
+
queue.async { self.stopNow() }
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// Blocks until the recording is finalized; returns the failure reason, if any.
|
|
665
|
+
func waitAndFinish() -> String? {
|
|
666
|
+
let margin: Double = 15
|
|
667
|
+
if completed.wait(timeout: .now() + duration + margin) == .timedOut {
|
|
668
|
+
requestStop()
|
|
669
|
+
_ = completed.wait(timeout: .now() + margin)
|
|
670
|
+
}
|
|
671
|
+
return failure
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
private func stopNow() {
|
|
675
|
+
guard !stopping else { return }
|
|
676
|
+
stopping = true
|
|
677
|
+
let group = DispatchGroup()
|
|
678
|
+
if let stream {
|
|
679
|
+
group.enter()
|
|
680
|
+
stream.stopCapture { _ in group.leave() }
|
|
681
|
+
}
|
|
682
|
+
group.notify(queue: queue) {
|
|
683
|
+
guard self.sessionStarted else {
|
|
684
|
+
if self.failure == nil { self.failure = "no frames were captured from the target window" }
|
|
685
|
+
self.completed.signal()
|
|
686
|
+
return
|
|
687
|
+
}
|
|
688
|
+
self.appendFinalFrame()
|
|
689
|
+
self.input.markAsFinished()
|
|
690
|
+
self.writer.finishWriting {
|
|
691
|
+
if self.writer.status != .completed, self.failure == nil {
|
|
692
|
+
self.failure = "video asset writer finished with status \(self.writer.status.rawValue)"
|
|
693
|
+
}
|
|
694
|
+
self.completed.signal()
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
private func appendFinalFrame() {
|
|
700
|
+
guard let firstSourceTime, let lastSourceTime, let lastPixelBuffer,
|
|
701
|
+
let firstFrameUptime, input.isReadyForMoreMediaData
|
|
702
|
+
else { return }
|
|
703
|
+
let elapsedNanoseconds = DispatchTime.now().uptimeNanoseconds - firstFrameUptime
|
|
704
|
+
let elapsedSeconds = min(Double(elapsedNanoseconds) / 1_000_000_000, duration)
|
|
705
|
+
var finalTime = CMTimeAdd(
|
|
706
|
+
firstSourceTime,
|
|
707
|
+
CMTime(seconds: elapsedSeconds, preferredTimescale: 600)
|
|
708
|
+
)
|
|
709
|
+
if CMTimeCompare(finalTime, lastSourceTime) <= 0 {
|
|
710
|
+
finalTime = CMTimeAdd(lastSourceTime, CMTime(value: 1, timescale: 15))
|
|
711
|
+
}
|
|
712
|
+
if !adaptor.append(lastPixelBuffer, withPresentationTime: finalTime), failure == nil {
|
|
713
|
+
failure = "video final-frame append failed: \(writer.error.map { String(describing: $0) } ?? "unknown error")"
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
private func frameStatus(_ sampleBuffer: CMSampleBuffer) -> SCFrameStatus {
|
|
718
|
+
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false)
|
|
719
|
+
as? [[SCStreamFrameInfo: Any]]
|
|
720
|
+
guard let raw = attachments?.first?[.status] else { return .complete }
|
|
721
|
+
if let status = raw as? SCFrameStatus { return status }
|
|
722
|
+
if let number = raw as? NSNumber, let status = SCFrameStatus(rawValue: number.intValue) { return status }
|
|
723
|
+
return .complete
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
|
|
727
|
+
guard outputType == .screen, !stopping else { return }
|
|
728
|
+
let status = frameStatus(sampleBuffer)
|
|
729
|
+
// ScreenCaptureKit marks unchanged-window frames as `idle`; they still
|
|
730
|
+
// carry the current window surface and must be encoded so a static UI
|
|
731
|
+
// produces a truthful non-zero-duration timeline.
|
|
732
|
+
guard status == .complete || status == .started || status == .idle else { return }
|
|
733
|
+
let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
|
734
|
+
if !sessionStarted {
|
|
735
|
+
writer.startSession(atSourceTime: timestamp)
|
|
736
|
+
sessionStarted = true
|
|
737
|
+
firstSourceTime = timestamp
|
|
738
|
+
firstFrameUptime = DispatchTime.now().uptimeNanoseconds
|
|
739
|
+
}
|
|
740
|
+
guard writer.status == .writing, input.isReadyForMoreMediaData,
|
|
741
|
+
let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
|
|
742
|
+
else { return }
|
|
743
|
+
if adaptor.append(pixelBuffer, withPresentationTime: timestamp) {
|
|
744
|
+
lastSourceTime = timestamp
|
|
745
|
+
lastPixelBuffer = pixelBuffer
|
|
746
|
+
return
|
|
747
|
+
}
|
|
748
|
+
if failure == nil {
|
|
749
|
+
failure = "video pixel buffer append failed: \(writer.error.map { String(describing: $0) } ?? "unknown error")"
|
|
750
|
+
}
|
|
751
|
+
stopNow()
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
|
755
|
+
// The captured window or its app died: finalize best-effort. A clean
|
|
756
|
+
// programmatic stopCapture never reaches this callback.
|
|
757
|
+
queue.async {
|
|
758
|
+
if !self.stopping, self.failure == nil {
|
|
759
|
+
self.failure = "capture stream stopped early: \(error.localizedDescription)"
|
|
760
|
+
}
|
|
761
|
+
self.stopNow()
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
private var recorderSignalSources: [DispatchSourceSignal] = []
|
|
767
|
+
private let recorderSignalLock = NSLock()
|
|
768
|
+
private var activeRecorder: WindowVideoRecorder?
|
|
769
|
+
private var recorderStopRequestedStorage = false
|
|
770
|
+
|
|
771
|
+
private var recorderStopRequested: Bool {
|
|
772
|
+
get {
|
|
773
|
+
recorderSignalLock.lock()
|
|
774
|
+
defer { recorderSignalLock.unlock() }
|
|
775
|
+
return recorderStopRequestedStorage
|
|
776
|
+
}
|
|
777
|
+
set {
|
|
778
|
+
recorderSignalLock.lock()
|
|
779
|
+
recorderStopRequestedStorage = newValue
|
|
780
|
+
recorderSignalLock.unlock()
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// Installs graceful-stop handling for the record-window command. This must run
|
|
785
|
+
// before any slow startup work (app activation, shareable-content
|
|
786
|
+
// enumeration), because the desktop backend may legitimately SIGTERM the
|
|
787
|
+
// recorder early to stop and finalize.
|
|
788
|
+
func installRecorderSignalHandlers() {
|
|
789
|
+
signal(SIGTERM, SIG_IGN)
|
|
790
|
+
signal(SIGINT, SIG_IGN)
|
|
791
|
+
for number in [SIGTERM, SIGINT] {
|
|
792
|
+
let source = DispatchSource.makeSignalSource(signal: number, queue: DispatchQueue.global())
|
|
793
|
+
source.setEventHandler {
|
|
794
|
+
recorderStopRequested = true
|
|
795
|
+
activeRecorder?.requestStop()
|
|
796
|
+
}
|
|
797
|
+
source.resume()
|
|
798
|
+
recorderSignalSources.append(source)
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
@available(macOS 12.3, *)
|
|
803
|
+
func recordWindowVideo(app: NSRunningApplication, window: AXUIElement, arguments: Arguments) throws {
|
|
804
|
+
let windowID = try correlatedWindowID(app, window: window)
|
|
805
|
+
let output = URL(fileURLWithPath: try arguments.required("out")).standardizedFileURL
|
|
806
|
+
try FileManager.default.createDirectory(
|
|
807
|
+
at: output.deletingLastPathComponent(),
|
|
808
|
+
withIntermediateDirectories: true
|
|
809
|
+
)
|
|
810
|
+
let temporary = URL(fileURLWithPath: "\(output.path).\(ProcessInfo.processInfo.processIdentifier).tmp")
|
|
811
|
+
|
|
812
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
813
|
+
var content: SCShareableContent?
|
|
814
|
+
var enumerationError: Error?
|
|
815
|
+
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: true) { result, error in
|
|
816
|
+
content = result
|
|
817
|
+
enumerationError = error
|
|
818
|
+
semaphore.signal()
|
|
819
|
+
}
|
|
820
|
+
guard semaphore.wait(timeout: .now() + 15) == .success else {
|
|
821
|
+
throw Failure("timed out enumerating capturable windows")
|
|
822
|
+
}
|
|
823
|
+
if let enumerationError {
|
|
824
|
+
throw Failure("could not enumerate capturable windows: \(enumerationError.localizedDescription)")
|
|
825
|
+
}
|
|
826
|
+
guard let content, let target = content.windows.first(where: { $0.windowID == windowID }) else {
|
|
827
|
+
throw Failure("window \(windowID) is not present among capturable on-screen windows")
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
let requested = try arguments.double("duration", default: 30)
|
|
831
|
+
let duration = min(max(requested, 0.5), 30)
|
|
832
|
+
let windowFrame = try frame(window).unwrap("focused window has no frame")
|
|
833
|
+
let scale = (NSScreen.screens.first { $0.frame.intersects(windowFrame) } ?? NSScreen.main)?.backingScaleFactor ?? 2
|
|
834
|
+
let pixelWidth = evenPixel(Double(windowFrame.width) * scale)
|
|
835
|
+
let pixelHeight = evenPixel(Double(windowFrame.height) * scale)
|
|
836
|
+
let configuration = SCStreamConfiguration()
|
|
837
|
+
configuration.width = pixelWidth
|
|
838
|
+
configuration.height = pixelHeight
|
|
839
|
+
// Independent-window streams do not scale up by default. Our encoder
|
|
840
|
+
// surface is Retina-sized, so without scalesToFit the 1x window surface is
|
|
841
|
+
// left in the top-left corner with unused space around it. Fill the exact
|
|
842
|
+
// window output surface while preserving the source aspect ratio.
|
|
843
|
+
configuration.scalesToFit = true
|
|
844
|
+
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 15)
|
|
845
|
+
configuration.showsCursor = false
|
|
846
|
+
|
|
847
|
+
let recorder = try WindowVideoRecorder(
|
|
848
|
+
output: temporary,
|
|
849
|
+
width: pixelWidth,
|
|
850
|
+
height: pixelHeight,
|
|
851
|
+
duration: duration
|
|
852
|
+
)
|
|
853
|
+
activeRecorder = recorder
|
|
854
|
+
try recorder.start(filter: SCContentFilter(desktopIndependentWindow: target), configuration: configuration)
|
|
855
|
+
print("recording-started")
|
|
856
|
+
fflush(stdout)
|
|
857
|
+
if recorderStopRequested { recorder.requestStop() }
|
|
858
|
+
let failure = recorder.waitAndFinish()
|
|
859
|
+
if let failure {
|
|
860
|
+
try? FileManager.default.removeItem(at: temporary)
|
|
861
|
+
throw Failure(failure)
|
|
862
|
+
}
|
|
863
|
+
// Publish atomically: a killed helper can never leave a partial artifact.
|
|
864
|
+
if FileManager.default.fileExists(atPath: output.path) {
|
|
865
|
+
try FileManager.default.removeItem(at: output)
|
|
866
|
+
}
|
|
867
|
+
try FileManager.default.moveItem(at: temporary, to: output)
|
|
868
|
+
try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: output.path)
|
|
869
|
+
print(output.path)
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
private func evenPixel(_ value: Double) -> Int {
|
|
873
|
+
max(2, (Int(value.rounded(.down)) / 2) * 2)
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
func isUseful(_ element: AXUIElement) -> Bool {
|
|
877
|
+
let usefulRoles: Set<String> = [
|
|
878
|
+
kAXButtonRole, kAXCheckBoxRole, kAXComboBoxRole, "AXLink",
|
|
879
|
+
kAXMenuItemRole, kAXPopUpButtonRole, kAXRadioButtonRole,
|
|
880
|
+
"AXSecureTextField", kAXSliderRole, kAXStaticTextRole,
|
|
881
|
+
kAXTextAreaRole, kAXTextFieldRole, kAXWindowRole,
|
|
882
|
+
]
|
|
883
|
+
return usefulRoles.contains(role(element)) || !fields(element).isEmpty
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
func run(_ arguments: Arguments) throws {
|
|
887
|
+
switch arguments.command {
|
|
888
|
+
case "doctor":
|
|
889
|
+
let accessibility: Bool
|
|
890
|
+
if arguments.has("prompt") {
|
|
891
|
+
let key = kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String
|
|
892
|
+
accessibility = AXIsProcessTrustedWithOptions([key: true] as CFDictionary)
|
|
893
|
+
} else {
|
|
894
|
+
accessibility = AXIsProcessTrusted()
|
|
895
|
+
}
|
|
896
|
+
let screenRecording = arguments.has("prompt")
|
|
897
|
+
? CGRequestScreenCaptureAccess()
|
|
898
|
+
: CGPreflightScreenCaptureAccess()
|
|
899
|
+
let screenCaptureKit = ProcessInfo.processInfo.isOperatingSystemAtLeast(
|
|
900
|
+
OperatingSystemVersion(majorVersion: 12, minorVersion: 3, patchVersion: 0)
|
|
901
|
+
)
|
|
902
|
+
print("macos=\(ProcessInfo.processInfo.operatingSystemVersionString)")
|
|
903
|
+
print("accessibility=\(accessibility ? "granted" : "missing")")
|
|
904
|
+
print("screen_recording=\(screenRecording ? "granted" : "missing")")
|
|
905
|
+
print("sck=\(screenCaptureKit ? "available" : "unsupported")")
|
|
906
|
+
|
|
907
|
+
case "list":
|
|
908
|
+
for app in NSWorkspace.shared.runningApplications
|
|
909
|
+
.filter({ !$0.isTerminated && $0.activationPolicy == .regular })
|
|
910
|
+
.sorted(by: { ($0.localizedName ?? "") < ($1.localizedName ?? "") })
|
|
911
|
+
{
|
|
912
|
+
print(
|
|
913
|
+
"app=\(quoted(app.localizedName ?? "?")) " +
|
|
914
|
+
"bundle=\(quoted(app.bundleIdentifier ?? "?")) pid=\(app.processIdentifier)"
|
|
915
|
+
)
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
case "wait-window":
|
|
919
|
+
try requireAccessibility()
|
|
920
|
+
let timeout = try arguments.double("timeout", default: 60)
|
|
921
|
+
let deadline = Date().addingTimeInterval(timeout)
|
|
922
|
+
while Date() < deadline {
|
|
923
|
+
if let app = try? findApplication(arguments), (try? frontWindow(app)) != nil {
|
|
924
|
+
if arguments.has("print-pid") {
|
|
925
|
+
print(app.processIdentifier)
|
|
926
|
+
} else {
|
|
927
|
+
print("ready app=\(quoted(app.localizedName ?? "?")) pid=\(app.processIdentifier)")
|
|
928
|
+
}
|
|
929
|
+
return
|
|
930
|
+
}
|
|
931
|
+
usleep(200_000)
|
|
932
|
+
}
|
|
933
|
+
throw Failure("window did not appear within \(timeout)s")
|
|
934
|
+
|
|
935
|
+
case "focus":
|
|
936
|
+
try requireAccessibility()
|
|
937
|
+
let app = try findApplication(arguments)
|
|
938
|
+
try activate(app)
|
|
939
|
+
print("focused app=\(quoted(app.localizedName ?? "?")) pid=\(app.processIdentifier)")
|
|
940
|
+
|
|
941
|
+
case "inspect":
|
|
942
|
+
try requireAccessibility()
|
|
943
|
+
let app = try findApplication(arguments)
|
|
944
|
+
let window = try frontWindow(app)
|
|
945
|
+
let depth = try arguments.integer("depth", default: 10)
|
|
946
|
+
let limit = try arguments.integer("limit", default: 80)
|
|
947
|
+
var shown = 0
|
|
948
|
+
for located in walk(window, maxDepth: depth) {
|
|
949
|
+
if !arguments.has("all"), !isUseful(located.element) { continue }
|
|
950
|
+
print(summary(located.element, path: located.path))
|
|
951
|
+
shown += 1
|
|
952
|
+
if shown >= limit { break }
|
|
953
|
+
}
|
|
954
|
+
print("shown=\(shown) limit=\(limit) depth=\(depth)")
|
|
955
|
+
|
|
956
|
+
case "describe":
|
|
957
|
+
try requireAccessibility()
|
|
958
|
+
let app = try findApplication(arguments)
|
|
959
|
+
let located = try locate(frontWindow(app), arguments: arguments)
|
|
960
|
+
try printJSON(record(located))
|
|
961
|
+
|
|
962
|
+
case "screenshot":
|
|
963
|
+
let app = try findApplication(arguments)
|
|
964
|
+
try activate(app)
|
|
965
|
+
let window = try frontWindow(app)
|
|
966
|
+
try screenshotWindow(app, window: window, output: arguments.required("out"))
|
|
967
|
+
|
|
968
|
+
case "window-id":
|
|
969
|
+
let app = try findApplication(arguments)
|
|
970
|
+
let window = try frontWindow(app)
|
|
971
|
+
print(try correlatedWindowID(app, window: window))
|
|
972
|
+
|
|
973
|
+
case "record-window":
|
|
974
|
+
// Graceful-stop handlers go in before any slow startup work so the
|
|
975
|
+
// desktop backend can always stop and finalize the recorder.
|
|
976
|
+
installRecorderSignalHandlers()
|
|
977
|
+
try requireAccessibility()
|
|
978
|
+
guard #available(macOS 12.3, *) else {
|
|
979
|
+
throw Failure("ScreenCaptureKit window recording requires macOS 12.3 or newer")
|
|
980
|
+
}
|
|
981
|
+
guard CGPreflightScreenCaptureAccess() else {
|
|
982
|
+
throw Failure("Screen Recording permission is required to record a window; run doctor --prompt")
|
|
983
|
+
}
|
|
984
|
+
let app = try findApplication(arguments)
|
|
985
|
+
try activate(app)
|
|
986
|
+
let window = try frontWindow(app)
|
|
987
|
+
try recordWindowVideo(app: app, window: window, arguments: arguments)
|
|
988
|
+
|
|
989
|
+
case "click":
|
|
990
|
+
try requireAccessibility()
|
|
991
|
+
let app = try findApplication(arguments)
|
|
992
|
+
try activate(app)
|
|
993
|
+
let located = try locate(frontWindow(app), arguments: arguments)
|
|
994
|
+
let method = try press(located, allowFrameFallback: arguments.has("frame-fallback"))
|
|
995
|
+
print("clicked method=\(method) path=\(located.path) role=\(role(located.element))")
|
|
996
|
+
|
|
997
|
+
case "set-value":
|
|
998
|
+
try requireAccessibility()
|
|
999
|
+
let app = try findApplication(arguments)
|
|
1000
|
+
try activate(app)
|
|
1001
|
+
let located = try locate(frontWindow(app), arguments: arguments)
|
|
1002
|
+
var settable = DarwinBoolean(false)
|
|
1003
|
+
let check = AXUIElementIsAttributeSettable(
|
|
1004
|
+
located.element,
|
|
1005
|
+
kAXValueAttribute as CFString,
|
|
1006
|
+
&settable
|
|
1007
|
+
)
|
|
1008
|
+
guard check == .success, settable.boolValue else {
|
|
1009
|
+
throw Failure("AX value is not settable for path=\(located.path) role=\(role(located.element))")
|
|
1010
|
+
}
|
|
1011
|
+
let value = try arguments.required("value")
|
|
1012
|
+
let result = AXUIElementSetAttributeValue(
|
|
1013
|
+
located.element,
|
|
1014
|
+
kAXValueAttribute as CFString,
|
|
1015
|
+
value as CFString
|
|
1016
|
+
)
|
|
1017
|
+
guard result == .success else {
|
|
1018
|
+
throw Failure("setting AX value failed: \(result.rawValue)")
|
|
1019
|
+
}
|
|
1020
|
+
print("set-value path=\(located.path) role=\(role(located.element))")
|
|
1021
|
+
|
|
1022
|
+
case "type":
|
|
1023
|
+
try requireAccessibility()
|
|
1024
|
+
let app = try findApplication(arguments)
|
|
1025
|
+
try activate(app)
|
|
1026
|
+
let focused = try focusedElement(app)
|
|
1027
|
+
let focusedRole = role(focused)
|
|
1028
|
+
let textRoles: Set<String> = [
|
|
1029
|
+
kAXComboBoxRole, "AXSecureTextField", kAXTextAreaRole, kAXTextFieldRole,
|
|
1030
|
+
]
|
|
1031
|
+
guard textRoles.contains(focusedRole) else {
|
|
1032
|
+
throw Failure("refusing to type: focused role is \(focusedRole), not a text input")
|
|
1033
|
+
}
|
|
1034
|
+
let text = try arguments.required("text")
|
|
1035
|
+
try sendText(text)
|
|
1036
|
+
print("typed characters=\(text.count) focused_role=\(focusedRole)")
|
|
1037
|
+
|
|
1038
|
+
case "key":
|
|
1039
|
+
try requireAccessibility()
|
|
1040
|
+
let app = try findApplication(arguments)
|
|
1041
|
+
try activate(app)
|
|
1042
|
+
let key = try arguments.required("key")
|
|
1043
|
+
try sendKey(name: key, modifiers: arguments.value("modifiers"))
|
|
1044
|
+
print("key=\(key)")
|
|
1045
|
+
|
|
1046
|
+
case "click-point":
|
|
1047
|
+
try requireAccessibility()
|
|
1048
|
+
let app = try findApplication(arguments)
|
|
1049
|
+
try activate(app)
|
|
1050
|
+
let windowFrame = try frame(frontWindow(app)).unwrap("front window has no frame")
|
|
1051
|
+
let x = try arguments.double("x")
|
|
1052
|
+
let y = try arguments.double("y")
|
|
1053
|
+
guard x >= 0, y >= 0, x <= windowFrame.width, y <= windowFrame.height else {
|
|
1054
|
+
throw Failure("relative point is outside the front window")
|
|
1055
|
+
}
|
|
1056
|
+
try clickGlobal(CGPoint(x: windowFrame.minX + x, y: windowFrame.minY + y))
|
|
1057
|
+
print("clicked relative=\(Int(x)),\(Int(y))")
|
|
1058
|
+
|
|
1059
|
+
case "quit":
|
|
1060
|
+
let app = try findApplication(arguments)
|
|
1061
|
+
let requested = arguments.has("force") ? app.forceTerminate() : app.terminate()
|
|
1062
|
+
guard requested else { throw Failure("quit request was rejected") }
|
|
1063
|
+
print("quit requested pid=\(app.processIdentifier)")
|
|
1064
|
+
|
|
1065
|
+
default:
|
|
1066
|
+
throw Failure(Arguments.usage)
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
extension Optional {
|
|
1071
|
+
func unwrap(_ message: String) throws -> Wrapped {
|
|
1072
|
+
guard let self else { throw Failure(message) }
|
|
1073
|
+
return self
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
do {
|
|
1078
|
+
try run(Arguments(Array(CommandLine.arguments.dropFirst())))
|
|
1079
|
+
} catch let error as Failure {
|
|
1080
|
+
FileHandle.standardError.write(Data("error: \(error.description)\n".utf8))
|
|
1081
|
+
exit(2)
|
|
1082
|
+
} catch {
|
|
1083
|
+
FileHandle.standardError.write(Data("error: \(error)\n".utf8))
|
|
1084
|
+
exit(2)
|
|
1085
|
+
}
|