ucu-mcp 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/src/mcp/tools/element-tools.js +14 -4
- package/dist/src/platform/base.d.ts +13 -2
- package/dist/src/platform/linux.d.ts +2 -2
- package/dist/src/platform/macos/element.d.ts +3 -2
- package/dist/src/platform/macos/element.js +139 -30
- package/dist/src/platform/macos/focus.d.ts +10 -0
- package/dist/src/platform/macos/focus.js +10 -0
- package/dist/src/platform/windows.d.ts +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,37 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.5.1] - 2026-06-16
|
|
9
|
+
|
|
10
|
+
方向4b AXPress verify-then-fallback + 方向5 FocusStealSuppression @deprecated 文档化。
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **方向4b AXPress verify-then-fallback(clickElement + clickMenuBarExtra)**:解决 Tauri/Electron 控件静默吞 AXPress(不抛异常也不执行)。双层策略:(1) 启发式跳过——对已知静默吞的应用(`AX_SILENT_APP_HINTS = ["tauri"]`,按 appName includes 匹配)直接坐标点击,跳过 AXPress;(2) verify 状态签名——对走 AXPress 的应用,采样前后 `value|focused|selected` 签名,spin 80ms 等异步生效,无变化则降级坐标。`ClickResult { method: "axpress"|"coordinate"; verified: boolean }` 透到 ActionReceipt.result,verified:false 时附 warning 引导模型用 screenshot/get_window_state 复核。无可观测状态的元素(纯按钮)保守判 verified:false 但不降级坐标(避免误伤)。共享 JXA 原语 `stateSignature`/`coordinateClick`/`spinMs`。
|
|
15
|
+
- 返回类型波及链:`clickElement`/`clickMenuBarExtra` `Promise<void>` → `Promise<ClickResult>`(base.ts 接口 + windows/linux stub + element.ts consumer + tools-layer mock)。
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
- **方向5 FocusStealSuppression @deprecated 文档化**:`focus.ts` 的 `saveFocus`/`restoreFocus` 加 `@deprecated` JSDoc,说明 CGEvent 在 HID 层工作不需前台 focus、焦点管理被有意禁用、`tools-layer.test.ts` 的 `not.toHaveBeenCalled()` 锁定禁用态。不删不接线(文档化设计决策,非遗漏)。`base.ts` 接口注释同步。
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- 无行为修复(4b/5 是 v0.5.0 的质量强化)。
|
|
24
|
+
|
|
25
|
+
### 真机验证(2026-06-16)
|
|
26
|
+
|
|
27
|
+
- ✅ doctor 全绿(3 native helpers + 权限 + 26 工具)。
|
|
28
|
+
- ✅ OCR(方向1)返回非空 elements,无 NSNull 崩溃。
|
|
29
|
+
- ✅ press_key 字母键 Cmd+M(方向2)成功合成,不报 Unknown key。
|
|
30
|
+
- ✅ describe_screen(方向3)结构正确,AX 失败时聚合到 errors[] 不抛出。
|
|
31
|
+
- ✅ focus_app(cc-switch)(方向4a)建立 tray target(windowId:"tray"),不抛 WINDOW_NOT_FOUND。
|
|
32
|
+
- ✅ click_menu_bar_extra(方向4b)返回 `method:"axpress", verified:true`——托盘菜单真实打开(OCR 确认 "About/Hide/Quit CC Switch" 菜单项可见)。
|
|
33
|
+
- ⚠️ 发现:cc-switch 托盘菜单是其**原生应用菜单**(About/Hide/Quit),"使用统计"在 WebView 设置面板内,不在托盘菜单。完整 "设置→使用统计" 流程需先打开设置窗口(非托盘菜单可达),留 follow-up。
|
|
34
|
+
|
|
35
|
+
### Tests
|
|
36
|
+
|
|
37
|
+
- 303→310(+7:clickElement ClickResult 解析、JXA stateSignature/spinMs/coordinateClick/sigBefore/sigAfter/preferCoord source 断言、Tauri 启发式 preferCoord=true、tools-layer method/verified 透出 + warnings)。
|
|
38
|
+
|
|
8
39
|
## [0.5.0] - 2026-06-15
|
|
9
40
|
|
|
10
41
|
cc-switch 操控诊断与视觉通道修复:打通 OCR + 字母快捷键 + 托盘 + 视觉降级 fallback + 内置 agent skill。
|
|
@@ -36,8 +36,13 @@ export function registerElementTools(registerTool) {
|
|
|
36
36
|
}, async (params) => {
|
|
37
37
|
const effectiveApp = params.app || getActiveTarget()?.appName;
|
|
38
38
|
const safetyCtx = await getSafetyContext();
|
|
39
|
-
await withSafety({ action: "click_element", params: { ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().clickElement(params.elementId, effectiveApp) });
|
|
40
|
-
|
|
39
|
+
const clickResult = await withSafety({ action: "click_element", params: { ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().clickElement(params.elementId, effectiveApp) });
|
|
40
|
+
const warnings = clickResult.verified
|
|
41
|
+
? []
|
|
42
|
+
: [clickResult.method === "coordinate"
|
|
43
|
+
? "AXPress produced no observable state change (or the app is known to silently swallow it); coordinate fallback was used. Re-observe with screenshot/get_window_state to confirm."
|
|
44
|
+
: "AXPress completed but the element exposed no observable state to verify against. Re-observe with screenshot/get_window_state to confirm."];
|
|
45
|
+
return actionResponse("click_element", { clicked: true, elementId: params.elementId, method: clickResult.method, verified: clickResult.verified }, { elementId: params.elementId, app: effectiveApp }, params.captureAfter, params.captureFormat, params.captureMaxWidth, warnings);
|
|
41
46
|
});
|
|
42
47
|
registerTool("set_value", "Set the value of an accessibility element", {
|
|
43
48
|
elementId: z.string().describe("AX element identifier"), value: z.string().describe("Value to set"), app: z.string().optional().describe("Target app"), ...captureAfterFields,
|
|
@@ -64,7 +69,12 @@ export function registerElementTools(registerTool) {
|
|
|
64
69
|
...captureAfterFields,
|
|
65
70
|
}, async (params) => {
|
|
66
71
|
const safetyCtx = await getSafetyContext();
|
|
67
|
-
await withSafety({ action: "click_menu_bar_extra", params: { ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().clickMenuBarExtra(params.app, { description: params.description, name: params.name, index: params.index }) });
|
|
68
|
-
|
|
72
|
+
const clickResult = await withSafety({ action: "click_menu_bar_extra", params: { ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().clickMenuBarExtra(params.app, { description: params.description, name: params.name, index: params.index }) });
|
|
73
|
+
const warnings = clickResult.verified
|
|
74
|
+
? []
|
|
75
|
+
: [clickResult.method === "coordinate"
|
|
76
|
+
? "AXPress produced no observable state change; coordinate fallback was used. Use find_element or screenshot to confirm the menu opened."
|
|
77
|
+
: "AXPress completed but the status item exposed no observable state to verify against. Use find_element or screenshot to confirm the menu opened."];
|
|
78
|
+
return actionResponse("click_menu_bar_extra", { clicked: true, app: params.app, method: clickResult.method, verified: clickResult.verified }, { app: params.app }, params.captureAfter, params.captureFormat, params.captureMaxWidth, warnings);
|
|
69
79
|
});
|
|
70
80
|
}
|
|
@@ -126,6 +126,17 @@ export interface WindowState {
|
|
|
126
126
|
focusedElement?: ElementInfo;
|
|
127
127
|
tree?: ElementInfo;
|
|
128
128
|
}
|
|
129
|
+
/**
|
|
130
|
+
* Outcome of an AX-driven click — surfaces whether AXPress was used and whether
|
|
131
|
+
* its effect was observable. `verified:false` means either the element exposed
|
|
132
|
+
* no observable state (inconclusive) or the click fell back to coordinates
|
|
133
|
+
* (silent AXPress swallow on Tauri/Electron). The tool layer surfaces this so
|
|
134
|
+
* the model can decide whether to re-observe via screenshot/get_window_state.
|
|
135
|
+
*/
|
|
136
|
+
export interface ClickResult {
|
|
137
|
+
method: "axpress" | "coordinate";
|
|
138
|
+
verified: boolean;
|
|
139
|
+
}
|
|
129
140
|
/**
|
|
130
141
|
* Structured text description of the screen — a fallback for environments where
|
|
131
142
|
* image content blocks are downgraded to URLs (so the model cannot see screenshots).
|
|
@@ -169,7 +180,7 @@ export interface Platform {
|
|
|
169
180
|
type(text: string, delay?: number): Promise<void>;
|
|
170
181
|
key(keys: string[]): Promise<void>;
|
|
171
182
|
findElement(options: FindElementOptions): Promise<FindElementResponse>;
|
|
172
|
-
clickElement(elementId: string, app?: string): Promise<
|
|
183
|
+
clickElement(elementId: string, app?: string): Promise<ClickResult>;
|
|
173
184
|
typeInElement(elementId: string, text: string, app?: string, clearFirst?: boolean): Promise<void>;
|
|
174
185
|
setElementValue?(elementId: string, value: string, app?: string): Promise<void>;
|
|
175
186
|
findMenuBarExtra?(app: string): Promise<unknown[]>;
|
|
@@ -177,7 +188,7 @@ export interface Platform {
|
|
|
177
188
|
description?: string;
|
|
178
189
|
name?: string;
|
|
179
190
|
index?: number;
|
|
180
|
-
}): Promise<
|
|
191
|
+
}): Promise<ClickResult>;
|
|
181
192
|
isScreenLocked?(): boolean;
|
|
182
193
|
saveFocus?(): Promise<void>;
|
|
183
194
|
restoreFocus?(): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Platform, ScreenRegion, ScreenSize, CursorPosition, WindowInfo, WindowState, OcrResult, FindElementOptions, FindElementResponse } from "./base.js";
|
|
1
|
+
import type { Platform, ScreenRegion, ScreenSize, CursorPosition, WindowInfo, WindowState, OcrResult, FindElementOptions, FindElementResponse, ClickResult } from "./base.js";
|
|
2
2
|
/**
|
|
3
3
|
* Linux platform adapter (AT-SPI2 + xdotool fallback)
|
|
4
4
|
* TODO: Implement with D-Bus AT-SPI2 bindings
|
|
@@ -17,7 +17,7 @@ export declare class LinuxPlatform implements Platform {
|
|
|
17
17
|
key(keys: string[]): Promise<void>;
|
|
18
18
|
ocr(_display?: number, _region?: ScreenRegion): Promise<OcrResult>;
|
|
19
19
|
findElement(_options: FindElementOptions): Promise<FindElementResponse>;
|
|
20
|
-
clickElement(_elementId: string, _app?: string): Promise<
|
|
20
|
+
clickElement(_elementId: string, _app?: string): Promise<ClickResult>;
|
|
21
21
|
typeInElement(_elementId: string, _text: string, _app?: string, _clearFirst?: boolean): Promise<void>;
|
|
22
22
|
readClipboard(): Promise<string>;
|
|
23
23
|
writeClipboard(text: string): Promise<void>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { MacOSPlatform } from "./base.js";
|
|
2
|
-
|
|
2
|
+
import type { ClickResult } from "../base.js";
|
|
3
|
+
export declare function clickElement(this: MacOSPlatform, elementId: string, app?: string): Promise<ClickResult>;
|
|
3
4
|
export declare function typeInElement(this: MacOSPlatform, elementId: string, text: string, app?: string, clearFirst?: boolean): Promise<void>;
|
|
4
5
|
export declare function setElementValue(this: MacOSPlatform, elementId: string, value: string, app?: string): Promise<void>;
|
|
5
6
|
export interface MenuBarExtraItem {
|
|
@@ -21,4 +22,4 @@ export interface MenuBarExtraSelector {
|
|
|
21
22
|
}
|
|
22
23
|
export declare function findMenuBarExtra(this: MacOSPlatform, app: string): Promise<MenuBarExtraItem[]>;
|
|
23
24
|
export declare function matchMenuBarExtra(items: MenuBarExtraItem[], selector: MenuBarExtraSelector): MenuBarExtraItem | undefined;
|
|
24
|
-
export declare function clickMenuBarExtra(this: MacOSPlatform, app: string, selector?: MenuBarExtraSelector): Promise<
|
|
25
|
+
export declare function clickMenuBarExtra(this: MacOSPlatform, app: string, selector?: MenuBarExtraSelector): Promise<ClickResult>;
|
|
@@ -2,6 +2,20 @@ import { execFileSync } from "node:child_process";
|
|
|
2
2
|
import { ElementNotFoundError } from "../../util/errors.js";
|
|
3
3
|
import { rethrowElementActionError } from "./helpers.js";
|
|
4
4
|
import { jxaElementActionHelpers } from "../jxa-helpers.js";
|
|
5
|
+
/**
|
|
6
|
+
* App-name hints for which AXPress is known to be silently swallowed
|
|
7
|
+
* (Tauri custom window decorations, some Electron controls). When the target
|
|
8
|
+
* app matches, clickElement/clickMenuBarExtra skip AXPress and go straight to
|
|
9
|
+
* a coordinate click. Conservative starting set — extend as more silent-swallow
|
|
10
|
+
* apps are confirmed.
|
|
11
|
+
*/
|
|
12
|
+
const AX_SILENT_APP_HINTS = ["tauri"];
|
|
13
|
+
function preferCoordinateClick(appName) {
|
|
14
|
+
if (!appName)
|
|
15
|
+
return false;
|
|
16
|
+
const n = appName.toLowerCase();
|
|
17
|
+
return AX_SILENT_APP_HINTS.some((h) => n.includes(h));
|
|
18
|
+
}
|
|
5
19
|
function prepareCache(elementId) {
|
|
6
20
|
this.evictExpiredCacheEntries();
|
|
7
21
|
const cached = this.elementCache.get(elementId);
|
|
@@ -10,22 +24,65 @@ function prepareCache(elementId) {
|
|
|
10
24
|
}
|
|
11
25
|
return this.elementCache.get(elementId) ?? null;
|
|
12
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Shared JXA snippet: read an observable state signature from an AX element.
|
|
29
|
+
* Used by clickElement/clickMenuBarExtra to verify whether AXPress actually
|
|
30
|
+
* changed anything (Tauri/Electron silently swallow AXPress without throwing).
|
|
31
|
+
* Returns a string concatenation of value/focused/selected (each defensively
|
|
32
|
+
* read; missing attributes contribute ''). An empty signature means the element
|
|
33
|
+
* exposes no observable state — verification is inconclusive.
|
|
34
|
+
*/
|
|
35
|
+
const JXA_STATE_SIGNATURE = `
|
|
36
|
+
function stateSignature(elem) {
|
|
37
|
+
var parts = [];
|
|
38
|
+
try { var v = elem.value(); parts.push('v=' + (v === undefined || v === null ? '' : String(v))); } catch(e) { parts.push('v='); }
|
|
39
|
+
try { parts.push('f=' + (elem.focused ? (elem.focused() ? 1 : 0) : '')); } catch(e) { parts.push('f='); }
|
|
40
|
+
try { parts.push('s=' + (elem.selected ? (elem.selected() ? 1 : 0) : '')); } catch(e) { parts.push('s='); }
|
|
41
|
+
return parts.join('|');
|
|
42
|
+
}
|
|
43
|
+
// Brief synchronous spin to let an async AXPress propagate state.
|
|
44
|
+
function spinMs(ms) { var t = Date.now(); while (Date.now() - t < ms) {} }
|
|
45
|
+
`;
|
|
46
|
+
/**
|
|
47
|
+
* Shared JXA snippet: perform a coordinate click at the element's bounds center.
|
|
48
|
+
* Returns the {x, y} used, or null if bounds could not be read.
|
|
49
|
+
*/
|
|
50
|
+
const JXA_COORDINATE_CLICK = `
|
|
51
|
+
function coordinateClick(elem) {
|
|
52
|
+
var pos = elem.position();
|
|
53
|
+
var sz = elem.size();
|
|
54
|
+
var cx = pos[0] + sz[0] / 2;
|
|
55
|
+
var cy = pos[1] + sz[1] / 2;
|
|
56
|
+
ObjC.import('CoreGraphics');
|
|
57
|
+
var src = $.CGEventSourceCreate($.kCGEventSourceStateHIDSystemState);
|
|
58
|
+
var pt = $.CGPointMake(cx, cy);
|
|
59
|
+
var down = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseDown, pt, $.kCGMouseButtonLeft);
|
|
60
|
+
$.CGEventPost($.kCGHIDEventTap, down);
|
|
61
|
+
var up = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseUp, pt, $.kCGMouseButtonLeft);
|
|
62
|
+
$.CGEventPost($.kCGHIDEventTap, up);
|
|
63
|
+
return {x: cx, y: cy};
|
|
64
|
+
}
|
|
65
|
+
`;
|
|
13
66
|
export async function clickElement(elementId, app) {
|
|
14
67
|
const elementIdLiteral = JSON.stringify(elementId);
|
|
15
68
|
const effectiveApp = app || this.activeTarget?.appName;
|
|
16
69
|
const appLiteral = JSON.stringify(effectiveApp || "");
|
|
17
70
|
const cachedDescriptor = prepareCache.call(this, elementId);
|
|
18
71
|
const cachedJson = JSON.stringify(cachedDescriptor);
|
|
72
|
+
const preferCoord = preferCoordinateClick(effectiveApp);
|
|
19
73
|
const jxaScript = `
|
|
20
74
|
var se = Application('System Events');
|
|
21
75
|
var _result = null;
|
|
22
76
|
${jxaElementActionHelpers()}
|
|
77
|
+
${JXA_STATE_SIGNATURE}
|
|
78
|
+
${JXA_COORDINATE_CLICK}
|
|
23
79
|
var elemPath = ${elementIdLiteral};
|
|
24
80
|
var appName = ${appLiteral};
|
|
25
81
|
// 容忍大小写/空格/连字符/下划线变体(cc-switch vs CC Switch vs cc_switch)
|
|
26
82
|
var _norm = function(s) { return String(s||'').toLowerCase().split(' ').join('').split('-').join('').split('_').join(''); };
|
|
27
83
|
var appNorm = _norm(appName);
|
|
28
84
|
var cached = ${cachedJson};
|
|
85
|
+
var preferCoord = ${preferCoord ? "true" : "false"};
|
|
29
86
|
|
|
30
87
|
var elem = resolveElementInApp(elemPath, appName) || resolveElementByFullPath(elemPath);
|
|
31
88
|
if (elem && !descriptorMatches(elem)) {
|
|
@@ -37,27 +94,51 @@ export async function clickElement(elementId, app) {
|
|
|
37
94
|
|
|
38
95
|
if (!elem) {
|
|
39
96
|
_result = {success: false, error: "Element not found: " + elemPath};
|
|
97
|
+
} else if (preferCoord) {
|
|
98
|
+
// 启发式命中(Tauri 等已知静默吞 AXPress 的应用):直接坐标点击
|
|
99
|
+
try {
|
|
100
|
+
coordinateClick(elem);
|
|
101
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
102
|
+
} catch(e) {
|
|
103
|
+
_result = {success: false, error: "Could not click element (coordinate): " + String(e.message || e)};
|
|
104
|
+
}
|
|
40
105
|
} else {
|
|
106
|
+
// AXPress + verify:采样前后状态签名,无变化则降级坐标
|
|
107
|
+
var sigBefore = stateSignature(elem);
|
|
108
|
+
var axpressThrew = false;
|
|
41
109
|
try {
|
|
42
110
|
elem.actions.AXPress.perform();
|
|
43
|
-
_result = {success: true};
|
|
44
111
|
} catch(e) {
|
|
112
|
+
axpressThrew = true;
|
|
113
|
+
}
|
|
114
|
+
if (axpressThrew) {
|
|
115
|
+
// AXPress 抛异常 → 坐标 fallback
|
|
45
116
|
try {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
var cx = pos[0] + sz[0] / 2;
|
|
49
|
-
var cy = pos[1] + sz[1] / 2;
|
|
50
|
-
ObjC.import('CoreGraphics');
|
|
51
|
-
var src = $.CGEventSourceCreate($.kCGEventSourceStateHIDSystemState);
|
|
52
|
-
var pt = $.CGPointMake(cx, cy);
|
|
53
|
-
var down = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseDown, pt, $.kCGMouseButtonLeft);
|
|
54
|
-
$.CGEventPost($.kCGHIDEventTap, down);
|
|
55
|
-
var up = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseUp, pt, $.kCGMouseButtonLeft);
|
|
56
|
-
$.CGEventPost($.kCGHIDEventTap, up);
|
|
57
|
-
_result = {success: true};
|
|
117
|
+
coordinateClick(elem);
|
|
118
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
58
119
|
} catch(e2) {
|
|
59
120
|
_result = {success: false, error: "Could not click element: " + String(e2.message || e2)};
|
|
60
121
|
}
|
|
122
|
+
} else {
|
|
123
|
+
// AXPress 未抛异常:spin 等待异步生效,再采样签名判断是否真的生效
|
|
124
|
+
spinMs(80);
|
|
125
|
+
var sigAfter = stateSignature(elem);
|
|
126
|
+
if (sigBefore !== '' && sigAfter !== sigBefore) {
|
|
127
|
+
// 状态变化 → AXPress 生效
|
|
128
|
+
_result = {success: true, method: "axpress", verified: true};
|
|
129
|
+
} else if (sigBefore === '' && sigAfter === '') {
|
|
130
|
+
// 无可观测状态(纯按钮无 value/focused/selected):无法 verify,
|
|
131
|
+
// 保守认为 AXPress 可能成功(不降级坐标,避免对正常按钮误伤)
|
|
132
|
+
_result = {success: true, method: "axpress", verified: false};
|
|
133
|
+
} else {
|
|
134
|
+
// 有可观测状态但无变化 → 静默吞,降级坐标
|
|
135
|
+
try {
|
|
136
|
+
coordinateClick(elem);
|
|
137
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
138
|
+
} catch(e3) {
|
|
139
|
+
_result = {success: false, error: "Could not click element (coordinate fallback): " + String(e3.message || e3)};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
61
142
|
}
|
|
62
143
|
}
|
|
63
144
|
JSON.stringify(_result);
|
|
@@ -73,9 +154,14 @@ export async function clickElement(elementId, app) {
|
|
|
73
154
|
? new Error(result.error)
|
|
74
155
|
: new ElementNotFoundError(elementId);
|
|
75
156
|
}
|
|
157
|
+
return {
|
|
158
|
+
method: result.method ?? "axpress",
|
|
159
|
+
verified: Boolean(result.verified),
|
|
160
|
+
};
|
|
76
161
|
}
|
|
77
162
|
catch (error) {
|
|
78
163
|
rethrowElementActionError(error, "click_element", elementId);
|
|
164
|
+
return { method: "axpress", verified: false }; // unreachable — rethrow throws
|
|
79
165
|
}
|
|
80
166
|
}
|
|
81
167
|
export async function typeInElement(elementId, text, app, clearFirst) {
|
|
@@ -372,7 +458,8 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
372
458
|
const host = target.host;
|
|
373
459
|
const tgtNameLiteral = JSON.stringify(target.name || "");
|
|
374
460
|
const tgtDescLiteral = JSON.stringify(target.description || "");
|
|
375
|
-
|
|
461
|
+
const preferCoord = preferCoordinateClick(app);
|
|
462
|
+
// AXPress + verify-then-fallback(与 clickElement 同模式,应对 Tauri 等静默吞)。
|
|
376
463
|
// host==="systemuiserver" 时在 SystemUIServer 进程上重定位(托盘 status item 由它托管),
|
|
377
464
|
// SystemUIServer.menuBarItems 顺序不稳定,用保存的 name/description 二次匹配定位具体 item。
|
|
378
465
|
// host==="self" 时按 app 进程的 menuBars()[mb].menuBarItems()[idx] 重定位(稳定)。
|
|
@@ -411,11 +498,14 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
411
498
|
var appName = ${appLiteral};
|
|
412
499
|
var tgtName = ${tgtNameLiteral};
|
|
413
500
|
var tgtDesc = ${tgtDescLiteral};
|
|
501
|
+
${JXA_STATE_SIGNATURE}
|
|
502
|
+
${JXA_COORDINATE_CLICK}
|
|
414
503
|
// 容忍大小写/空格/连字符/下划线变体(cc-switch vs CC Switch vs cc_switch)
|
|
415
504
|
var _norm = function(s) { return String(s||'').toLowerCase().split(' ').join('').split('-').join('').split('_').join(''); };
|
|
416
505
|
var appNorm = _norm(appName);
|
|
417
506
|
var tgtNameNorm = _norm(tgtName);
|
|
418
507
|
var tgtDescNorm = _norm(tgtDesc);
|
|
508
|
+
var preferCoord = ${preferCoord ? "true" : "false"};
|
|
419
509
|
var _matchApp = function(itemNorm) {
|
|
420
510
|
if (!itemNorm) return false;
|
|
421
511
|
return itemNorm === appNorm || itemNorm.indexOf(appNorm) !== -1 || appNorm.indexOf(itemNorm) !== -1;
|
|
@@ -430,21 +520,36 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
430
520
|
}
|
|
431
521
|
${resolveItemBlock}
|
|
432
522
|
if (!_result && item) {
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
var
|
|
440
|
-
var
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
523
|
+
if (preferCoord) {
|
|
524
|
+
// 启发式命中:直接坐标点击
|
|
525
|
+
coordinateClick(item);
|
|
526
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
527
|
+
} else {
|
|
528
|
+
// AXPress + verify
|
|
529
|
+
var sigBefore = stateSignature(item);
|
|
530
|
+
var axpressThrew = false;
|
|
531
|
+
try {
|
|
532
|
+
item.actions.AXPress.perform();
|
|
533
|
+
} catch(e) {
|
|
534
|
+
axpressThrew = true;
|
|
535
|
+
}
|
|
536
|
+
if (axpressThrew) {
|
|
537
|
+
coordinateClick(item);
|
|
538
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
539
|
+
} else {
|
|
540
|
+
spinMs(80);
|
|
541
|
+
var sigAfter = stateSignature(item);
|
|
542
|
+
if (sigBefore !== '' && sigAfter !== sigBefore) {
|
|
543
|
+
_result = {success: true, method: "axpress", verified: true};
|
|
544
|
+
} else if (sigBefore === '' && sigAfter === '') {
|
|
545
|
+
// 托盘 status item 通常无可观测状态(无 value/selected);AXPress 开菜单的效果
|
|
546
|
+
// 难以从 item 本身验证。保守认为成功(菜单是否打开由后续 find_element 判断)。
|
|
547
|
+
_result = {success: true, method: "axpress", verified: false};
|
|
548
|
+
} else {
|
|
549
|
+
coordinateClick(item);
|
|
550
|
+
_result = {success: true, method: "coordinate", verified: false};
|
|
551
|
+
}
|
|
552
|
+
}
|
|
448
553
|
}
|
|
449
554
|
}
|
|
450
555
|
} catch(e) {
|
|
@@ -458,10 +563,14 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
458
563
|
}
|
|
459
564
|
catch (error) {
|
|
460
565
|
rethrowElementActionError(error, "click_menu_bar_extra", app);
|
|
461
|
-
return; // unreachable
|
|
566
|
+
return { method: "axpress", verified: false }; // unreachable — rethrow throws
|
|
462
567
|
}
|
|
463
568
|
const result = JSON.parse(out);
|
|
464
569
|
if (!result.success) {
|
|
465
570
|
throw new Error(`click_menu_bar_extra failed in ${app}: ${result.error}`);
|
|
466
571
|
}
|
|
572
|
+
return {
|
|
573
|
+
method: result.method ?? "axpress",
|
|
574
|
+
verified: Boolean(result.verified),
|
|
575
|
+
};
|
|
467
576
|
}
|
|
@@ -1,3 +1,13 @@
|
|
|
1
1
|
import type { MacOSPlatform } from "./base.js";
|
|
2
|
+
/**
|
|
3
|
+
* @deprecated Focus management is intentionally disabled. CGEvent input
|
|
4
|
+
* synthesis (click/type/key) works at the HID layer and does not require the
|
|
5
|
+
* target app to be frontmost, so stealing/restoring focus is unnecessary and
|
|
6
|
+
* could disrupt the user. `saveFocus`/`restoreFocus` are retained for API
|
|
7
|
+
* compatibility but are NOT called from the action pipeline — see the
|
|
8
|
+
* `not.toHaveBeenCalled()` assertions in `tools-layer.test.ts` that lock this
|
|
9
|
+
* disabled state. Do not wire these into new code; prefer CGEvent-based input.
|
|
10
|
+
*/
|
|
2
11
|
export declare function saveFocus(this: MacOSPlatform): Promise<void>;
|
|
12
|
+
/** @deprecated See {@link saveFocus} — intentionally disabled, not called from the action pipeline. */
|
|
3
13
|
export declare function restoreFocus(this: MacOSPlatform): Promise<void>;
|
|
@@ -1,4 +1,13 @@
|
|
|
1
1
|
import { execFileSync } from "node:child_process";
|
|
2
|
+
/**
|
|
3
|
+
* @deprecated Focus management is intentionally disabled. CGEvent input
|
|
4
|
+
* synthesis (click/type/key) works at the HID layer and does not require the
|
|
5
|
+
* target app to be frontmost, so stealing/restoring focus is unnecessary and
|
|
6
|
+
* could disrupt the user. `saveFocus`/`restoreFocus` are retained for API
|
|
7
|
+
* compatibility but are NOT called from the action pipeline — see the
|
|
8
|
+
* `not.toHaveBeenCalled()` assertions in `tools-layer.test.ts` that lock this
|
|
9
|
+
* disabled state. Do not wire these into new code; prefer CGEvent-based input.
|
|
10
|
+
*/
|
|
2
11
|
export async function saveFocus() {
|
|
3
12
|
try {
|
|
4
13
|
const apps = await this.listApps();
|
|
@@ -16,6 +25,7 @@ export async function saveFocus() {
|
|
|
16
25
|
this.savedFocus = undefined;
|
|
17
26
|
}
|
|
18
27
|
}
|
|
28
|
+
/** @deprecated See {@link saveFocus} — intentionally disabled, not called from the action pipeline. */
|
|
19
29
|
export async function restoreFocus() {
|
|
20
30
|
if (!this.savedFocus)
|
|
21
31
|
return;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Platform, ScreenRegion, ScreenSize, CursorPosition, WindowInfo, WindowState, OcrResult, FindElementOptions, FindElementResponse } from "./base.js";
|
|
1
|
+
import type { Platform, ScreenRegion, ScreenSize, CursorPosition, WindowInfo, WindowState, OcrResult, FindElementOptions, FindElementResponse, ClickResult } from "./base.js";
|
|
2
2
|
export declare class WindowsPlatform implements Platform {
|
|
3
3
|
screenshot(_display?: number, _region?: ScreenRegion): Promise<Buffer>;
|
|
4
4
|
getScreenSize(_display?: number): ScreenSize;
|
|
@@ -13,7 +13,7 @@ export declare class WindowsPlatform implements Platform {
|
|
|
13
13
|
key(_keys: string[]): Promise<void>;
|
|
14
14
|
ocr(_display?: number, _region?: ScreenRegion): Promise<OcrResult>;
|
|
15
15
|
findElement(_options: FindElementOptions): Promise<FindElementResponse>;
|
|
16
|
-
clickElement(_elementId: string, _app?: string): Promise<
|
|
16
|
+
clickElement(_elementId: string, _app?: string): Promise<ClickResult>;
|
|
17
17
|
typeInElement(_elementId: string, _text: string, _app?: string, _clearFirst?: boolean): Promise<void>;
|
|
18
18
|
readClipboard(): Promise<string>;
|
|
19
19
|
writeClipboard(text: string): Promise<void>;
|