ucu-mcp 0.5.2 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/dist/src/mcp/tools/app-tools.js +7 -0
- package/dist/src/mcp/tools/helpers.js +32 -22
- package/dist/src/mcp/tools/input-tools.js +16 -10
- package/dist/src/mcp/tools/keyboard-tools.js +8 -4
- package/dist/src/mcp/tools/screen-tools.js +85 -53
- package/dist/src/platform/base.d.ts +17 -6
- package/dist/src/platform/macos/base.d.ts +12 -3
- package/dist/src/platform/macos/base.js +10 -2
- package/dist/src/platform/macos/element.d.ts +2 -0
- package/dist/src/platform/macos/element.js +67 -68
- package/dist/src/platform/macos/helpers.js +10 -5
- package/dist/src/platform/macos/input.d.ts +7 -6
- package/dist/src/platform/macos/input.js +31 -7
- package/dist/src/platform/macos/screen.js +10 -2
- package/dist/src/platform/macos/window.d.ts +2 -0
- package/dist/src/platform/macos/window.js +49 -26
- package/dist/src/safety/permissions.d.ts +2 -0
- package/dist/src/safety/permissions.js +34 -2
- package/dist/src/utils/input.d.ts +14 -8
- package/dist/src/utils/input.js +102 -43
- package/native/skylight/main.swift +402 -0
- package/native/skylight/skylight-helper +0 -0
- package/package.json +2 -2
- package/skills/ucu-mcp/SKILL.md +17 -0
|
@@ -2,6 +2,17 @@ import { execFileSync } from "node:child_process";
|
|
|
2
2
|
import { ElementNotFoundError } from "../../util/errors.js";
|
|
3
3
|
import { rethrowElementActionError } from "./helpers.js";
|
|
4
4
|
import { jxaElementActionHelpers } from "../jxa-helpers.js";
|
|
5
|
+
const DEFAULT_AXPRESS_SPIN_MS = 50;
|
|
6
|
+
const MAX_AXPRESS_SPIN_MS = 80;
|
|
7
|
+
function getAxPressSpinMs() {
|
|
8
|
+
const env = process.env.UCU_AXPRESS_SPIN_MS;
|
|
9
|
+
if (!env)
|
|
10
|
+
return DEFAULT_AXPRESS_SPIN_MS;
|
|
11
|
+
const parsed = parseInt(env, 10);
|
|
12
|
+
if (isNaN(parsed))
|
|
13
|
+
return DEFAULT_AXPRESS_SPIN_MS;
|
|
14
|
+
return Math.max(0, Math.min(parsed, MAX_AXPRESS_SPIN_MS));
|
|
15
|
+
}
|
|
5
16
|
/**
|
|
6
17
|
* App-name hints for which AXPress is known to be silently swallowed
|
|
7
18
|
* (Tauri custom window decorations, some Electron controls). When the target
|
|
@@ -44,23 +55,16 @@ const JXA_STATE_SIGNATURE = `
|
|
|
44
55
|
function spinMs(ms) { var t = Date.now(); while (Date.now() - t < ms) {} }
|
|
45
56
|
`;
|
|
46
57
|
/**
|
|
47
|
-
* Shared JXA snippet:
|
|
48
|
-
*
|
|
58
|
+
* Shared JXA snippet: compute the element's bounds center (for coordinate fallback).
|
|
59
|
+
* Does NOT post any event — the actual click is performed by the TS input layer
|
|
60
|
+
* (this.click) so it routes through per-process posting (skylight-helper) when a
|
|
61
|
+
* pid is available, instead of the JXA HID-tap that would move the global cursor.
|
|
49
62
|
*/
|
|
50
|
-
const
|
|
51
|
-
function
|
|
63
|
+
const JXA_BOUNDS_CENTER = `
|
|
64
|
+
function boundsCenter(elem) {
|
|
52
65
|
var pos = elem.position();
|
|
53
66
|
var sz = elem.size();
|
|
54
|
-
|
|
55
|
-
var cy = pos[1] + sz[1] / 2;
|
|
56
|
-
ObjC.import('CoreGraphics');
|
|
57
|
-
var src = $.CGEventSourceCreate($.kCGEventSourceStateHIDSystemState);
|
|
58
|
-
var pt = $.CGPointMake(cx, cy);
|
|
59
|
-
var down = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseDown, pt, $.kCGMouseButtonLeft);
|
|
60
|
-
$.CGEventPost($.kCGHIDEventTap, down);
|
|
61
|
-
var up = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseUp, pt, $.kCGMouseButtonLeft);
|
|
62
|
-
$.CGEventPost($.kCGHIDEventTap, up);
|
|
63
|
-
return {x: cx, y: cy};
|
|
67
|
+
return {x: pos[0] + sz[0] / 2, y: pos[1] + sz[1] / 2};
|
|
64
68
|
}
|
|
65
69
|
`;
|
|
66
70
|
export async function clickElement(elementId, app) {
|
|
@@ -75,7 +79,7 @@ export async function clickElement(elementId, app) {
|
|
|
75
79
|
var _result = null;
|
|
76
80
|
${jxaElementActionHelpers()}
|
|
77
81
|
${JXA_STATE_SIGNATURE}
|
|
78
|
-
${
|
|
82
|
+
${JXA_BOUNDS_CENTER}
|
|
79
83
|
var elemPath = ${elementIdLiteral};
|
|
80
84
|
var appName = ${appLiteral};
|
|
81
85
|
// 容忍大小写/空格/连字符/下划线变体(cc-switch vs CC Switch vs cc_switch)
|
|
@@ -95,15 +99,11 @@ export async function clickElement(elementId, app) {
|
|
|
95
99
|
if (!elem) {
|
|
96
100
|
_result = {success: false, error: "Element not found: " + elemPath};
|
|
97
101
|
} else if (preferCoord) {
|
|
98
|
-
// 启发式命中(Tauri 等已知静默吞 AXPress
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
_result = {success: true, method: "coordinate", verified: false};
|
|
102
|
-
} catch(e) {
|
|
103
|
-
_result = {success: false, error: "Could not click element (coordinate): " + String(e.message || e)};
|
|
104
|
-
}
|
|
102
|
+
// 启发式命中(Tauri 等已知静默吞 AXPress 的应用):请求坐标点击(TS 层执行 per-process)
|
|
103
|
+
var c = boundsCenter(elem);
|
|
104
|
+
_result = {success: true, needCoordinateClick: true, cx: c.x, cy: c.y};
|
|
105
105
|
} else {
|
|
106
|
-
// AXPress + verify
|
|
106
|
+
// AXPress + verify:采样前后状态签名,无变化则请求坐标点击
|
|
107
107
|
var sigBefore = stateSignature(elem);
|
|
108
108
|
var axpressThrew = false;
|
|
109
109
|
try {
|
|
@@ -112,57 +112,50 @@ export async function clickElement(elementId, app) {
|
|
|
112
112
|
axpressThrew = true;
|
|
113
113
|
}
|
|
114
114
|
if (axpressThrew) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
coordinateClick(elem);
|
|
118
|
-
_result = {success: true, method: "coordinate", verified: false};
|
|
119
|
-
} catch(e2) {
|
|
120
|
-
_result = {success: false, error: "Could not click element: " + String(e2.message || e2)};
|
|
121
|
-
}
|
|
115
|
+
var c1 = boundsCenter(elem);
|
|
116
|
+
_result = {success: true, needCoordinateClick: true, cx: c1.x, cy: c1.y};
|
|
122
117
|
} else {
|
|
123
|
-
|
|
124
|
-
spinMs(80);
|
|
118
|
+
spinMs(${getAxPressSpinMs()});
|
|
125
119
|
var sigAfter = stateSignature(elem);
|
|
126
120
|
if (sigBefore !== '' && sigAfter !== sigBefore) {
|
|
127
|
-
// 状态变化 → AXPress 生效
|
|
128
121
|
_result = {success: true, method: "axpress", verified: true};
|
|
129
122
|
} else if (sigBefore === '' && sigAfter === '') {
|
|
130
|
-
// 无可观测状态(纯按钮无 value/focused/selected):无法 verify,
|
|
131
|
-
// 保守认为 AXPress 可能成功(不降级坐标,避免对正常按钮误伤)
|
|
132
123
|
_result = {success: true, method: "axpress", verified: false};
|
|
133
124
|
} else {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
coordinateClick(elem);
|
|
137
|
-
_result = {success: true, method: "coordinate", verified: false};
|
|
138
|
-
} catch(e3) {
|
|
139
|
-
_result = {success: false, error: "Could not click element (coordinate fallback): " + String(e3.message || e3)};
|
|
140
|
-
}
|
|
125
|
+
var c2 = boundsCenter(elem);
|
|
126
|
+
_result = {success: true, needCoordinateClick: true, cx: c2.x, cy: c2.y};
|
|
141
127
|
}
|
|
142
128
|
}
|
|
143
129
|
}
|
|
144
130
|
JSON.stringify(_result);
|
|
145
131
|
`;
|
|
132
|
+
let result;
|
|
146
133
|
try {
|
|
147
134
|
const out = execFileSync("osascript", [
|
|
148
135
|
"-l", "JavaScript",
|
|
149
136
|
"-e", jxaScript,
|
|
150
137
|
], { encoding: "utf-8", timeout: 15000 }).trim();
|
|
151
|
-
|
|
152
|
-
if (!result.success) {
|
|
153
|
-
throw result.error
|
|
154
|
-
? new Error(result.error)
|
|
155
|
-
: new ElementNotFoundError(elementId);
|
|
156
|
-
}
|
|
157
|
-
return {
|
|
158
|
-
method: result.method ?? "axpress",
|
|
159
|
-
verified: Boolean(result.verified),
|
|
160
|
-
};
|
|
138
|
+
result = JSON.parse(out);
|
|
161
139
|
}
|
|
162
140
|
catch (error) {
|
|
163
141
|
rethrowElementActionError(error, "click_element", elementId);
|
|
164
142
|
return { method: "axpress", verified: false }; // unreachable — rethrow throws
|
|
165
143
|
}
|
|
144
|
+
if (!result.success) {
|
|
145
|
+
// Route through rethrowElementActionError so "element not found" / accessibility
|
|
146
|
+
// errors are converted to the proper UcuError subclasses (ElementNotFoundError etc.).
|
|
147
|
+
rethrowElementActionError(new Error(result.error || "click_element failed"), "click_element", elementId);
|
|
148
|
+
}
|
|
149
|
+
// JXA requested a coordinate click → perform it via the input layer (per-process
|
|
150
|
+
// when a pid is available, so the global cursor does not move).
|
|
151
|
+
if (result.needCoordinateClick && typeof result.cx === "number" && typeof result.cy === "number") {
|
|
152
|
+
await this.click(Math.round(result.cx), Math.round(result.cy), "left", false);
|
|
153
|
+
return { method: "coordinate", verified: false };
|
|
154
|
+
}
|
|
155
|
+
return {
|
|
156
|
+
method: result.method ?? "axpress",
|
|
157
|
+
verified: Boolean(result.verified),
|
|
158
|
+
};
|
|
166
159
|
}
|
|
167
160
|
export async function typeInElement(elementId, text, app, clearFirst) {
|
|
168
161
|
const textLiteral = JSON.stringify(text);
|
|
@@ -324,7 +317,7 @@ export async function findMenuBarExtra(app) {
|
|
|
324
317
|
return itemNorm === appNorm || itemNorm.indexOf(appNorm) !== -1 || appNorm.indexOf(itemNorm) !== -1;
|
|
325
318
|
};
|
|
326
319
|
|
|
327
|
-
var _readItem = function(item, mb, i, host) {
|
|
320
|
+
var _readItem = function(item, mb, i, host, hostPid) {
|
|
328
321
|
var desc = '', nm = '';
|
|
329
322
|
try { desc = item.description(); } catch(e) {}
|
|
330
323
|
try { nm = item.name(); } catch(e) {}
|
|
@@ -332,7 +325,7 @@ export async function findMenuBarExtra(app) {
|
|
|
332
325
|
try { pos = item.position(); } catch(e) {}
|
|
333
326
|
try { sz = item.size(); } catch(e) {}
|
|
334
327
|
if (sz[0] === 0 && sz[1] === 0) return null;
|
|
335
|
-
return {menuBar: mb, index: i, name: nm, description: desc, x: pos[0], y: pos[1], width: sz[0], height: sz[1], host: host};
|
|
328
|
+
return {menuBar: mb, index: i, name: nm, description: desc, x: pos[0], y: pos[1], width: sz[0], height: sz[1], host: host, pid: hostPid || 0};
|
|
336
329
|
};
|
|
337
330
|
|
|
338
331
|
try {
|
|
@@ -346,7 +339,9 @@ export async function findMenuBarExtra(app) {
|
|
|
346
339
|
|
|
347
340
|
var items = [];
|
|
348
341
|
|
|
349
|
-
// 阶段 1:app 自身 menuBarItems(host:"self")
|
|
342
|
+
// 阶段 1:app 自身 menuBarItems(host:"self",用 app 自身 pid)
|
|
343
|
+
var appPid = 0;
|
|
344
|
+
if (p) { try { appPid = p.unixId(); } catch(e) {} }
|
|
350
345
|
if (p) {
|
|
351
346
|
try {
|
|
352
347
|
var menuBars = p.menuBars();
|
|
@@ -354,7 +349,7 @@ export async function findMenuBarExtra(app) {
|
|
|
354
349
|
var mbItems;
|
|
355
350
|
try { mbItems = menuBars[mb].menuBarItems(); } catch(e) { continue; }
|
|
356
351
|
for (var i = 0; i < mbItems.length; i++) {
|
|
357
|
-
var rec = _readItem(mbItems[i], mb, i, "self");
|
|
352
|
+
var rec = _readItem(mbItems[i], mb, i, "self", appPid);
|
|
358
353
|
if (rec) items.push(rec);
|
|
359
354
|
}
|
|
360
355
|
}
|
|
@@ -372,6 +367,8 @@ export async function findMenuBarExtra(app) {
|
|
|
372
367
|
if (!hasNonApple) {
|
|
373
368
|
try {
|
|
374
369
|
var suiProcs = se.processes.byName("SystemUIServer");
|
|
370
|
+
var suiPid = 0;
|
|
371
|
+
if (suiProcs) { try { suiPid = suiProcs.unixId(); } catch(e) {} }
|
|
375
372
|
if (suiProcs) {
|
|
376
373
|
var suiBars = suiProcs.menuBars();
|
|
377
374
|
for (var smb = 0; smb < suiBars.length; smb++) {
|
|
@@ -384,7 +381,7 @@ export async function findMenuBarExtra(app) {
|
|
|
384
381
|
try { sNm = sItem.name(); } catch(e) {}
|
|
385
382
|
// 按 description/name 匹配目标 app(status item 的 description 通常是 app 名)
|
|
386
383
|
if (_matchApp(_norm(sDesc)) || _matchApp(_norm(sNm))) {
|
|
387
|
-
var sRec = _readItem(sItem, smb, si, "systemuiserver");
|
|
384
|
+
var sRec = _readItem(sItem, smb, si, "systemuiserver", suiPid);
|
|
388
385
|
if (sRec) {
|
|
389
386
|
// 保留匹配信号,供 click 二次定位
|
|
390
387
|
sRec.name = sNm; sRec.description = sDesc;
|
|
@@ -499,7 +496,7 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
499
496
|
var tgtName = ${tgtNameLiteral};
|
|
500
497
|
var tgtDesc = ${tgtDescLiteral};
|
|
501
498
|
${JXA_STATE_SIGNATURE}
|
|
502
|
-
${
|
|
499
|
+
${JXA_BOUNDS_CENTER}
|
|
503
500
|
// 容忍大小写/空格/连字符/下划线变体(cc-switch vs CC Switch vs cc_switch)
|
|
504
501
|
var _norm = function(s) { return String(s||'').toLowerCase().split(' ').join('').split('-').join('').split('_').join(''); };
|
|
505
502
|
var appNorm = _norm(appName);
|
|
@@ -521,9 +518,9 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
521
518
|
${resolveItemBlock}
|
|
522
519
|
if (!_result && item) {
|
|
523
520
|
if (preferCoord) {
|
|
524
|
-
//
|
|
525
|
-
|
|
526
|
-
_result = {success: true,
|
|
521
|
+
// 启发式命中:请求坐标点击(TS 层执行 per-process)
|
|
522
|
+
var c0 = boundsCenter(item);
|
|
523
|
+
_result = {success: true, needCoordinateClick: true, cx: c0.x, cy: c0.y};
|
|
527
524
|
} else {
|
|
528
525
|
// AXPress + verify
|
|
529
526
|
var sigBefore = stateSignature(item);
|
|
@@ -534,20 +531,18 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
534
531
|
axpressThrew = true;
|
|
535
532
|
}
|
|
536
533
|
if (axpressThrew) {
|
|
537
|
-
|
|
538
|
-
_result = {success: true,
|
|
534
|
+
var c1 = boundsCenter(item);
|
|
535
|
+
_result = {success: true, needCoordinateClick: true, cx: c1.x, cy: c1.y};
|
|
539
536
|
} else {
|
|
540
|
-
spinMs(
|
|
537
|
+
spinMs(${getAxPressSpinMs()});
|
|
541
538
|
var sigAfter = stateSignature(item);
|
|
542
539
|
if (sigBefore !== '' && sigAfter !== sigBefore) {
|
|
543
540
|
_result = {success: true, method: "axpress", verified: true};
|
|
544
541
|
} else if (sigBefore === '' && sigAfter === '') {
|
|
545
|
-
// 托盘 status item 通常无可观测状态(无 value/selected);AXPress 开菜单的效果
|
|
546
|
-
// 难以从 item 本身验证。保守认为成功(菜单是否打开由后续 find_element 判断)。
|
|
547
542
|
_result = {success: true, method: "axpress", verified: false};
|
|
548
543
|
} else {
|
|
549
|
-
|
|
550
|
-
_result = {success: true,
|
|
544
|
+
var c2 = boundsCenter(item);
|
|
545
|
+
_result = {success: true, needCoordinateClick: true, cx: c2.x, cy: c2.y};
|
|
551
546
|
}
|
|
552
547
|
}
|
|
553
548
|
}
|
|
@@ -569,6 +564,10 @@ export async function clickMenuBarExtra(app, selector = {}) {
|
|
|
569
564
|
if (!result.success) {
|
|
570
565
|
throw new Error(`click_menu_bar_extra failed in ${app}: ${result.error}`);
|
|
571
566
|
}
|
|
567
|
+
if (result.needCoordinateClick && typeof result.cx === "number" && typeof result.cy === "number") {
|
|
568
|
+
await this.click(Math.round(result.cx), Math.round(result.cy), "left", false);
|
|
569
|
+
return { method: "coordinate", verified: false };
|
|
570
|
+
}
|
|
572
571
|
return {
|
|
573
572
|
method: result.method ?? "axpress",
|
|
574
573
|
verified: Boolean(result.verified),
|
|
@@ -35,17 +35,22 @@ export function rethrowInputError(error, operation) {
|
|
|
35
35
|
throw new InputSynthesisError(`${operation} failed: ${errorMessage(error)}`);
|
|
36
36
|
}
|
|
37
37
|
export function normalizeAppName(name) {
|
|
38
|
-
|
|
38
|
+
// Drop all non-alphanumeric chars so app name variants match across
|
|
39
|
+
// formattings: "CC Switch" / "cc-switch" / "cc_switch" / "CC.Switch"
|
|
40
|
+
// all collapse to "ccswitch". This is what users (and LLMs) typically
|
|
41
|
+
// produce from casual memory, and it's the only way focus_app can
|
|
42
|
+
// resolve tray vs. windowed app identity reliably.
|
|
43
|
+
return name.trim().toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
39
44
|
}
|
|
40
45
|
export function appNameMatches(processName, requestedApp) {
|
|
41
46
|
const process = normalizeAppName(processName);
|
|
42
47
|
const requested = normalizeAppName(requestedApp);
|
|
43
48
|
if (!process || !requested)
|
|
44
49
|
return false;
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
50
|
+
// normalizeAppName strips all non-alphanumerics, so equality is the only
|
|
51
|
+
// meaningful comparison. (Prior startsWith/includes with spaces were dead code
|
|
52
|
+
// since normalized strings contain no spaces/hyphens.)
|
|
53
|
+
return process === requested || process.includes(requested) || requested.includes(process);
|
|
49
54
|
}
|
|
50
55
|
export function selectWindowForApp(windows, requestedApp) {
|
|
51
56
|
const requested = normalizeAppName(requestedApp);
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { MacOSPlatform } from "./base.js";
|
|
2
2
|
import type { CursorPosition } from "../base.js";
|
|
3
|
-
|
|
4
|
-
export declare function
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
3
|
+
import type { DispatchMethod } from "../base.js";
|
|
4
|
+
export declare function click(this: MacOSPlatform, x: number, y: number, button?: "left" | "right" | "middle", doubleClick?: boolean): Promise<DispatchMethod | void>;
|
|
5
|
+
export declare function move(this: MacOSPlatform, x: number, y: number): Promise<DispatchMethod | void>;
|
|
6
|
+
export declare function drag(this: MacOSPlatform, startX: number, startY: number, endX: number, endY: number, button?: "left" | "right" | "middle", duration?: number): Promise<DispatchMethod | void>;
|
|
7
|
+
export declare function scroll(this: MacOSPlatform, x: number, y: number, deltaX: number, deltaY: number): Promise<DispatchMethod | void>;
|
|
7
8
|
export declare function getCursorPosition(this: MacOSPlatform): CursorPosition;
|
|
8
|
-
export declare function type(this: MacOSPlatform, text: string, delay?: number): Promise<void>;
|
|
9
|
-
export declare function key(this: MacOSPlatform, keys: string[]): Promise<void>;
|
|
9
|
+
export declare function type(this: MacOSPlatform, text: string, delay?: number): Promise<DispatchMethod | void>;
|
|
10
|
+
export declare function key(this: MacOSPlatform, keys: string[]): Promise<DispatchMethod | void>;
|
|
@@ -2,13 +2,37 @@ import { execFileSync } from "node:child_process";
|
|
|
2
2
|
import { click as inputClick, doubleClick as inputDoubleClick, move as inputMove, drag as inputDrag, scroll as inputScroll, typeText, pressShortcut } from "../../utils/input.js";
|
|
3
3
|
import { PlatformError } from "../../util/errors.js";
|
|
4
4
|
import { rethrowInputError, errorMessage } from "./helpers.js";
|
|
5
|
+
/**
|
|
6
|
+
* Process names that must NEVER receive per-process input injection (security).
|
|
7
|
+
* Per-process posting bypasses the frontmost/focus requirement, so without this
|
|
8
|
+
* denylist an attacker could deliver keystrokes/clicks directly into a password
|
|
9
|
+
* manager or auth dialog. These fall back to HID-tap (which at least requires
|
|
10
|
+
* the app to be frontmost / visible to the user).
|
|
11
|
+
*/
|
|
12
|
+
const SENSITIVE_PROCESS_HINTS = [
|
|
13
|
+
"keychainaccess", "1password", "lastpass", "bitwarden", "securityagent",
|
|
14
|
+
"authd", "loginwindow", "applekeychain", "dashlane",
|
|
15
|
+
];
|
|
16
|
+
/** Resolve the per-process event target from the active focus_app target (pid + windowNumber). */
|
|
17
|
+
function targetOf() {
|
|
18
|
+
const t = this.activeTarget;
|
|
19
|
+
if (!t || !t.pid || t.pid <= 0)
|
|
20
|
+
return undefined;
|
|
21
|
+
// Security: never route per-process events into sensitive apps (password managers / auth).
|
|
22
|
+
const name = (t.appName || "").toLowerCase();
|
|
23
|
+
if (SENSITIVE_PROCESS_HINTS.some((h) => name.includes(h))) {
|
|
24
|
+
return undefined; // forces HID-tap fallback (requires frontmost)
|
|
25
|
+
}
|
|
26
|
+
return { pid: t.pid, windowNumber: t.windowNumber };
|
|
27
|
+
}
|
|
5
28
|
export async function click(x, y, button, doubleClick) {
|
|
6
29
|
try {
|
|
30
|
+
const target = targetOf.call(this);
|
|
7
31
|
if (doubleClick) {
|
|
8
|
-
await inputDoubleClick(x, y, button);
|
|
32
|
+
return await inputDoubleClick(x, y, button, process.platform, target);
|
|
9
33
|
}
|
|
10
34
|
else {
|
|
11
|
-
await inputClick(x, y, button);
|
|
35
|
+
return await inputClick(x, y, button, process.platform, target);
|
|
12
36
|
}
|
|
13
37
|
}
|
|
14
38
|
catch (error) {
|
|
@@ -17,7 +41,7 @@ export async function click(x, y, button, doubleClick) {
|
|
|
17
41
|
}
|
|
18
42
|
export async function move(x, y) {
|
|
19
43
|
try {
|
|
20
|
-
await inputMove(x, y);
|
|
44
|
+
return await inputMove(x, y, process.platform, targetOf.call(this));
|
|
21
45
|
}
|
|
22
46
|
catch (error) {
|
|
23
47
|
rethrowInputError(error, "move");
|
|
@@ -25,7 +49,7 @@ export async function move(x, y) {
|
|
|
25
49
|
}
|
|
26
50
|
export async function drag(startX, startY, endX, endY, button, duration) {
|
|
27
51
|
try {
|
|
28
|
-
await inputDrag(startX, startY, endX, endY, button, duration);
|
|
52
|
+
return await inputDrag(startX, startY, endX, endY, button, duration, process.platform, targetOf.call(this));
|
|
29
53
|
}
|
|
30
54
|
catch (error) {
|
|
31
55
|
rethrowInputError(error, "drag");
|
|
@@ -33,7 +57,7 @@ export async function drag(startX, startY, endX, endY, button, duration) {
|
|
|
33
57
|
}
|
|
34
58
|
export async function scroll(x, y, deltaX, deltaY) {
|
|
35
59
|
try {
|
|
36
|
-
await inputScroll(x, y, deltaX, deltaY);
|
|
60
|
+
return await inputScroll(x, y, deltaX, deltaY, process.platform, targetOf.call(this));
|
|
37
61
|
}
|
|
38
62
|
catch (error) {
|
|
39
63
|
rethrowInputError(error, "scroll");
|
|
@@ -55,8 +79,8 @@ export function getCursorPosition() {
|
|
|
55
79
|
}
|
|
56
80
|
}
|
|
57
81
|
export async function type(text, delay) {
|
|
58
|
-
await typeText(text, delay);
|
|
82
|
+
return await typeText(text, delay, process.platform, targetOf.call(this));
|
|
59
83
|
}
|
|
60
84
|
export async function key(keys) {
|
|
61
|
-
await pressShortcut(keys);
|
|
85
|
+
return await pressShortcut(keys, process.platform, targetOf.call(this));
|
|
62
86
|
}
|
|
@@ -26,8 +26,14 @@ export async function screenshotWindow(windowId, options) {
|
|
|
26
26
|
return this.screenshot(undefined, win.bounds, options);
|
|
27
27
|
}
|
|
28
28
|
export function getScreenSize(display) {
|
|
29
|
+
const idx = display ?? 0;
|
|
30
|
+
const now = Date.now();
|
|
31
|
+
if (this.screenSizeCache &&
|
|
32
|
+
this.screenSizeCache.display === idx &&
|
|
33
|
+
now - this.screenSizeCache.cachedAt <= this.screenSizeCacheTtlMs) {
|
|
34
|
+
return this.screenSizeCache.size;
|
|
35
|
+
}
|
|
29
36
|
try {
|
|
30
|
-
const idx = display ?? 0;
|
|
31
37
|
const out = execFileSync("osascript", [
|
|
32
38
|
"-l", "JavaScript",
|
|
33
39
|
"-e",
|
|
@@ -40,7 +46,9 @@ export function getScreenSize(display) {
|
|
|
40
46
|
var scaleFactor = screen.backingScaleFactor;
|
|
41
47
|
JSON.stringify({width:Math.round(frame.size.width),height:Math.round(frame.size.height),scaleFactor:scaleFactor})`,
|
|
42
48
|
], { encoding: "utf-8", timeout: 5000 }).trim();
|
|
43
|
-
|
|
49
|
+
const size = JSON.parse(out);
|
|
50
|
+
this.screenSizeCache = { display: idx, cachedAt: Date.now(), size };
|
|
51
|
+
return size;
|
|
44
52
|
}
|
|
45
53
|
catch (error) {
|
|
46
54
|
logger.warn("getScreenSize failed, using fallback", { error: errorMessage(error) });
|
|
@@ -2,5 +2,7 @@ import type { MacOSPlatform } from "./base.js";
|
|
|
2
2
|
import type { AppInfo, AppTarget, WindowInfo, BrowserContext } from "../base.js";
|
|
3
3
|
export declare function listApps(this: MacOSPlatform): Promise<AppInfo[]>;
|
|
4
4
|
export declare function focusApp(this: MacOSPlatform, app: string): Promise<AppTarget>;
|
|
5
|
+
/** Call the skylight-helper keepAlive command for a pid (best-effort, non-throwing). */
|
|
6
|
+
export declare function keepTargetAxAlive(this: MacOSPlatform, pid: number): Promise<void>;
|
|
5
7
|
export declare function getActiveBrowserContext(this: MacOSPlatform, app?: string): Promise<BrowserContext | undefined>;
|
|
6
8
|
export declare function listWindows(this: MacOSPlatform, _includeMinimized?: boolean): Promise<WindowInfo[]>;
|
|
@@ -58,14 +58,20 @@ export async function focusApp(app) {
|
|
|
58
58
|
try {
|
|
59
59
|
const extras = await this.findMenuBarExtra(app);
|
|
60
60
|
if (extras.length > 0) {
|
|
61
|
+
// Use the hosting process pid (SystemUIServer or app itself) so per-process
|
|
62
|
+
// event posting works for tray clicks (no global cursor move).
|
|
63
|
+
const trayPid = extras[0]?.pid || 0;
|
|
61
64
|
this.activeTarget = {
|
|
62
65
|
targetId: randomUUID(),
|
|
63
66
|
appName: app,
|
|
64
|
-
pid:
|
|
67
|
+
pid: trayPid,
|
|
65
68
|
windowId: "tray",
|
|
66
69
|
title: "",
|
|
67
70
|
capturedAt: new Date().toISOString(),
|
|
68
71
|
};
|
|
72
|
+
if (trayPid > 0) {
|
|
73
|
+
this.keepTargetAxAlive(trayPid).catch(() => { });
|
|
74
|
+
}
|
|
69
75
|
return this.activeTarget;
|
|
70
76
|
}
|
|
71
77
|
}
|
|
@@ -90,9 +96,29 @@ export async function focusApp(app) {
|
|
|
90
96
|
windowId: target.id,
|
|
91
97
|
title: target.title,
|
|
92
98
|
capturedAt: new Date().toISOString(),
|
|
99
|
+
windowNumber: target.windowNumber,
|
|
93
100
|
};
|
|
101
|
+
// AX keepalive: keep the target's accessibility tree live when occluded/background,
|
|
102
|
+
// so AXPress on Electron/Tauri controls doesn't silently no-op. Best-effort.
|
|
103
|
+
this.keepTargetAxAlive(target.pid).catch(() => { });
|
|
94
104
|
return this.activeTarget;
|
|
95
105
|
}
|
|
106
|
+
/** Call the skylight-helper keepAlive command for a pid (best-effort, non-throwing). */
|
|
107
|
+
export async function keepTargetAxAlive(pid) {
|
|
108
|
+
const helperPath = resolveNativeHelper.call(this, "skylight", "skylight-helper");
|
|
109
|
+
if (!helperPath)
|
|
110
|
+
return;
|
|
111
|
+
try {
|
|
112
|
+
execFileSync(helperPath, [], {
|
|
113
|
+
input: JSON.stringify({ command: "keepAlive", pid }),
|
|
114
|
+
encoding: "utf8",
|
|
115
|
+
timeout: 5000,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// keepAlive is best-effort; ignore failures (helper missing / SPI unavailable).
|
|
120
|
+
}
|
|
121
|
+
}
|
|
96
122
|
export async function getActiveBrowserContext(app) {
|
|
97
123
|
const appName = app || this.activeTarget?.appName;
|
|
98
124
|
if (!appName)
|
|
@@ -152,34 +178,30 @@ export async function listWindows(_includeMinimized) {
|
|
|
152
178
|
bounds: { ...window.bounds },
|
|
153
179
|
}));
|
|
154
180
|
}
|
|
155
|
-
if (this.
|
|
156
|
-
return this.
|
|
181
|
+
if (this.windowCachePromise) {
|
|
182
|
+
return this.windowCachePromise.then((windows) => windows.map((window) => ({ ...window, bounds: { ...window.bounds } })));
|
|
157
183
|
}
|
|
158
|
-
this.
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
184
|
+
this.windowCachePromise = (async () => {
|
|
185
|
+
try {
|
|
186
|
+
const nativeResult = listWindowsNative.call(this);
|
|
187
|
+
const windows = nativeResult !== null ? nativeResult : await listWindowsJxa.call(this);
|
|
188
|
+
this.windowCache = {
|
|
189
|
+
cachedAt: Date.now(),
|
|
190
|
+
windows: windows.map((window) => ({
|
|
191
|
+
...window,
|
|
192
|
+
bounds: { ...window.bounds },
|
|
193
|
+
})),
|
|
194
|
+
};
|
|
195
|
+
return windows;
|
|
164
196
|
}
|
|
165
|
-
|
|
166
|
-
|
|
197
|
+
catch {
|
|
198
|
+
return [];
|
|
167
199
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
})),
|
|
174
|
-
};
|
|
175
|
-
return windows;
|
|
176
|
-
}
|
|
177
|
-
catch {
|
|
178
|
-
return [];
|
|
179
|
-
}
|
|
180
|
-
finally {
|
|
181
|
-
this.windowCacheInFlight = false;
|
|
182
|
-
}
|
|
200
|
+
finally {
|
|
201
|
+
this.windowCachePromise = undefined;
|
|
202
|
+
}
|
|
203
|
+
})();
|
|
204
|
+
return this.windowCachePromise.then((windows) => windows.map((window) => ({ ...window, bounds: { ...window.bounds } })));
|
|
183
205
|
}
|
|
184
206
|
function listWindowsNative() {
|
|
185
207
|
try {
|
|
@@ -201,6 +223,7 @@ function listWindowsNative() {
|
|
|
201
223
|
bounds: w.bounds,
|
|
202
224
|
isMinimized: !w.isOnScreen,
|
|
203
225
|
isOnScreen: w.isOnScreen,
|
|
226
|
+
windowNumber: w.windowNumber,
|
|
204
227
|
}));
|
|
205
228
|
}
|
|
206
229
|
catch {
|
|
@@ -8,6 +8,8 @@ export interface PermissionDetail {
|
|
|
8
8
|
granted: boolean;
|
|
9
9
|
instructions: string;
|
|
10
10
|
}
|
|
11
|
+
/** @internal Test hook to reset the permission cache. */
|
|
12
|
+
export declare function __resetPermissionCache(): void;
|
|
11
13
|
/**
|
|
12
14
|
* Get the name of the terminal app that the user needs to authorize.
|
|
13
15
|
*/
|
|
@@ -1,6 +1,30 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { promisify } from "node:util";
|
|
3
3
|
const execFileAsync = promisify(execFile);
|
|
4
|
+
// Short-lived cache for individual permission checks. Permissions rarely change
|
|
5
|
+
// within a single tool sequence, but we keep TTL short so that a user granting
|
|
6
|
+
// permission mid-session is picked up quickly.
|
|
7
|
+
const PERMISSION_CACHE_TTL_MS = { granted: 5000, denied: 1000 };
|
|
8
|
+
const permissionCache = {};
|
|
9
|
+
/** @internal Test hook to reset the permission cache. */
|
|
10
|
+
export function __resetPermissionCache() {
|
|
11
|
+
permissionCache.accessibility = undefined;
|
|
12
|
+
permissionCache.screenRecording = undefined;
|
|
13
|
+
}
|
|
14
|
+
function getCachedPermission(type) {
|
|
15
|
+
const entry = permissionCache[type];
|
|
16
|
+
if (!entry)
|
|
17
|
+
return undefined;
|
|
18
|
+
const ttl = entry.granted ? PERMISSION_CACHE_TTL_MS.granted : PERMISSION_CACHE_TTL_MS.denied;
|
|
19
|
+
if (Date.now() - entry.cachedAt > ttl) {
|
|
20
|
+
permissionCache[type] = undefined;
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
return entry.granted;
|
|
24
|
+
}
|
|
25
|
+
function setCachedPermission(type, granted) {
|
|
26
|
+
permissionCache[type] = { granted, cachedAt: Date.now() };
|
|
27
|
+
}
|
|
4
28
|
/**
|
|
5
29
|
* Get the name of the terminal app that the user needs to authorize.
|
|
6
30
|
*/
|
|
@@ -133,7 +157,11 @@ export async function checkPermission(type) {
|
|
|
133
157
|
}
|
|
134
158
|
const appName = getTerminalAppName();
|
|
135
159
|
if (type === "accessibility") {
|
|
136
|
-
const
|
|
160
|
+
const cached = getCachedPermission("accessibility");
|
|
161
|
+
const granted = cached ?? await checkAccessibility();
|
|
162
|
+
if (cached === undefined) {
|
|
163
|
+
setCachedPermission("accessibility", granted);
|
|
164
|
+
}
|
|
137
165
|
if (!granted) {
|
|
138
166
|
// Trigger the macOS system prompt for Accessibility
|
|
139
167
|
await requestAccessibilityWithPrompt();
|
|
@@ -146,7 +174,11 @@ export async function checkPermission(type) {
|
|
|
146
174
|
}
|
|
147
175
|
return { granted: true };
|
|
148
176
|
}
|
|
149
|
-
const
|
|
177
|
+
const cached = getCachedPermission("screenRecording");
|
|
178
|
+
const granted = cached ?? await checkScreenRecording();
|
|
179
|
+
if (cached === undefined) {
|
|
180
|
+
setCachedPermission("screenRecording", granted);
|
|
181
|
+
}
|
|
150
182
|
if (!granted) {
|
|
151
183
|
// Open System Settings for Screen Recording
|
|
152
184
|
await openPermissionSettings("screenRecording");
|
|
@@ -9,14 +9,20 @@
|
|
|
9
9
|
* Windows: Uses SendInput (stub).
|
|
10
10
|
* Linux: Uses xdotool (stub).
|
|
11
11
|
*/
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
export
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
export declare function
|
|
19
|
-
export declare function
|
|
12
|
+
import type { DispatchMethod } from "../platform/base.js";
|
|
13
|
+
/** Per-process event target. When pid > 0, input is routed via skylight-helper (no cursor move). */
|
|
14
|
+
export interface InputTarget {
|
|
15
|
+
pid?: number;
|
|
16
|
+
windowNumber?: number;
|
|
17
|
+
}
|
|
18
|
+
export declare function click(x: number, y: number, button?: "left" | "right" | "middle", _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
19
|
+
export declare function doubleClick(x: number, y: number, button?: "left" | "right" | "middle", _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
20
|
+
export declare function move(x: number, y: number, _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
21
|
+
export declare function drag(fromX: number, fromY: number, toX: number, toY: number, button?: "left" | "right" | "middle", duration?: number, _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
22
|
+
export declare function scroll(x: number, y: number, deltaX: number, deltaY: number, _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
23
|
+
export declare function typeText(text: string, delay?: number, _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
24
|
+
export declare function pressKey(key: string, modifiers?: string[], _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
25
|
+
export declare function pressShortcut(keys: string[], _platform?: string, target?: InputTarget): Promise<DispatchMethod | void>;
|
|
20
26
|
export declare function getCursorPosition(_platform?: string): Promise<{
|
|
21
27
|
x: number;
|
|
22
28
|
y: number;
|