ucu-mcp 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/src/mcp/tools/app-tools.js +7 -0
- package/dist/src/mcp/tools/input-tools.js +16 -10
- package/dist/src/mcp/tools/keyboard-tools.js +8 -4
- package/dist/src/platform/base.d.ts +17 -6
- package/dist/src/platform/macos/base.d.ts +2 -1
- package/dist/src/platform/macos/base.js +2 -1
- package/dist/src/platform/macos/element.d.ts +2 -0
- package/dist/src/platform/macos/element.js +54 -66
- package/dist/src/platform/macos/input.d.ts +7 -6
- package/dist/src/platform/macos/input.js +31 -7
- package/dist/src/platform/macos/window.d.ts +2 -0
- package/dist/src/platform/macos/window.js +28 -1
- package/dist/src/utils/input.d.ts +14 -8
- package/dist/src/utils/input.js +102 -43
- package/native/skylight/main.swift +402 -0
- package/native/skylight/skylight-helper +0 -0
- package/package.json +2 -2
- package/skills/ucu-mcp/SKILL.md +17 -0
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
// skylight-helper: per-process event posting via private SkyLight SPIs.
|
|
2
|
+
//
|
|
3
|
+
// Posts mouse/keyboard events to a SPECIFIC process (by pid) without moving the
|
|
4
|
+
// global cursor or stealing foreground — matching Codex computer-use behavior.
|
|
5
|
+
// Uses SLEventPostToPid (private SkyLight) for mouse, CGEventPostToPid (public)
|
|
6
|
+
// for keyboard. Falls back to HID-tap posting (which moves the cursor) when the
|
|
7
|
+
// target is frontmost/canvas-app or when SPIs are unavailable.
|
|
8
|
+
//
|
|
9
|
+
// stdin/stdout JSON protocol mirrors cgevent-helper (line-oriented).
|
|
10
|
+
|
|
11
|
+
import CoreGraphics
|
|
12
|
+
import Foundation
|
|
13
|
+
import AppKit
|
|
14
|
+
|
|
15
|
+
// MARK: - SkyLight SPI loading
|
|
16
|
+
|
|
17
|
+
private let RTLD_DEFAULT = UnsafeMutableRawPointer(bitPattern: -2)!
|
|
18
|
+
|
|
19
|
+
// Force-load SkyLight so its symbols register in the global namespace.
|
|
20
|
+
_ = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
|
|
21
|
+
|
|
22
|
+
private func sym<T>(_ name: String, as _: T.Type) -> T? {
|
|
23
|
+
guard let p = dlsym(RTLD_DEFAULT, name) else { return nil }
|
|
24
|
+
return unsafeBitCast(p, to: T.self)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// void SLEventPostToPid(pid_t, CGEventRef)
|
|
28
|
+
private typealias SLEventPostToPidFn = @convention(c) (pid_t, CGEvent) -> Void
|
|
29
|
+
private let slPostToPid: SLEventPostToPidFn? = sym("SLEventPostToPid", as: SLEventPostToPidFn.self)
|
|
30
|
+
|
|
31
|
+
// void CGEventSetWindowLocation(CGEventRef, CGPoint) — private
|
|
32
|
+
private typealias SetWindowLocFn = @convention(c) (CGEvent, CGPoint) -> Void
|
|
33
|
+
private let setWindowLoc: SetWindowLocFn? = sym("CGEventSetWindowLocation", as: SetWindowLocFn.self)
|
|
34
|
+
|
|
35
|
+
// void SLEventSetIntegerValueField(CGEventRef, uint32_t, int64_t) — private
|
|
36
|
+
private typealias SetIntFieldFn = @convention(c) (CGEvent, UInt32, Int64) -> Void
|
|
37
|
+
private let setIntField: SetIntFieldFn? = sym("SLEventSetIntegerValueField", as: SetIntFieldFn.self)
|
|
38
|
+
|
|
39
|
+
// CGSConnectionID CGSMainConnectionID(void)
|
|
40
|
+
private typealias ConnIDFn = @convention(c) () -> UInt32
|
|
41
|
+
private let mainConnID: ConnIDFn? = sym("CGSMainConnectionID", as: ConnIDFn.self)
|
|
42
|
+
|
|
43
|
+
// CGError SLSGetWindowOwner(CGSConnectionID, CGWindowID, CGSConnectionID*)
|
|
44
|
+
private typealias GetWindowOwnerFn = @convention(c) (UInt32, UInt32, UnsafeMutablePointer<UInt32>) -> Int32
|
|
45
|
+
private let getWindowOwner: GetWindowOwnerFn? = sym("SLSGetWindowOwner", as: GetWindowOwnerFn.self)
|
|
46
|
+
|
|
47
|
+
// OSStatus SLSGetConnectionPSN(CGSConnectionID, ProcessSerialNumber*)
|
|
48
|
+
private typealias GetConnPSNFn = @convention(c) (UInt32, UnsafeMutableRawPointer) -> Int32
|
|
49
|
+
private let getConnPSN: GetConnPSNFn? = sym("SLSGetConnectionPSN", as: GetConnPSNFn.self)
|
|
50
|
+
|
|
51
|
+
// OSStatus _SLPSGetFrontProcess(ProcessSerialNumber*)
|
|
52
|
+
private typealias GetFrontPSNFn = @convention(c) (UnsafeMutableRawPointer) -> Int32
|
|
53
|
+
private let getFrontPSN: GetFrontPSNFn? = sym("_SLPSGetFrontProcess", as: GetFrontPSNFn.self)
|
|
54
|
+
|
|
55
|
+
// OSStatus SLPSPostEventRecordTo(ProcessSerialNumber*, uint8_t*)
|
|
56
|
+
private typealias PostEventRecFn = @convention(c) (UnsafeRawPointer, UnsafePointer<UInt8>) -> Int32
|
|
57
|
+
private let postEventRec: PostEventRecFn? = sym("SLPSPostEventRecordTo", as: PostEventRecFn.self)
|
|
58
|
+
|
|
59
|
+
/// True if all SPIs needed for per-process mouse posting are present.
|
|
60
|
+
private let skylightMouseAvailable: Bool = (slPostToPid != nil)
|
|
61
|
+
|
|
62
|
+
// MARK: - I/O helpers
|
|
63
|
+
|
|
64
|
+
struct Input: Decodable {
|
|
65
|
+
let command: String
|
|
66
|
+
let pid: Int32?
|
|
67
|
+
let windowNumber: Int?
|
|
68
|
+
let x: Double?
|
|
69
|
+
let y: Double?
|
|
70
|
+
let fromX: Double?
|
|
71
|
+
let fromY: Double?
|
|
72
|
+
let toX: Double?
|
|
73
|
+
let toY: Double?
|
|
74
|
+
let button: String?
|
|
75
|
+
let durationMs: Int?
|
|
76
|
+
let deltaX: Int?
|
|
77
|
+
let deltaY: Int?
|
|
78
|
+
let keyCode: Int?
|
|
79
|
+
let flags: Int64?
|
|
80
|
+
let keys: [KeyEntry]?
|
|
81
|
+
struct KeyEntry: Decodable { let code: Int; let shift: Bool? }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func out(_ json: String) {
|
|
85
|
+
FileHandle.standardOutput.write((json + "\n").data(using: .utf8)!)
|
|
86
|
+
fflush(stdout)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
func btn(_ s: String?) -> CGMouseButton {
|
|
90
|
+
switch s { case "right": return .right; case "middle": return .center; default: return .left }
|
|
91
|
+
}
|
|
92
|
+
func downT(_ b: CGMouseButton) -> CGEventType {
|
|
93
|
+
switch b { case .right: return .rightMouseDown; case .center: return .otherMouseDown; default: return .leftMouseDown }
|
|
94
|
+
}
|
|
95
|
+
func upT(_ b: CGMouseButton) -> CGEventType {
|
|
96
|
+
switch b { case .right: return .rightMouseUp; case .center: return .otherMouseUp; default: return .leftMouseUp }
|
|
97
|
+
}
|
|
98
|
+
func dragT(_ b: CGMouseButton) -> CGEventType {
|
|
99
|
+
switch b { case .right: return .rightMouseDragged; case .center: return .otherMouseDragged; default: return .leftMouseDragged }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// MARK: - Event construction & posting
|
|
103
|
+
|
|
104
|
+
/// Build a mouse CGEvent via the NSEvent bridge (raw CGEventCreateMouseEvent events are
|
|
105
|
+
/// filtered by Chromium's renderer IPC). Returns nil on failure.
|
|
106
|
+
func makeMouseEvent(_ type: CGEventType, at point: CGPoint, button: CGMouseButton,
|
|
107
|
+
clickCount: Int = 1, windowNumber: Int = 0) -> CGEvent? {
|
|
108
|
+
let nsType: NSEvent.EventType
|
|
109
|
+
switch type {
|
|
110
|
+
case .leftMouseDown: nsType = .leftMouseDown
|
|
111
|
+
case .leftMouseUp: nsType = .leftMouseUp
|
|
112
|
+
case .rightMouseDown: nsType = .rightMouseDown
|
|
113
|
+
case .rightMouseUp: nsType = .rightMouseUp
|
|
114
|
+
case .otherMouseDown: nsType = .otherMouseDown
|
|
115
|
+
case .otherMouseUp: nsType = .otherMouseUp
|
|
116
|
+
case .leftMouseDragged: nsType = .leftMouseDragged
|
|
117
|
+
case .rightMouseDragged: nsType = .rightMouseDragged
|
|
118
|
+
case .otherMouseDragged: nsType = .otherMouseDragged
|
|
119
|
+
case .mouseMoved: nsType = .mouseMoved
|
|
120
|
+
default: return nil
|
|
121
|
+
}
|
|
122
|
+
guard let ns = NSEvent.mouseEvent(
|
|
123
|
+
with: nsType, location: point, modifierFlags: [],
|
|
124
|
+
timestamp: 0, windowNumber: windowNumber, context: nil,
|
|
125
|
+
eventNumber: 0, clickCount: clickCount, pressure: (type == .mouseMoved) ? 0 : 1.0
|
|
126
|
+
), let cg = ns.cgEvent else { return nil }
|
|
127
|
+
return cg
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/// Stamp a mouse event with the fields Chromium's synthetic-event filter requires,
|
|
131
|
+
/// then post it to the target pid via SLEventPostToPid (no global cursor move).
|
|
132
|
+
func postPerPid(_ event: CGEvent, pid: Int32, at point: CGPoint, windowNumber: Int, button: CGMouseButton = .left) {
|
|
133
|
+
event.location = point
|
|
134
|
+
// Stamp the correct button number (0=left, 1=right, 2=center) — NOT hardcoded 0.
|
|
135
|
+
event.setIntegerValueField(.mouseEventButtonNumber, value: Int64(button.rawValue))
|
|
136
|
+
event.setIntegerValueField(.mouseEventSubtype, value: 3)
|
|
137
|
+
if windowNumber > 0 {
|
|
138
|
+
event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(windowNumber))
|
|
139
|
+
event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(windowNumber))
|
|
140
|
+
}
|
|
141
|
+
// Private SPIs: window-local point + Chromium pid latch (field 40).
|
|
142
|
+
setWindowLoc?(event, point)
|
|
143
|
+
setIntField?(event, 40, Int64(pid))
|
|
144
|
+
event.timestamp = clock_gettime_nsec_np(CLOCK_UPTIME_RAW)
|
|
145
|
+
slPostToPid?(pid, event)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/// HID-tap fallback (moves the cursor). Used for frontmost/canvas apps where
|
|
149
|
+
/// per-pid posting is filtered.
|
|
150
|
+
func postHidTap(_ event: CGEvent) {
|
|
151
|
+
event.post(tap: CGEventTapLocation.cghidEventTap)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/// True if the target pid is the frontmost app (canvas/GPU apps filter per-pid
|
|
155
|
+
/// routes — must use HID-tap there).
|
|
156
|
+
func isFrontmost(_ pid: Int32) -> Bool {
|
|
157
|
+
return NSRunningApplication(processIdentifier: pid)?.isActive == true
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// MARK: - Focus-without-raise (SLPSPostEventRecordTo)
|
|
161
|
+
|
|
162
|
+
/// Flip the target's AppKit-active state without raising its window (no Space switch).
|
|
163
|
+
func focusWithoutRaise(pid: Int32, windowID: Int) {
|
|
164
|
+
guard let postRec = postEventRec, let frontFn = getFrontPSN else { return }
|
|
165
|
+
// Resolve target PSN via window owner.
|
|
166
|
+
guard let connFn = mainConnID, let ownerFn = getWindowOwner, let psnFn = getConnPSN,
|
|
167
|
+
windowID > 0 else { return }
|
|
168
|
+
var prevPSN = [UInt32](repeating: 0, count: 2)
|
|
169
|
+
var targetPSN = [UInt32](repeating: 0, count: 2)
|
|
170
|
+
let prevOk = prevPSN.withUnsafeMutableBytes { raw in frontFn(raw.baseAddress!) != 0 }
|
|
171
|
+
if prevOk {
|
|
172
|
+
let cid = connFn()
|
|
173
|
+
var ownerCid: UInt32 = 0
|
|
174
|
+
guard ownerFn(cid, UInt32(windowID), &ownerCid) == 0 else { return }
|
|
175
|
+
guard psnFn(ownerCid, &targetPSN) == 0 else { return }
|
|
176
|
+
} else {
|
|
177
|
+
return
|
|
178
|
+
}
|
|
179
|
+
var buf = [UInt8](repeating: 0, count: 0xF8) // 248 bytes
|
|
180
|
+
buf[0x04] = 0xF8
|
|
181
|
+
buf[0x08] = 0x0D
|
|
182
|
+
let wid = UInt32(windowID)
|
|
183
|
+
buf[0x3C] = UInt8(wid & 0xFF); buf[0x3D] = UInt8((wid >> 8) & 0xFF)
|
|
184
|
+
buf[0x3E] = UInt8((wid >> 16) & 0xFF); buf[0x3F] = UInt8((wid >> 24) & 0xFF)
|
|
185
|
+
// Defocus previous front.
|
|
186
|
+
buf[0x8A] = 0x02
|
|
187
|
+
_ = prevPSN.withUnsafeBytes { psnRaw in buf.withUnsafeBufferPointer { bp in postRec(psnRaw.baseAddress!, bp.baseAddress!) } }
|
|
188
|
+
// Focus target.
|
|
189
|
+
buf[0x8A] = 0x01
|
|
190
|
+
_ = targetPSN.withUnsafeBytes { psnRaw in buf.withUnsafeBufferPointer { bp in postRec(psnRaw.baseAddress!, bp.baseAddress!) } }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// MARK: - AX keepalive (root-cause fix for Electron/Tauri AXPress silent failure)
|
|
194
|
+
|
|
195
|
+
/// Keep the target app's AX tree alive when occluded/background.
|
|
196
|
+
///
|
|
197
|
+
/// Writes the AXManualAccessibility attribute (persists for the app's lifetime,
|
|
198
|
+
/// independent of this helper process). AXEnhancedUserInterface is NOT set
|
|
199
|
+
/// unconditionally — it's known to slow/break some apps (Xcode, Slack, Office);
|
|
200
|
+
/// gate it behind the UCU_AX_ENHANCED env var if needed.
|
|
201
|
+
///
|
|
202
|
+
/// The observer-based remote-aware keepalive (_AXObserverAddNotificationAndCheckRemote)
|
|
203
|
+
/// requires a long-lived process to hold the subscription; this helper is one-shot
|
|
204
|
+
/// (readLine → exit), so the observer would be dropped on exit. We therefore rely on
|
|
205
|
+
/// the persistent attribute writes, which cover non-Chromium occluded apps. Full
|
|
206
|
+
/// Chromium keepalive would require a daemon helper (future work).
|
|
207
|
+
func doKeepAlive(_ p: Input) -> String {
|
|
208
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
209
|
+
let app = AXUIElementCreateApplication(pid)
|
|
210
|
+
// AXManualAccessibility: safe, persistent, tells the app to expose its AX tree.
|
|
211
|
+
let manualStatus = AXUIElementSetAttributeValue(app, "AXManualAccessibility" as CFString, kCFBooleanTrue!)
|
|
212
|
+
// AXEnhancedUserInterface: opt-in only (breaks some apps when unconditional).
|
|
213
|
+
var enhancedStatus: AXError = .cannotComplete
|
|
214
|
+
if ProcessInfo.processInfo.environment["UCU_AX_ENHANCED"] != nil {
|
|
215
|
+
enhancedStatus = AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue!)
|
|
216
|
+
}
|
|
217
|
+
let ok = (manualStatus == .success)
|
|
218
|
+
return "{\"ok\":\(ok),\"method\":\"ax-keepalive\",\"manual\":\(manualStatus == .success),\"enhanced\":\(enhancedStatus == .success)}"
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// MARK: - Commands
|
|
222
|
+
|
|
223
|
+
func doClick(_ p: Input) -> String {
|
|
224
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
225
|
+
let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0)
|
|
226
|
+
let b = btn(p.button)
|
|
227
|
+
let winNum = p.windowNumber ?? 0
|
|
228
|
+
// Canvas/frontmost apps filter per-pid routes → HID-tap fallback.
|
|
229
|
+
if !skylightMouseAvailable || isFrontmost(pid) {
|
|
230
|
+
guard let dn = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
|
|
231
|
+
let up = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b)
|
|
232
|
+
else { return "{\"error\":\"fail\"}" }
|
|
233
|
+
postHidTap(dn); postHidTap(up)
|
|
234
|
+
return "{\"ok\":true,\"method\":\"hid-tap\"}"
|
|
235
|
+
}
|
|
236
|
+
focusWithoutRaise(pid: pid, windowID: winNum)
|
|
237
|
+
// Leading mouseMoved at target (cursor-state primer).
|
|
238
|
+
if let mv = makeMouseEvent(.mouseMoved, at: loc, button: b, windowNumber: winNum) {
|
|
239
|
+
postPerPid(mv, pid: pid, at: loc, windowNumber: winNum, button: b); usleep(15_000)
|
|
240
|
+
}
|
|
241
|
+
// Off-screen primer click @ (-1,-1) — satisfies Chromium user-activation gate.
|
|
242
|
+
let off = CGPoint(x: -1, y: -1)
|
|
243
|
+
if let pd = makeMouseEvent(.leftMouseDown, at: off, button: .left, windowNumber: winNum),
|
|
244
|
+
let pu = makeMouseEvent(.leftMouseUp, at: off, button: .left, windowNumber: winNum) {
|
|
245
|
+
postPerPid(pd, pid: pid, at: off, windowNumber: winNum); usleep(1_000)
|
|
246
|
+
postPerPid(pu, pid: pid, at: off, windowNumber: winNum); usleep(100_000)
|
|
247
|
+
}
|
|
248
|
+
// Target click.
|
|
249
|
+
guard let dn = makeMouseEvent(downT(b), at: loc, button: b, windowNumber: winNum),
|
|
250
|
+
let up = makeMouseEvent(upT(b), at: loc, button: b, windowNumber: winNum)
|
|
251
|
+
else { return "{\"error\":\"fail\"}" }
|
|
252
|
+
postPerPid(dn, pid: pid, at: loc, windowNumber: winNum, button: b); usleep(1_000)
|
|
253
|
+
postPerPid(up, pid: pid, at: loc, windowNumber: winNum, button: b)
|
|
254
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
func doDoubleClick(_ p: Input) -> String {
|
|
258
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
259
|
+
let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0)
|
|
260
|
+
let b = btn(p.button)
|
|
261
|
+
let winNum = p.windowNumber ?? 0
|
|
262
|
+
if !skylightMouseAvailable || isFrontmost(pid) {
|
|
263
|
+
return hidTapDouble(loc: loc, b: b)
|
|
264
|
+
}
|
|
265
|
+
focusWithoutRaise(pid: pid, windowID: winNum)
|
|
266
|
+
for state in [1, 2] {
|
|
267
|
+
guard let dn = makeMouseEvent(downT(b), at: loc, button: b, clickCount: state, windowNumber: winNum),
|
|
268
|
+
let up = makeMouseEvent(upT(b), at: loc, button: b, clickCount: state, windowNumber: winNum)
|
|
269
|
+
else { return "{\"error\":\"fail\"}" }
|
|
270
|
+
postPerPid(dn, pid: pid, at: loc, windowNumber: winNum, button: b); usleep(1_000)
|
|
271
|
+
postPerPid(up, pid: pid, at: loc, windowNumber: winNum, button: b)
|
|
272
|
+
if state == 1 { usleep(80_000) }
|
|
273
|
+
}
|
|
274
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
func hidTapDouble(loc: CGPoint, b: CGMouseButton) -> String {
|
|
278
|
+
guard let d1 = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
|
|
279
|
+
let u1 = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b),
|
|
280
|
+
let d2 = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
|
|
281
|
+
let u2 = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b)
|
|
282
|
+
else { return "{\"error\":\"fail\"}" }
|
|
283
|
+
d1.setIntegerValueField(.mouseEventClickState, value: 1); u1.setIntegerValueField(.mouseEventClickState, value: 1)
|
|
284
|
+
d2.setIntegerValueField(.mouseEventClickState, value: 2); u2.setIntegerValueField(.mouseEventClickState, value: 2)
|
|
285
|
+
postHidTap(d1); postHidTap(u1); postHidTap(d2); postHidTap(u2)
|
|
286
|
+
return "{\"ok\":true,\"method\":\"hid-tap\"}"
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
func doMove(_ p: Input) -> String {
|
|
290
|
+
guard let pid = p.pid, pid > 0, skylightMouseAvailable, !isFrontmost(pid) else {
|
|
291
|
+
let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0)
|
|
292
|
+
guard let ev = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: loc, mouseButton: .left)
|
|
293
|
+
else { return "{\"error\":\"fail\"}" }
|
|
294
|
+
postHidTap(ev); return "{\"ok\":true,\"method\":\"hid-tap\"}"
|
|
295
|
+
}
|
|
296
|
+
let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0)
|
|
297
|
+
let winNum = p.windowNumber ?? 0
|
|
298
|
+
if let ev = makeMouseEvent(.mouseMoved, at: loc, button: .left, windowNumber: winNum) {
|
|
299
|
+
postPerPid(ev, pid: pid, at: loc, windowNumber: winNum, button: .left)
|
|
300
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
301
|
+
}
|
|
302
|
+
return "{\"error\":\"fail\"}"
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
func doDrag(_ p: Input) -> String {
|
|
306
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
307
|
+
let from = CGPoint(x: p.fromX ?? 0, y: p.fromY ?? 0)
|
|
308
|
+
let to = CGPoint(x: p.toX ?? 0, y: p.toY ?? 0)
|
|
309
|
+
let ms = p.durationMs ?? 300; let b = btn(p.button)
|
|
310
|
+
let winNum = p.windowNumber ?? 0
|
|
311
|
+
let steps = max(2, min(60, Int(ceil(Double(ms) / 16.0))))
|
|
312
|
+
let delay = max(0, (ms * 1000) / steps)
|
|
313
|
+
if !skylightMouseAvailable || isFrontmost(pid) {
|
|
314
|
+
guard let dn = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: from, mouseButton: b)
|
|
315
|
+
else { return "{\"error\":\"fail\"}" }
|
|
316
|
+
postHidTap(dn)
|
|
317
|
+
for n in 1...steps {
|
|
318
|
+
let t = Double(n) / Double(steps)
|
|
319
|
+
let pt = CGPoint(x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
|
|
320
|
+
if let ev = CGEvent(mouseEventSource: nil, mouseType: dragT(b), mouseCursorPosition: pt, mouseButton: b) { postHidTap(ev) }
|
|
321
|
+
if delay > 0 && n < steps { usleep(UInt32(delay)) }
|
|
322
|
+
}
|
|
323
|
+
if let up = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: to, mouseButton: b) { postHidTap(up) }
|
|
324
|
+
return "{\"ok\":true,\"method\":\"hid-tap\"}"
|
|
325
|
+
}
|
|
326
|
+
focusWithoutRaise(pid: pid, windowID: winNum)
|
|
327
|
+
guard let dn = makeMouseEvent(downT(b), at: from, button: b, windowNumber: winNum)
|
|
328
|
+
else { return "{\"error\":\"fail\"}" }
|
|
329
|
+
postPerPid(dn, pid: pid, at: from, windowNumber: winNum, button: b)
|
|
330
|
+
for n in 1...steps {
|
|
331
|
+
let t = Double(n) / Double(steps)
|
|
332
|
+
let pt = CGPoint(x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
|
|
333
|
+
if let ev = makeMouseEvent(dragT(b), at: pt, button: b, windowNumber: winNum) {
|
|
334
|
+
postPerPid(ev, pid: pid, at: pt, windowNumber: winNum, button: b)
|
|
335
|
+
}
|
|
336
|
+
if delay > 0 && n < steps { usleep(UInt32(delay)) }
|
|
337
|
+
}
|
|
338
|
+
if let up = makeMouseEvent(upT(b), at: to, button: b, windowNumber: winNum) {
|
|
339
|
+
postPerPid(up, pid: pid, at: to, windowNumber: winNum, button: b)
|
|
340
|
+
}
|
|
341
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
func doScroll(_ p: Input) -> String {
|
|
345
|
+
// Scroll events use CGEventCreateScrollWheelEvent; per-pid scroll posting is
|
|
346
|
+
// unreliable across app types, so we route via HID-tap but still report method.
|
|
347
|
+
let dy = Int32(-(p.deltaY ?? 0)); let dx = Int32(p.deltaX ?? 0)
|
|
348
|
+
guard let ev = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: dy, wheel2: dx, wheel3: 0)
|
|
349
|
+
else { return "{\"error\":\"fail\"}" }
|
|
350
|
+
// Best-effort per-pid if we have a pid; scroll doesn't move the cursor noticeably anyway.
|
|
351
|
+
if let pid = p.pid, pid > 0, slPostToPid != nil, !isFrontmost(pid) {
|
|
352
|
+
slPostToPid?(pid, ev)
|
|
353
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
354
|
+
}
|
|
355
|
+
postHidTap(ev)
|
|
356
|
+
return "{\"ok\":true,\"method\":\"hid-tap\"}"
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
func doPressKey(_ p: Input) -> String {
|
|
360
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
361
|
+
let code = UInt16(p.keyCode ?? 0); let flags = CGEventFlags(rawValue: UInt64(p.flags ?? 0))
|
|
362
|
+
guard let dn = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
|
|
363
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
|
|
364
|
+
else { return "{\"error\":\"fail\"}" }
|
|
365
|
+
dn.flags = flags; up.flags = flags
|
|
366
|
+
// Keyboard: public CGEventPostToPid is sufficient (no SkyLight needed).
|
|
367
|
+
dn.postToPid(pid); up.postToPid(pid)
|
|
368
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
func doTypeBatch(_ p: Input) -> String {
|
|
372
|
+
guard let pid = p.pid, pid > 0 else { return "{\"error\":\"no pid\"}" }
|
|
373
|
+
guard let keys = p.keys else { return "{\"error\":\"missing keys\"}" }
|
|
374
|
+
let SHIFT = CGEventFlags(rawValue: 0x00020000)
|
|
375
|
+
for entry in keys {
|
|
376
|
+
let code = UInt16(entry.code); let flags: CGEventFlags = (entry.shift ?? false) ? SHIFT : []
|
|
377
|
+
guard let dn = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
|
|
378
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false) else { continue }
|
|
379
|
+
dn.flags = flags; up.flags = flags
|
|
380
|
+
dn.postToPid(pid); up.postToPid(pid)
|
|
381
|
+
}
|
|
382
|
+
return "{\"ok\":true,\"method\":\"per-pid\"}"
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// MARK: - Dispatch
|
|
386
|
+
|
|
387
|
+
guard let line = readLine(), let data = line.data(using: .utf8),
|
|
388
|
+
let input = try? JSONDecoder().decode(Input.self, from: data)
|
|
389
|
+
else { out("{\"error\":\"invalid JSON\"}"); exit(1) }
|
|
390
|
+
|
|
391
|
+
switch input.command {
|
|
392
|
+
case "click": out(doClick(input))
|
|
393
|
+
case "doubleClick": out(doDoubleClick(input))
|
|
394
|
+
case "move": out(doMove(input))
|
|
395
|
+
case "drag": out(doDrag(input))
|
|
396
|
+
case "scroll": out(doScroll(input))
|
|
397
|
+
case "pressKey": out(doPressKey(input))
|
|
398
|
+
case "typeBatch": out(doTypeBatch(input))
|
|
399
|
+
case "keepAlive": out(doKeepAlive(input))
|
|
400
|
+
case "ping": out(skylightMouseAvailable ? "{\"ok\":true,\"skylight\":true}" : "{\"ok\":true,\"skylight\":false}")
|
|
401
|
+
default: out("{\"error\":\"unknown\"}")
|
|
402
|
+
}
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ucu-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "MCP server for Universal Computer Use — desktop automation for AI agents via Model Context Protocol",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"test:macos-gui": "UCU_MACOS_GUI_SMOKE=1 vitest run tests/integration/macos-gui-smoke.test.ts",
|
|
29
29
|
"test:client-cli": "UCU_CLIENT_CLI_SMOKE=1 vitest run tests/integration/client-cli-smoke.test.ts",
|
|
30
30
|
"prepublishOnly": "npx vitest run tests/unit/ && npm run build",
|
|
31
|
-
"build:native": "cd native/cgevent && swiftc -O -o cgevent-helper main.swift -framework CoreGraphics -framework Foundation && cd ../ocr && swiftc -O -o ocr-helper main.swift -framework Vision -framework AppKit && cd ../windowlist && swiftc -O -o windowlist-helper main.swift -framework CoreGraphics -framework Foundation"
|
|
31
|
+
"build:native": "cd native/cgevent && swiftc -O -o cgevent-helper main.swift -framework CoreGraphics -framework Foundation && cd ../ocr && swiftc -O -o ocr-helper main.swift -framework Vision -framework AppKit && cd ../windowlist && swiftc -O -o windowlist-helper main.swift -framework CoreGraphics -framework Foundation && cd ../skylight && swiftc -O -o skylight-helper main.swift -framework CoreGraphics -framework Foundation -framework AppKit -framework Cocoa"
|
|
32
32
|
},
|
|
33
33
|
"keywords": [
|
|
34
34
|
"mcp",
|
package/skills/ucu-mcp/SKILL.md
CHANGED
|
@@ -63,6 +63,20 @@ whether your click actually landed:
|
|
|
63
63
|
A `warnings[]` array in the receipt explains the fallback. Never assume a
|
|
64
64
|
coordinate-fallback click succeeded without checking.
|
|
65
65
|
|
|
66
|
+
### Dispatch method (v0.6.0+) — background operation
|
|
67
|
+
|
|
68
|
+
Every input tool (`click`, `double_click`, `scroll`, `drag`, `move`, `type_text`,
|
|
69
|
+
`press_key`) returns a `result.dispatch` field:
|
|
70
|
+
|
|
71
|
+
| `dispatch` | Meaning |
|
|
72
|
+
|---|---|
|
|
73
|
+
| `"per-pid"` | Event posted to the target process via SLEventPostToPid/CGEventPostToPid — **no global cursor move, no foreground theft**. This is the default when `focus_app` has established a target. |
|
|
74
|
+
| `"hid-tap"` | Event posted to the global HID event tap (moves the cursor, may disturb foreground). Happens when: no active target (call `focus_app` first), the target is frontmost, or the app is a canvas/GPU app (Blender/Unity/games) that filters per-pid events. |
|
|
75
|
+
|
|
76
|
+
When `dispatch:"hid-tap"`, a `warnings[]` entry explains it. To avoid cursor
|
|
77
|
+
movement, always `focus_app` the target before input actions, so events route
|
|
78
|
+
per-process (Codex-style background operation).
|
|
79
|
+
|
|
66
80
|
## Tool selection — AX vs vision vs tray
|
|
67
81
|
|
|
68
82
|
**AX-first** (precise, survives layout shifts):
|
|
@@ -112,6 +126,9 @@ also opaque.
|
|
|
112
126
|
|
|
113
127
|
## Operating Rules
|
|
114
128
|
|
|
129
|
+
- **`focus_app` before input.** Input events route per-process (no cursor move,
|
|
130
|
+
no foreground theft) only when an active target with a pid is established.
|
|
131
|
+
Without `focus_app`, events fall back to HID-tap (moves the cursor).
|
|
115
132
|
- **Re-observe before every action.** The screen changes between your calls.
|
|
116
133
|
A `focus_app` from 5 calls ago may be stale; a window may have closed.
|
|
117
134
|
- **AX-first, coordinates only as fallback.** AX clicks are precise and
|