screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
17
|
import { spawn } from "node:child_process";
|
|
18
18
|
import { EventEmitter } from "node:events";
|
|
19
|
+
import fs from "node:fs";
|
|
19
20
|
import path from "node:path";
|
|
20
21
|
import { createInterface } from "node:readline";
|
|
21
22
|
/**
|
|
@@ -26,8 +27,12 @@ const METHOD_TIMEOUTS = {
|
|
|
26
27
|
"app.launch": 30_000,
|
|
27
28
|
"cg.captureScreen": 15_000,
|
|
28
29
|
"cg.captureWindow": 15_000,
|
|
30
|
+
"cg.captureWindowBuffer": 15_000,
|
|
31
|
+
"cg.typeText": 30_000, // L2-66 fix: long text keystroke simulation needs more time
|
|
29
32
|
"vision.ocr": 20_000,
|
|
33
|
+
"vision.ocrRegion": 20_000,
|
|
30
34
|
"vision.findText": 20_000,
|
|
35
|
+
"ax.getMenuBar": 30_000,
|
|
31
36
|
};
|
|
32
37
|
/**
|
|
33
38
|
* Resolves the correct native bridge binary path for the current platform.
|
|
@@ -35,10 +40,24 @@ const METHOD_TIMEOUTS = {
|
|
|
35
40
|
function defaultBinaryPath() {
|
|
36
41
|
// import.meta.dirname is Node 20+; for Node 18 derive from import.meta.url
|
|
37
42
|
const base = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
|
|
38
|
-
|
|
43
|
+
const platform = process.platform;
|
|
44
|
+
const arch = process.arch;
|
|
45
|
+
const isWindows = platform === "win32";
|
|
46
|
+
const binaryName = isWindows ? "windows-bridge.exe" : "macos-bridge";
|
|
47
|
+
// 1. Check prebuilt binary shipped via npm (bin/<platform>-<arch>/)
|
|
48
|
+
const prebuilt = path.resolve(base, `../../bin/${platform}-${arch}/${binaryName}`);
|
|
49
|
+
if (fs.existsSync(prebuilt)) {
|
|
50
|
+
return prebuilt;
|
|
51
|
+
}
|
|
52
|
+
// 2. Fall back to local dev build paths
|
|
53
|
+
if (isWindows) {
|
|
39
54
|
return path.resolve(base, "../../native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe");
|
|
40
55
|
}
|
|
41
|
-
// macOS
|
|
56
|
+
// macOS — try arch-specific path first, then generic release
|
|
57
|
+
const archSpecific = path.resolve(base, `../../native/macos-bridge/.build/${arch}-apple-macosx/release/macos-bridge`);
|
|
58
|
+
if (fs.existsSync(archSpecific)) {
|
|
59
|
+
return archSpecific;
|
|
60
|
+
}
|
|
42
61
|
return path.resolve(base, "../../native/macos-bridge/.build/release/macos-bridge");
|
|
43
62
|
}
|
|
44
63
|
/**
|
|
@@ -54,7 +73,27 @@ export class BridgeClient extends EventEmitter {
|
|
|
54
73
|
pending = new Map();
|
|
55
74
|
binaryPath;
|
|
56
75
|
restarting = false;
|
|
76
|
+
/** Resolves when the current restart completes (callers can await it) */
|
|
77
|
+
restartPromise = null;
|
|
57
78
|
started = false;
|
|
79
|
+
consecutiveTimeouts = 0;
|
|
80
|
+
consecutiveRestarts = 0;
|
|
81
|
+
lastRestartAt = 0;
|
|
82
|
+
/** Serializes stdin writes to prevent interleaving of large payloads */
|
|
83
|
+
writeQueue = Promise.resolve();
|
|
84
|
+
/** Max concurrent bridge requests to prevent DoS */
|
|
85
|
+
maxConcurrent = 20;
|
|
86
|
+
activeRequests = 0;
|
|
87
|
+
/** Force-restart bridge after this many consecutive RPC timeouts */
|
|
88
|
+
static MAX_CONSECUTIVE_TIMEOUTS = 3;
|
|
89
|
+
/** Give up restarting after this many consecutive restart failures */
|
|
90
|
+
static MAX_CONSECUTIVE_RESTARTS = 8;
|
|
91
|
+
/** Reset restart counter if bridge stays alive for this long */
|
|
92
|
+
static RESTART_HEALTH_WINDOW_MS = 15_000;
|
|
93
|
+
/** Base delay between restart attempts (doubles each retry) */
|
|
94
|
+
static RESTART_BASE_DELAY_MS = 500;
|
|
95
|
+
/** After hitting max restarts, pause for this long before allowing retries */
|
|
96
|
+
static RESTART_COOLDOWN_MS = 60_000;
|
|
58
97
|
constructor(binaryPath) {
|
|
59
98
|
super();
|
|
60
99
|
this.binaryPath = binaryPath ?? defaultBinaryPath();
|
|
@@ -64,25 +103,102 @@ export class BridgeClient extends EventEmitter {
|
|
|
64
103
|
return;
|
|
65
104
|
await this.spawn();
|
|
66
105
|
this.started = true;
|
|
106
|
+
// Verify bridge is responsive before returning
|
|
107
|
+
try {
|
|
108
|
+
const pingOk = await Promise.race([
|
|
109
|
+
this._sendRaw("ping", undefined, 5_000).then(() => true),
|
|
110
|
+
new Promise((r) => setTimeout(() => r(false), 5_000)),
|
|
111
|
+
]);
|
|
112
|
+
if (!pingOk) {
|
|
113
|
+
const stderrContext = this.recentStderr.length > 0
|
|
114
|
+
? `\nBridge stderr:\n ${this.recentStderr.slice(-5).join("\n ")}`
|
|
115
|
+
: "";
|
|
116
|
+
console.error(`[BridgeClient] Bridge did not respond to initial ping.${stderrContext}\n` +
|
|
117
|
+
`Check: System Settings > Privacy & Security > Accessibility permissions for your terminal app.`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
// Non-fatal — bridge may still respond to subsequent calls
|
|
122
|
+
}
|
|
67
123
|
}
|
|
68
124
|
async stop() {
|
|
69
125
|
this.started = false;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
126
|
+
this.killProcess();
|
|
127
|
+
this.rejectAllPending("Bridge stopped");
|
|
128
|
+
}
|
|
129
|
+
async call(method, params, timeoutMs) {
|
|
130
|
+
// Wait for a slot instead of immediately rejecting — handles bursts from
|
|
131
|
+
// intelligence wrapper + perception generating multiple bridge calls per tool
|
|
132
|
+
if (this.activeRequests >= this.maxConcurrent) {
|
|
133
|
+
const waitStart = Date.now();
|
|
134
|
+
while (this.activeRequests >= this.maxConcurrent) {
|
|
135
|
+
if (Date.now() - waitStart > 5_000) {
|
|
136
|
+
throw new Error("Bridge overloaded: too many concurrent requests (waited 5s)");
|
|
137
|
+
}
|
|
138
|
+
await new Promise(r => setTimeout(r, 50));
|
|
139
|
+
}
|
|
73
140
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
141
|
+
this.activeRequests++;
|
|
142
|
+
try {
|
|
143
|
+
return await this._callInner(method, params, timeoutMs);
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
this.activeRequests--;
|
|
79
147
|
}
|
|
80
148
|
}
|
|
81
|
-
|
|
149
|
+
/**
|
|
150
|
+
* Low-level send: writes a JSON-RPC request and waits for response.
|
|
151
|
+
* Does NOT check restart state or process liveness — used inside restart()
|
|
152
|
+
* to avoid deadlock (this.call → _callInner awaits restartPromise = deadlock).
|
|
153
|
+
*/
|
|
154
|
+
async _sendRaw(method, params, timeoutMs) {
|
|
155
|
+
const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
|
|
156
|
+
if (!this.process || !this.process.stdin?.writable) {
|
|
157
|
+
throw new Error("Bridge process not available for raw send");
|
|
158
|
+
}
|
|
159
|
+
const id = this.nextId++;
|
|
160
|
+
const request = { id, method };
|
|
161
|
+
if (params)
|
|
162
|
+
request.params = params;
|
|
163
|
+
return new Promise((resolve, reject) => {
|
|
164
|
+
const timer = setTimeout(() => {
|
|
165
|
+
this.pending.delete(id);
|
|
166
|
+
reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
|
|
167
|
+
}, effectiveTimeout);
|
|
168
|
+
this.pending.set(id, {
|
|
169
|
+
resolve: resolve,
|
|
170
|
+
reject,
|
|
171
|
+
timer,
|
|
172
|
+
});
|
|
173
|
+
const line = JSON.stringify(request);
|
|
174
|
+
this.writeQueue = this.writeQueue.then(() => {
|
|
175
|
+
return new Promise((writeResolve) => {
|
|
176
|
+
try {
|
|
177
|
+
this.process.stdin.write(line + "\n", () => writeResolve());
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
this.pending.delete(id);
|
|
181
|
+
clearTimeout(timer);
|
|
182
|
+
reject(new Error(`Bridge stdin write failed for "${method}"`));
|
|
183
|
+
writeResolve();
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
async _callInner(method, params, timeoutMs) {
|
|
82
190
|
const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
|
|
191
|
+
// Wait for any in-progress restart before proceeding
|
|
192
|
+
if (this.restarting && this.restartPromise) {
|
|
193
|
+
await this.restartPromise;
|
|
194
|
+
}
|
|
83
195
|
if (!this.process || this.process.exitCode !== null) {
|
|
84
196
|
await this.restart();
|
|
85
197
|
}
|
|
198
|
+
// Guard: restart may have failed, no usable process
|
|
199
|
+
if (!this.process || !this.process.stdin?.writable) {
|
|
200
|
+
throw new Error(`Bridge unavailable after ${this.consecutiveRestarts} restart attempts`);
|
|
201
|
+
}
|
|
86
202
|
const id = this.nextId++;
|
|
87
203
|
const request = { id, method };
|
|
88
204
|
if (params) {
|
|
@@ -91,15 +207,34 @@ export class BridgeClient extends EventEmitter {
|
|
|
91
207
|
return new Promise((resolve, reject) => {
|
|
92
208
|
const timer = setTimeout(() => {
|
|
93
209
|
this.pending.delete(id);
|
|
210
|
+
this.consecutiveTimeouts++;
|
|
94
211
|
reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
|
|
212
|
+
// Force restart if bridge appears stalled
|
|
213
|
+
if (this.consecutiveTimeouts >= BridgeClient.MAX_CONSECUTIVE_TIMEOUTS) {
|
|
214
|
+
this.consecutiveTimeouts = 0;
|
|
215
|
+
this.restart().catch(() => { });
|
|
216
|
+
}
|
|
95
217
|
}, effectiveTimeout);
|
|
96
218
|
this.pending.set(id, {
|
|
97
219
|
resolve: resolve,
|
|
98
220
|
reject,
|
|
99
221
|
timer,
|
|
100
222
|
});
|
|
101
|
-
const line = JSON.stringify(request)
|
|
102
|
-
|
|
223
|
+
const line = JSON.stringify(request);
|
|
224
|
+
// Serialize stdin writes to prevent interleaving of large payloads
|
|
225
|
+
this.writeQueue = this.writeQueue.then(() => {
|
|
226
|
+
return new Promise((writeResolve) => {
|
|
227
|
+
try {
|
|
228
|
+
this.process.stdin.write(line + "\n", () => writeResolve());
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
this.pending.delete(id);
|
|
232
|
+
clearTimeout(timer);
|
|
233
|
+
reject(new Error(`Bridge stdin write failed for "${method}"`));
|
|
234
|
+
writeResolve();
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
});
|
|
103
238
|
});
|
|
104
239
|
}
|
|
105
240
|
async ping() {
|
|
@@ -108,19 +243,30 @@ export class BridgeClient extends EventEmitter {
|
|
|
108
243
|
async checkPermissions() {
|
|
109
244
|
return this.call("check_permissions");
|
|
110
245
|
}
|
|
246
|
+
/** Recent stderr lines from the bridge process (kept for diagnostics) */
|
|
247
|
+
recentStderr = [];
|
|
248
|
+
static MAX_STDERR_LINES = 20;
|
|
249
|
+
/** Get recent stderr output for diagnostics */
|
|
250
|
+
getRecentStderr() {
|
|
251
|
+
return [...this.recentStderr];
|
|
252
|
+
}
|
|
111
253
|
async spawn() {
|
|
112
254
|
const child = spawn(this.binaryPath, [], {
|
|
113
255
|
stdio: ["pipe", "pipe", "pipe"],
|
|
114
256
|
});
|
|
257
|
+
// Track which process this is so stale event handlers don't trigger restarts
|
|
258
|
+
const spawnedProcess = child;
|
|
115
259
|
child.on("error", (err) => {
|
|
116
260
|
this.emit("error", err);
|
|
117
|
-
if
|
|
261
|
+
// Only auto-restart if this is still the active process
|
|
262
|
+
if (this.started && this.process === spawnedProcess) {
|
|
118
263
|
this.restart().catch(() => { });
|
|
119
264
|
}
|
|
120
265
|
});
|
|
121
266
|
child.on("exit", (code) => {
|
|
122
267
|
this.emit("exit", code);
|
|
123
|
-
if
|
|
268
|
+
// Only auto-restart if this is still the active process and not mid-restart
|
|
269
|
+
if (this.started && !this.restarting && this.process === spawnedProcess) {
|
|
124
270
|
this.restart().catch(() => { });
|
|
125
271
|
}
|
|
126
272
|
});
|
|
@@ -129,9 +275,16 @@ export class BridgeClient extends EventEmitter {
|
|
|
129
275
|
rl.on("line", (line) => {
|
|
130
276
|
this.handleLine(line);
|
|
131
277
|
});
|
|
132
|
-
//
|
|
278
|
+
// Capture stderr for diagnostics and emit
|
|
133
279
|
child.stderr?.on("data", (data) => {
|
|
134
|
-
|
|
280
|
+
const text = data.toString();
|
|
281
|
+
this.emit("stderr", text);
|
|
282
|
+
for (const line of text.split("\n").filter(Boolean)) {
|
|
283
|
+
this.recentStderr.push(line);
|
|
284
|
+
if (this.recentStderr.length > BridgeClient.MAX_STDERR_LINES) {
|
|
285
|
+
this.recentStderr.shift();
|
|
286
|
+
}
|
|
287
|
+
}
|
|
135
288
|
});
|
|
136
289
|
this.process = child;
|
|
137
290
|
}
|
|
@@ -154,6 +307,11 @@ export class BridgeClient extends EventEmitter {
|
|
|
154
307
|
return;
|
|
155
308
|
this.pending.delete(response.id);
|
|
156
309
|
clearTimeout(pending.timer);
|
|
310
|
+
// Any response (success or error) means bridge is alive — reset crash counters
|
|
311
|
+
this.consecutiveTimeouts = 0;
|
|
312
|
+
if (this.consecutiveRestarts > 0) {
|
|
313
|
+
this.consecutiveRestarts = 0;
|
|
314
|
+
}
|
|
157
315
|
if (response.error) {
|
|
158
316
|
pending.reject(new Error(response.error.message));
|
|
159
317
|
}
|
|
@@ -161,27 +319,91 @@ export class BridgeClient extends EventEmitter {
|
|
|
161
319
|
pending.resolve(response.result);
|
|
162
320
|
}
|
|
163
321
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
return;
|
|
167
|
-
this.restarting = true;
|
|
168
|
-
// Reject all pending requests
|
|
322
|
+
/** Reject all pending requests with the given reason. */
|
|
323
|
+
rejectAllPending(reason) {
|
|
169
324
|
for (const [id, pending] of this.pending) {
|
|
170
325
|
clearTimeout(pending.timer);
|
|
171
|
-
pending.reject(new Error(
|
|
326
|
+
pending.reject(new Error(reason));
|
|
172
327
|
this.pending.delete(id);
|
|
173
328
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
329
|
+
}
|
|
330
|
+
/** Kill the current process and null the reference. */
|
|
331
|
+
killProcess() {
|
|
332
|
+
if (this.process) {
|
|
333
|
+
try {
|
|
334
|
+
this.process.kill("SIGKILL");
|
|
178
335
|
}
|
|
179
|
-
|
|
180
|
-
this.
|
|
336
|
+
catch { /* already dead */ }
|
|
337
|
+
this.process = null;
|
|
181
338
|
}
|
|
182
|
-
|
|
183
|
-
|
|
339
|
+
}
|
|
340
|
+
async restart() {
|
|
341
|
+
// If restart already in progress, wait for it instead of returning early.
|
|
342
|
+
// Without this, concurrent callers see this.process=null and throw "Bridge unavailable".
|
|
343
|
+
if (this.restarting) {
|
|
344
|
+
if (this.restartPromise)
|
|
345
|
+
await this.restartPromise;
|
|
346
|
+
return;
|
|
184
347
|
}
|
|
348
|
+
this.restarting = true;
|
|
349
|
+
const doRestart = async () => {
|
|
350
|
+
this.rejectAllPending("Bridge process crashed, restarting");
|
|
351
|
+
try {
|
|
352
|
+
// Reset restart counter if bridge was healthy for a while
|
|
353
|
+
if (Date.now() - this.lastRestartAt > BridgeClient.RESTART_HEALTH_WINDOW_MS) {
|
|
354
|
+
this.consecutiveRestarts = 0;
|
|
355
|
+
}
|
|
356
|
+
this.consecutiveRestarts++;
|
|
357
|
+
this.lastRestartAt = Date.now();
|
|
358
|
+
// Too many consecutive restarts — enter cooldown instead of permanent death
|
|
359
|
+
if (this.consecutiveRestarts > BridgeClient.MAX_CONSECUTIVE_RESTARTS) {
|
|
360
|
+
this.emit("error", new Error(`Bridge restart limit reached (${BridgeClient.MAX_CONSECUTIVE_RESTARTS} consecutive failures). ` +
|
|
361
|
+
`Cooling down for ${BridgeClient.RESTART_COOLDOWN_MS / 1000}s before retrying.`));
|
|
362
|
+
// Wait for cooldown period, then reset and allow retry
|
|
363
|
+
await new Promise((r) => setTimeout(r, BridgeClient.RESTART_COOLDOWN_MS));
|
|
364
|
+
this.consecutiveRestarts = 1; // Reset but count this as attempt #1
|
|
365
|
+
}
|
|
366
|
+
// Exponential backoff: 500ms, 1s, 2s, 4s, 8s
|
|
367
|
+
const delay = BridgeClient.RESTART_BASE_DELAY_MS * Math.pow(2, this.consecutiveRestarts - 1);
|
|
368
|
+
this.killProcess();
|
|
369
|
+
// Wait for backoff delay — let old process fully die
|
|
370
|
+
await new Promise((r) => setTimeout(r, Math.min(delay, 8_000)));
|
|
371
|
+
await this.spawn();
|
|
372
|
+
// Verify the new process is responsive
|
|
373
|
+
// NOTE: call _sendRaw directly instead of this.call() to avoid deadlock —
|
|
374
|
+
// this.call() → _callInner() awaits this.restartPromise, which IS this function.
|
|
375
|
+
const pingTimeout = 5_000;
|
|
376
|
+
const pingOk = await Promise.race([
|
|
377
|
+
this._sendRaw("ping", undefined, pingTimeout).then(() => true),
|
|
378
|
+
new Promise((r) => setTimeout(() => r(false), pingTimeout)),
|
|
379
|
+
]);
|
|
380
|
+
if (!pingOk) {
|
|
381
|
+
const stderrContext = this.recentStderr.length > 0
|
|
382
|
+
? `\nBridge stderr:\n ${this.recentStderr.slice(-5).join("\n ")}`
|
|
383
|
+
: "";
|
|
384
|
+
throw new Error(`Native bridge did not respond to ping within 5s.${stderrContext}\n` +
|
|
385
|
+
`Possible causes:\n` +
|
|
386
|
+
` 1. Accessibility permissions not granted — open System Settings > Privacy & Security > Accessibility and add your terminal app\n` +
|
|
387
|
+
` 2. Bridge binary may be corrupted — run: npm run build:native\n` +
|
|
388
|
+
` 3. Another bridge process may be stuck — check: pgrep macos-bridge`);
|
|
389
|
+
}
|
|
390
|
+
// Healthy restart — reset counters (bridge is alive and responsive)
|
|
391
|
+
this.consecutiveTimeouts = 0;
|
|
392
|
+
this.consecutiveRestarts = 0;
|
|
393
|
+
this.emit("restart");
|
|
394
|
+
}
|
|
395
|
+
catch (err) {
|
|
396
|
+
// Spawn failed — kill zombie if any
|
|
397
|
+
this.killProcess();
|
|
398
|
+
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
399
|
+
}
|
|
400
|
+
finally {
|
|
401
|
+
this.restarting = false;
|
|
402
|
+
this.restartPromise = null;
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
this.restartPromise = doRestart();
|
|
406
|
+
await this.restartPromise;
|
|
185
407
|
}
|
|
186
408
|
}
|
|
187
409
|
/**
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
/**
|
|
4
|
+
* Observer state helpers — read/write observer state file.
|
|
5
|
+
* Used by: observer-daemon.ts (writes), playbook engine (reads), MCP tools (reads).
|
|
6
|
+
*/
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import { readJsonWithRecovery, writeFileAtomicSync } from "../util/atomic-write.js";
|
|
9
|
+
import { DEFAULT_POPUP_PATTERNS, OBSERVER_DIR, OBSERVER_STATE_FILE, OBSERVER_COMMANDS_FILE, OBSERVER_PID_FILE, CAPTURE_LOCK_FILE, } from "./types.js";
|
|
10
|
+
/** Read current observer state from disk. Returns null if not running. */
|
|
11
|
+
export function readObserverState() {
|
|
12
|
+
return readJsonWithRecovery(OBSERVER_STATE_FILE);
|
|
13
|
+
}
|
|
14
|
+
/** Write observer state to disk (atomic). */
|
|
15
|
+
export function writeObserverState(state) {
|
|
16
|
+
fs.mkdirSync(OBSERVER_DIR, { recursive: true });
|
|
17
|
+
writeFileAtomicSync(OBSERVER_STATE_FILE, JSON.stringify(state, null, 2));
|
|
18
|
+
}
|
|
19
|
+
/** Get PID of running observer daemon, or null if not running. */
|
|
20
|
+
export function getObserverDaemonPid() {
|
|
21
|
+
try {
|
|
22
|
+
const pid = Number(fs.readFileSync(OBSERVER_PID_FILE, "utf-8").trim());
|
|
23
|
+
if (Number.isNaN(pid))
|
|
24
|
+
return null;
|
|
25
|
+
// Check if process is alive
|
|
26
|
+
try {
|
|
27
|
+
process.kill(pid, 0);
|
|
28
|
+
return pid;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// Process not running — stale PID file
|
|
32
|
+
try {
|
|
33
|
+
fs.unlinkSync(OBSERVER_PID_FILE);
|
|
34
|
+
}
|
|
35
|
+
catch { /* ignore */ }
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Match OCR text against popup patterns. Returns first match or null. */
|
|
44
|
+
export function detectPopup(ocrText, patterns = DEFAULT_POPUP_PATTERNS) {
|
|
45
|
+
const lowerText = ocrText.toLowerCase();
|
|
46
|
+
for (const p of patterns) {
|
|
47
|
+
const regex = new RegExp(p.pattern, "i");
|
|
48
|
+
if (regex.test(lowerText)) {
|
|
49
|
+
return {
|
|
50
|
+
matchedText: ocrText.substring(0, 200),
|
|
51
|
+
pattern: p.pattern,
|
|
52
|
+
dismissAction: p.action,
|
|
53
|
+
detectedAt: new Date().toISOString(),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
// ── Command file helpers ──
|
|
60
|
+
/** Read all commands from the command file. */
|
|
61
|
+
export function readObserverCommands() {
|
|
62
|
+
const data = readJsonWithRecovery(OBSERVER_COMMANDS_FILE);
|
|
63
|
+
return data ?? [];
|
|
64
|
+
}
|
|
65
|
+
/** Write commands to disk (atomic). */
|
|
66
|
+
export function writeObserverCommands(commands) {
|
|
67
|
+
fs.mkdirSync(OBSERVER_DIR, { recursive: true });
|
|
68
|
+
writeFileAtomicSync(OBSERVER_COMMANDS_FILE, JSON.stringify(commands, null, 2));
|
|
69
|
+
}
|
|
70
|
+
/** Submit a new command. Returns the command ID. */
|
|
71
|
+
export function submitObserverCommand(cmd) {
|
|
72
|
+
const commands = readObserverCommands();
|
|
73
|
+
const id = `cmd_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
74
|
+
const newCmd = {
|
|
75
|
+
...cmd,
|
|
76
|
+
id,
|
|
77
|
+
status: "pending",
|
|
78
|
+
createdAt: new Date().toISOString(),
|
|
79
|
+
};
|
|
80
|
+
commands.push(newCmd);
|
|
81
|
+
// Cap at 50 commands, evict oldest completed/errored first
|
|
82
|
+
if (commands.length > 50) {
|
|
83
|
+
const done = commands.filter((c) => c.status === "done" || c.status === "error");
|
|
84
|
+
if (done.length > 0) {
|
|
85
|
+
const removeId = done[0].id;
|
|
86
|
+
const idx = commands.findIndex((c) => c.id === removeId);
|
|
87
|
+
if (idx >= 0)
|
|
88
|
+
commands.splice(idx, 1);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
commands.shift();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
writeObserverCommands(commands);
|
|
95
|
+
return id;
|
|
96
|
+
}
|
|
97
|
+
/** Get a command by ID. */
|
|
98
|
+
export function getObserverCommand(id) {
|
|
99
|
+
const commands = readObserverCommands();
|
|
100
|
+
return commands.find((c) => c.id === id) ?? null;
|
|
101
|
+
}
|
|
102
|
+
/** Get the latest OCR text from observer (if running and has data). */
|
|
103
|
+
export function getObserverOcrText() {
|
|
104
|
+
const state = readObserverState();
|
|
105
|
+
if (!state?.running || !state.lastFrame)
|
|
106
|
+
return null;
|
|
107
|
+
return state.lastFrame.ocrText;
|
|
108
|
+
}
|
|
109
|
+
/** Get detected popup from observer (if any). */
|
|
110
|
+
export function getObserverPopup() {
|
|
111
|
+
const state = readObserverState();
|
|
112
|
+
if (!state?.running)
|
|
113
|
+
return null;
|
|
114
|
+
return state.popup;
|
|
115
|
+
}
|
|
116
|
+
// ── Capture lock helpers ──
|
|
117
|
+
// Prevents observer daemon and perception coordinator from capturing simultaneously.
|
|
118
|
+
const LOCK_STALE_MS = 10_000; // Locks older than 10s are considered stale
|
|
119
|
+
/**
|
|
120
|
+
* Acquire the capture lock. Returns true if acquired, false if held by another process.
|
|
121
|
+
* Lock contains PID + timestamp so stale locks from crashed processes can be cleaned up.
|
|
122
|
+
*/
|
|
123
|
+
export function acquireCaptureLock() {
|
|
124
|
+
fs.mkdirSync(OBSERVER_DIR, { recursive: true });
|
|
125
|
+
try {
|
|
126
|
+
// Check existing lock
|
|
127
|
+
const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
|
|
128
|
+
if (existing) {
|
|
129
|
+
const parts = existing.split(":");
|
|
130
|
+
const lockPid = Number(parts[0]);
|
|
131
|
+
const lockTime = Number(parts[1]);
|
|
132
|
+
// Stale lock check
|
|
133
|
+
if (Date.now() - lockTime > LOCK_STALE_MS) {
|
|
134
|
+
// Lock is stale — safe to overwrite
|
|
135
|
+
}
|
|
136
|
+
else if (lockPid !== process.pid) {
|
|
137
|
+
// Check if holding process is alive
|
|
138
|
+
try {
|
|
139
|
+
process.kill(lockPid, 0);
|
|
140
|
+
return false; // Process alive, lock is valid
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
// Process dead — stale lock
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
// Lock is ours or stale — fall through to acquire
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// No lock file — safe to create
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
fs.writeFileSync(CAPTURE_LOCK_FILE, `${process.pid}:${Date.now()}`);
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Release the capture lock (only if we hold it).
|
|
162
|
+
*/
|
|
163
|
+
export function releaseCaptureLock() {
|
|
164
|
+
try {
|
|
165
|
+
const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
|
|
166
|
+
const lockPid = Number(existing.split(":")[0]);
|
|
167
|
+
if (lockPid === process.pid) {
|
|
168
|
+
fs.unlinkSync(CAPTURE_LOCK_FILE);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
// No lock file or already cleaned up
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Check if the capture lock is currently held (by any process).
|
|
177
|
+
*/
|
|
178
|
+
export function isCaptureLocked() {
|
|
179
|
+
try {
|
|
180
|
+
const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
|
|
181
|
+
if (!existing)
|
|
182
|
+
return false;
|
|
183
|
+
const parts = existing.split(":");
|
|
184
|
+
const lockPid = Number(parts[0]);
|
|
185
|
+
const lockTime = Number(parts[1]);
|
|
186
|
+
if (Date.now() - lockTime > LOCK_STALE_MS)
|
|
187
|
+
return false;
|
|
188
|
+
try {
|
|
189
|
+
process.kill(lockPid, 0);
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
return false; // Process dead
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
/**
|
|
4
|
+
* Observer types — background app-level visual monitoring
|
|
5
|
+
*
|
|
6
|
+
* The observer daemon watches a single app window via CGWindowListCreateImage,
|
|
7
|
+
* runs OCR only when pixels change, and exposes state via a JSON file.
|
|
8
|
+
* The playbook engine reads this file — zero overhead on the hot path.
|
|
9
|
+
*/
|
|
10
|
+
import os from "node:os";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
/** Default popup patterns covering common OS and app dialogs */
|
|
13
|
+
export const DEFAULT_POPUP_PATTERNS = [
|
|
14
|
+
// Save dialogs
|
|
15
|
+
{ pattern: "Do you want to save", action: "click_cancel", buttonText: "Don't Save" },
|
|
16
|
+
{ pattern: "Save changes", action: "click_cancel", buttonText: "Don't Save" },
|
|
17
|
+
{ pattern: "would you like to save", action: "click_cancel", buttonText: "Don't Save" },
|
|
18
|
+
// Permission dialogs
|
|
19
|
+
{ pattern: "would like to access", action: "click_allow", buttonText: "Allow" },
|
|
20
|
+
{ pattern: "wants to access", action: "click_allow", buttonText: "Allow" },
|
|
21
|
+
{ pattern: "requesting permission", action: "click_allow", buttonText: "Allow" },
|
|
22
|
+
// Cookie banners
|
|
23
|
+
{ pattern: "Accept all cookies", action: "click_ok", buttonText: "Accept" },
|
|
24
|
+
{ pattern: "cookie preferences", action: "click_ok", buttonText: "Accept All" },
|
|
25
|
+
// Update prompts
|
|
26
|
+
{ pattern: "update is available", action: "click_cancel", buttonText: "Later" },
|
|
27
|
+
{ pattern: "Remind Me Later", action: "click_cancel", buttonText: "Remind Me Later" },
|
|
28
|
+
{ pattern: "Update Now", action: "click_cancel", buttonText: "Not Now" },
|
|
29
|
+
// Generic modals
|
|
30
|
+
{ pattern: "Are you sure", action: "click_ok", buttonText: "OK" },
|
|
31
|
+
{ pattern: "Close without saving", action: "click_ok", buttonText: "Close" },
|
|
32
|
+
// Chrome specific
|
|
33
|
+
{ pattern: "Chrome is being controlled", action: "press_escape" },
|
|
34
|
+
{ pattern: "Restore pages", action: "press_escape" },
|
|
35
|
+
// macOS specific
|
|
36
|
+
{ pattern: "allow notifications", action: "click_deny", buttonText: "Don't Allow" },
|
|
37
|
+
];
|
|
38
|
+
export const OBSERVER_DIR = path.join(os.homedir(), ".screenhand", "observer");
|
|
39
|
+
export const OBSERVER_STATE_FILE = path.join(OBSERVER_DIR, "state.json");
|
|
40
|
+
export const OBSERVER_COMMANDS_FILE = path.join(OBSERVER_DIR, "commands.json");
|
|
41
|
+
export const OBSERVER_PID_FILE = path.join(OBSERVER_DIR, "observer.pid");
|
|
42
|
+
export const OBSERVER_LOG_FILE = path.join(OBSERVER_DIR, "observer.log");
|
|
43
|
+
export const CAPTURE_LOCK_FILE = path.join(OBSERVER_DIR, "capture.lock");
|