screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -16,6 +16,7 @@
16
16
  // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
17
  import { spawn } from "node:child_process";
18
18
  import { EventEmitter } from "node:events";
19
+ import fs from "node:fs";
19
20
  import path from "node:path";
20
21
  import { createInterface } from "node:readline";
21
22
  /**
@@ -26,8 +27,12 @@ const METHOD_TIMEOUTS = {
26
27
  "app.launch": 30_000,
27
28
  "cg.captureScreen": 15_000,
28
29
  "cg.captureWindow": 15_000,
30
+ "cg.captureWindowBuffer": 15_000,
31
+ "cg.typeText": 30_000, // L2-66 fix: long text keystroke simulation needs more time
29
32
  "vision.ocr": 20_000,
33
+ "vision.ocrRegion": 20_000,
30
34
  "vision.findText": 20_000,
35
+ "ax.getMenuBar": 30_000,
31
36
  };
32
37
  /**
33
38
  * Resolves the correct native bridge binary path for the current platform.
@@ -35,10 +40,24 @@ const METHOD_TIMEOUTS = {
35
40
  function defaultBinaryPath() {
36
41
  // import.meta.dirname is Node 20+; for Node 18 derive from import.meta.url
37
42
  const base = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
38
- if (process.platform === "win32") {
43
+ const platform = process.platform;
44
+ const arch = process.arch;
45
+ const isWindows = platform === "win32";
46
+ const binaryName = isWindows ? "windows-bridge.exe" : "macos-bridge";
47
+ // 1. Check prebuilt binary shipped via npm (bin/<platform>-<arch>/)
48
+ const prebuilt = path.resolve(base, `../../bin/${platform}-${arch}/${binaryName}`);
49
+ if (fs.existsSync(prebuilt)) {
50
+ return prebuilt;
51
+ }
52
+ // 2. Fall back to local dev build paths
53
+ if (isWindows) {
39
54
  return path.resolve(base, "../../native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe");
40
55
  }
41
- // macOS (default)
56
+ // macOS — try arch-specific path first, then generic release
57
+ const archSpecific = path.resolve(base, `../../native/macos-bridge/.build/${arch}-apple-macosx/release/macos-bridge`);
58
+ if (fs.existsSync(archSpecific)) {
59
+ return archSpecific;
60
+ }
42
61
  return path.resolve(base, "../../native/macos-bridge/.build/release/macos-bridge");
43
62
  }
44
63
  /**
@@ -54,7 +73,27 @@ export class BridgeClient extends EventEmitter {
54
73
  pending = new Map();
55
74
  binaryPath;
56
75
  restarting = false;
76
+ /** Resolves when the current restart completes (callers can await it) */
77
+ restartPromise = null;
57
78
  started = false;
79
+ consecutiveTimeouts = 0;
80
+ consecutiveRestarts = 0;
81
+ lastRestartAt = 0;
82
+ /** Serializes stdin writes to prevent interleaving of large payloads */
83
+ writeQueue = Promise.resolve();
84
+ /** Max concurrent bridge requests to prevent DoS */
85
+ maxConcurrent = 20;
86
+ activeRequests = 0;
87
+ /** Force-restart bridge after this many consecutive RPC timeouts */
88
+ static MAX_CONSECUTIVE_TIMEOUTS = 3;
89
+ /** Give up restarting after this many consecutive restart failures */
90
+ static MAX_CONSECUTIVE_RESTARTS = 8;
91
+ /** Reset restart counter if bridge stays alive for this long */
92
+ static RESTART_HEALTH_WINDOW_MS = 15_000;
93
+ /** Base delay between restart attempts (doubles each retry) */
94
+ static RESTART_BASE_DELAY_MS = 500;
95
+ /** After hitting max restarts, pause for this long before allowing retries */
96
+ static RESTART_COOLDOWN_MS = 60_000;
58
97
  constructor(binaryPath) {
59
98
  super();
60
99
  this.binaryPath = binaryPath ?? defaultBinaryPath();
@@ -64,25 +103,102 @@ export class BridgeClient extends EventEmitter {
64
103
  return;
65
104
  await this.spawn();
66
105
  this.started = true;
106
+ // Verify bridge is responsive before returning
107
+ try {
108
+ const pingOk = await Promise.race([
109
+ this._sendRaw("ping", undefined, 5_000).then(() => true),
110
+ new Promise((r) => setTimeout(() => r(false), 5_000)),
111
+ ]);
112
+ if (!pingOk) {
113
+ const stderrContext = this.recentStderr.length > 0
114
+ ? `\nBridge stderr:\n ${this.recentStderr.slice(-5).join("\n ")}`
115
+ : "";
116
+ console.error(`[BridgeClient] Bridge did not respond to initial ping.${stderrContext}\n` +
117
+ `Check: System Settings > Privacy & Security > Accessibility permissions for your terminal app.`);
118
+ }
119
+ }
120
+ catch {
121
+ // Non-fatal — bridge may still respond to subsequent calls
122
+ }
67
123
  }
68
124
  async stop() {
69
125
  this.started = false;
70
- if (this.process) {
71
- this.process.kill();
72
- this.process = null;
126
+ this.killProcess();
127
+ this.rejectAllPending("Bridge stopped");
128
+ }
129
+ async call(method, params, timeoutMs) {
130
+ // Wait for a slot instead of immediately rejecting — handles bursts from
131
+ // intelligence wrapper + perception generating multiple bridge calls per tool
132
+ if (this.activeRequests >= this.maxConcurrent) {
133
+ const waitStart = Date.now();
134
+ while (this.activeRequests >= this.maxConcurrent) {
135
+ if (Date.now() - waitStart > 5_000) {
136
+ throw new Error("Bridge overloaded: too many concurrent requests (waited 5s)");
137
+ }
138
+ await new Promise(r => setTimeout(r, 50));
139
+ }
73
140
  }
74
- // Reject all pending
75
- for (const [id, pending] of this.pending) {
76
- clearTimeout(pending.timer);
77
- pending.reject(new Error("Bridge stopped"));
78
- this.pending.delete(id);
141
+ this.activeRequests++;
142
+ try {
143
+ return await this._callInner(method, params, timeoutMs);
144
+ }
145
+ finally {
146
+ this.activeRequests--;
79
147
  }
80
148
  }
81
- async call(method, params, timeoutMs) {
149
+ /**
150
+ * Low-level send: writes a JSON-RPC request and waits for response.
151
+ * Does NOT check restart state or process liveness — used inside restart()
152
+ * to avoid deadlock (this.call → _callInner awaits restartPromise = deadlock).
153
+ */
154
+ async _sendRaw(method, params, timeoutMs) {
155
+ const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
156
+ if (!this.process || !this.process.stdin?.writable) {
157
+ throw new Error("Bridge process not available for raw send");
158
+ }
159
+ const id = this.nextId++;
160
+ const request = { id, method };
161
+ if (params)
162
+ request.params = params;
163
+ return new Promise((resolve, reject) => {
164
+ const timer = setTimeout(() => {
165
+ this.pending.delete(id);
166
+ reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
167
+ }, effectiveTimeout);
168
+ this.pending.set(id, {
169
+ resolve: resolve,
170
+ reject,
171
+ timer,
172
+ });
173
+ const line = JSON.stringify(request);
174
+ this.writeQueue = this.writeQueue.then(() => {
175
+ return new Promise((writeResolve) => {
176
+ try {
177
+ this.process.stdin.write(line + "\n", () => writeResolve());
178
+ }
179
+ catch {
180
+ this.pending.delete(id);
181
+ clearTimeout(timer);
182
+ reject(new Error(`Bridge stdin write failed for "${method}"`));
183
+ writeResolve();
184
+ }
185
+ });
186
+ });
187
+ });
188
+ }
189
+ async _callInner(method, params, timeoutMs) {
82
190
  const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
191
+ // Wait for any in-progress restart before proceeding
192
+ if (this.restarting && this.restartPromise) {
193
+ await this.restartPromise;
194
+ }
83
195
  if (!this.process || this.process.exitCode !== null) {
84
196
  await this.restart();
85
197
  }
198
+ // Guard: restart may have failed, no usable process
199
+ if (!this.process || !this.process.stdin?.writable) {
200
+ throw new Error(`Bridge unavailable after ${this.consecutiveRestarts} restart attempts`);
201
+ }
86
202
  const id = this.nextId++;
87
203
  const request = { id, method };
88
204
  if (params) {
@@ -91,15 +207,34 @@ export class BridgeClient extends EventEmitter {
91
207
  return new Promise((resolve, reject) => {
92
208
  const timer = setTimeout(() => {
93
209
  this.pending.delete(id);
210
+ this.consecutiveTimeouts++;
94
211
  reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
212
+ // Force restart if bridge appears stalled
213
+ if (this.consecutiveTimeouts >= BridgeClient.MAX_CONSECUTIVE_TIMEOUTS) {
214
+ this.consecutiveTimeouts = 0;
215
+ this.restart().catch(() => { });
216
+ }
95
217
  }, effectiveTimeout);
96
218
  this.pending.set(id, {
97
219
  resolve: resolve,
98
220
  reject,
99
221
  timer,
100
222
  });
101
- const line = JSON.stringify(request) + "\n";
102
- this.process.stdin.write(line);
223
+ const line = JSON.stringify(request);
224
+ // Serialize stdin writes to prevent interleaving of large payloads
225
+ this.writeQueue = this.writeQueue.then(() => {
226
+ return new Promise((writeResolve) => {
227
+ try {
228
+ this.process.stdin.write(line + "\n", () => writeResolve());
229
+ }
230
+ catch {
231
+ this.pending.delete(id);
232
+ clearTimeout(timer);
233
+ reject(new Error(`Bridge stdin write failed for "${method}"`));
234
+ writeResolve();
235
+ }
236
+ });
237
+ });
103
238
  });
104
239
  }
105
240
  async ping() {
@@ -108,19 +243,30 @@ export class BridgeClient extends EventEmitter {
108
243
  async checkPermissions() {
109
244
  return this.call("check_permissions");
110
245
  }
246
+ /** Recent stderr lines from the bridge process (kept for diagnostics) */
247
+ recentStderr = [];
248
+ static MAX_STDERR_LINES = 20;
249
+ /** Get recent stderr output for diagnostics */
250
+ getRecentStderr() {
251
+ return [...this.recentStderr];
252
+ }
111
253
  async spawn() {
112
254
  const child = spawn(this.binaryPath, [], {
113
255
  stdio: ["pipe", "pipe", "pipe"],
114
256
  });
257
+ // Track which process this is so stale event handlers don't trigger restarts
258
+ const spawnedProcess = child;
115
259
  child.on("error", (err) => {
116
260
  this.emit("error", err);
117
- if (this.started) {
261
+ // Only auto-restart if this is still the active process
262
+ if (this.started && this.process === spawnedProcess) {
118
263
  this.restart().catch(() => { });
119
264
  }
120
265
  });
121
266
  child.on("exit", (code) => {
122
267
  this.emit("exit", code);
123
- if (this.started && !this.restarting) {
268
+ // Only auto-restart if this is still the active process and not mid-restart
269
+ if (this.started && !this.restarting && this.process === spawnedProcess) {
124
270
  this.restart().catch(() => { });
125
271
  }
126
272
  });
@@ -129,9 +275,16 @@ export class BridgeClient extends EventEmitter {
129
275
  rl.on("line", (line) => {
130
276
  this.handleLine(line);
131
277
  });
132
- // Log stderr
278
+ // Capture stderr for diagnostics and emit
133
279
  child.stderr?.on("data", (data) => {
134
- this.emit("stderr", data.toString());
280
+ const text = data.toString();
281
+ this.emit("stderr", text);
282
+ for (const line of text.split("\n").filter(Boolean)) {
283
+ this.recentStderr.push(line);
284
+ if (this.recentStderr.length > BridgeClient.MAX_STDERR_LINES) {
285
+ this.recentStderr.shift();
286
+ }
287
+ }
135
288
  });
136
289
  this.process = child;
137
290
  }
@@ -154,6 +307,11 @@ export class BridgeClient extends EventEmitter {
154
307
  return;
155
308
  this.pending.delete(response.id);
156
309
  clearTimeout(pending.timer);
310
+ // Any response (success or error) means bridge is alive — reset crash counters
311
+ this.consecutiveTimeouts = 0;
312
+ if (this.consecutiveRestarts > 0) {
313
+ this.consecutiveRestarts = 0;
314
+ }
157
315
  if (response.error) {
158
316
  pending.reject(new Error(response.error.message));
159
317
  }
@@ -161,27 +319,91 @@ export class BridgeClient extends EventEmitter {
161
319
  pending.resolve(response.result);
162
320
  }
163
321
  }
164
- async restart() {
165
- if (this.restarting)
166
- return;
167
- this.restarting = true;
168
- // Reject all pending requests
322
+ /** Reject all pending requests with the given reason. */
323
+ rejectAllPending(reason) {
169
324
  for (const [id, pending] of this.pending) {
170
325
  clearTimeout(pending.timer);
171
- pending.reject(new Error("Bridge process crashed, restarting"));
326
+ pending.reject(new Error(reason));
172
327
  this.pending.delete(id);
173
328
  }
174
- try {
175
- if (this.process) {
176
- this.process.kill();
177
- this.process = null;
329
+ }
330
+ /** Kill the current process and null the reference. */
331
+ killProcess() {
332
+ if (this.process) {
333
+ try {
334
+ this.process.kill("SIGKILL");
178
335
  }
179
- await this.spawn();
180
- this.emit("restart");
336
+ catch { /* already dead */ }
337
+ this.process = null;
181
338
  }
182
- finally {
183
- this.restarting = false;
339
+ }
340
+ async restart() {
341
+ // If restart already in progress, wait for it instead of returning early.
342
+ // Without this, concurrent callers see this.process=null and throw "Bridge unavailable".
343
+ if (this.restarting) {
344
+ if (this.restartPromise)
345
+ await this.restartPromise;
346
+ return;
184
347
  }
348
+ this.restarting = true;
349
+ const doRestart = async () => {
350
+ this.rejectAllPending("Bridge process crashed, restarting");
351
+ try {
352
+ // Reset restart counter if bridge was healthy for a while
353
+ if (Date.now() - this.lastRestartAt > BridgeClient.RESTART_HEALTH_WINDOW_MS) {
354
+ this.consecutiveRestarts = 0;
355
+ }
356
+ this.consecutiveRestarts++;
357
+ this.lastRestartAt = Date.now();
358
+ // Too many consecutive restarts — enter cooldown instead of permanent death
359
+ if (this.consecutiveRestarts > BridgeClient.MAX_CONSECUTIVE_RESTARTS) {
360
+ this.emit("error", new Error(`Bridge restart limit reached (${BridgeClient.MAX_CONSECUTIVE_RESTARTS} consecutive failures). ` +
361
+ `Cooling down for ${BridgeClient.RESTART_COOLDOWN_MS / 1000}s before retrying.`));
362
+ // Wait for cooldown period, then reset and allow retry
363
+ await new Promise((r) => setTimeout(r, BridgeClient.RESTART_COOLDOWN_MS));
364
+ this.consecutiveRestarts = 1; // Reset but count this as attempt #1
365
+ }
366
+ // Exponential backoff: 500ms, 1s, 2s, 4s, 8s
367
+ const delay = BridgeClient.RESTART_BASE_DELAY_MS * Math.pow(2, this.consecutiveRestarts - 1);
368
+ this.killProcess();
369
+ // Wait for backoff delay — let old process fully die
370
+ await new Promise((r) => setTimeout(r, Math.min(delay, 8_000)));
371
+ await this.spawn();
372
+ // Verify the new process is responsive
373
+ // NOTE: call _sendRaw directly instead of this.call() to avoid deadlock —
374
+ // this.call() → _callInner() awaits this.restartPromise, which IS this function.
375
+ const pingTimeout = 5_000;
376
+ const pingOk = await Promise.race([
377
+ this._sendRaw("ping", undefined, pingTimeout).then(() => true),
378
+ new Promise((r) => setTimeout(() => r(false), pingTimeout)),
379
+ ]);
380
+ if (!pingOk) {
381
+ const stderrContext = this.recentStderr.length > 0
382
+ ? `\nBridge stderr:\n ${this.recentStderr.slice(-5).join("\n ")}`
383
+ : "";
384
+ throw new Error(`Native bridge did not respond to ping within 5s.${stderrContext}\n` +
385
+ `Possible causes:\n` +
386
+ ` 1. Accessibility permissions not granted — open System Settings > Privacy & Security > Accessibility and add your terminal app\n` +
387
+ ` 2. Bridge binary may be corrupted — run: npm run build:native\n` +
388
+ ` 3. Another bridge process may be stuck — check: pgrep macos-bridge`);
389
+ }
390
+ // Healthy restart — reset counters (bridge is alive and responsive)
391
+ this.consecutiveTimeouts = 0;
392
+ this.consecutiveRestarts = 0;
393
+ this.emit("restart");
394
+ }
395
+ catch (err) {
396
+ // Spawn failed — kill zombie if any
397
+ this.killProcess();
398
+ this.emit("error", err instanceof Error ? err : new Error(String(err)));
399
+ }
400
+ finally {
401
+ this.restarting = false;
402
+ this.restartPromise = null;
403
+ }
404
+ };
405
+ this.restartPromise = doRestart();
406
+ await this.restartPromise;
185
407
  }
186
408
  }
187
409
  /**
@@ -0,0 +1,199 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /**
4
+ * Observer state helpers — read/write observer state file.
5
+ * Used by: observer-daemon.ts (writes), playbook engine (reads), MCP tools (reads).
6
+ */
7
+ import fs from "node:fs";
8
+ import { readJsonWithRecovery, writeFileAtomicSync } from "../util/atomic-write.js";
9
+ import { DEFAULT_POPUP_PATTERNS, OBSERVER_DIR, OBSERVER_STATE_FILE, OBSERVER_COMMANDS_FILE, OBSERVER_PID_FILE, CAPTURE_LOCK_FILE, } from "./types.js";
10
+ /** Read current observer state from disk. Returns null if not running. */
11
+ export function readObserverState() {
12
+ return readJsonWithRecovery(OBSERVER_STATE_FILE);
13
+ }
14
+ /** Write observer state to disk (atomic). */
15
+ export function writeObserverState(state) {
16
+ fs.mkdirSync(OBSERVER_DIR, { recursive: true });
17
+ writeFileAtomicSync(OBSERVER_STATE_FILE, JSON.stringify(state, null, 2));
18
+ }
19
+ /** Get PID of running observer daemon, or null if not running. */
20
+ export function getObserverDaemonPid() {
21
+ try {
22
+ const pid = Number(fs.readFileSync(OBSERVER_PID_FILE, "utf-8").trim());
23
+ if (Number.isNaN(pid))
24
+ return null;
25
+ // Check if process is alive
26
+ try {
27
+ process.kill(pid, 0);
28
+ return pid;
29
+ }
30
+ catch {
31
+ // Process not running — stale PID file
32
+ try {
33
+ fs.unlinkSync(OBSERVER_PID_FILE);
34
+ }
35
+ catch { /* ignore */ }
36
+ return null;
37
+ }
38
+ }
39
+ catch {
40
+ return null;
41
+ }
42
+ }
43
+ /** Match OCR text against popup patterns. Returns first match or null. */
44
+ export function detectPopup(ocrText, patterns = DEFAULT_POPUP_PATTERNS) {
45
+ const lowerText = ocrText.toLowerCase();
46
+ for (const p of patterns) {
47
+ const regex = new RegExp(p.pattern, "i");
48
+ if (regex.test(lowerText)) {
49
+ return {
50
+ matchedText: ocrText.substring(0, 200),
51
+ pattern: p.pattern,
52
+ dismissAction: p.action,
53
+ detectedAt: new Date().toISOString(),
54
+ };
55
+ }
56
+ }
57
+ return null;
58
+ }
59
+ // ── Command file helpers ──
60
+ /** Read all commands from the command file. */
61
+ export function readObserverCommands() {
62
+ const data = readJsonWithRecovery(OBSERVER_COMMANDS_FILE);
63
+ return data ?? [];
64
+ }
65
+ /** Write commands to disk (atomic). */
66
+ export function writeObserverCommands(commands) {
67
+ fs.mkdirSync(OBSERVER_DIR, { recursive: true });
68
+ writeFileAtomicSync(OBSERVER_COMMANDS_FILE, JSON.stringify(commands, null, 2));
69
+ }
70
+ /** Submit a new command. Returns the command ID. */
71
+ export function submitObserverCommand(cmd) {
72
+ const commands = readObserverCommands();
73
+ const id = `cmd_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
74
+ const newCmd = {
75
+ ...cmd,
76
+ id,
77
+ status: "pending",
78
+ createdAt: new Date().toISOString(),
79
+ };
80
+ commands.push(newCmd);
81
+ // Cap at 50 commands, evict oldest completed/errored first
82
+ if (commands.length > 50) {
83
+ const done = commands.filter((c) => c.status === "done" || c.status === "error");
84
+ if (done.length > 0) {
85
+ const removeId = done[0].id;
86
+ const idx = commands.findIndex((c) => c.id === removeId);
87
+ if (idx >= 0)
88
+ commands.splice(idx, 1);
89
+ }
90
+ else {
91
+ commands.shift();
92
+ }
93
+ }
94
+ writeObserverCommands(commands);
95
+ return id;
96
+ }
97
+ /** Get a command by ID. */
98
+ export function getObserverCommand(id) {
99
+ const commands = readObserverCommands();
100
+ return commands.find((c) => c.id === id) ?? null;
101
+ }
102
+ /** Get the latest OCR text from observer (if running and has data). */
103
+ export function getObserverOcrText() {
104
+ const state = readObserverState();
105
+ if (!state?.running || !state.lastFrame)
106
+ return null;
107
+ return state.lastFrame.ocrText;
108
+ }
109
+ /** Get detected popup from observer (if any). */
110
+ export function getObserverPopup() {
111
+ const state = readObserverState();
112
+ if (!state?.running)
113
+ return null;
114
+ return state.popup;
115
+ }
116
+ // ── Capture lock helpers ──
117
+ // Prevents observer daemon and perception coordinator from capturing simultaneously.
118
+ const LOCK_STALE_MS = 10_000; // Locks older than 10s are considered stale
119
+ /**
120
+ * Acquire the capture lock. Returns true if acquired, false if held by another process.
121
+ * Lock contains PID + timestamp so stale locks from crashed processes can be cleaned up.
122
+ */
123
+ export function acquireCaptureLock() {
124
+ fs.mkdirSync(OBSERVER_DIR, { recursive: true });
125
+ try {
126
+ // Check existing lock
127
+ const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
128
+ if (existing) {
129
+ const parts = existing.split(":");
130
+ const lockPid = Number(parts[0]);
131
+ const lockTime = Number(parts[1]);
132
+ // Stale lock check
133
+ if (Date.now() - lockTime > LOCK_STALE_MS) {
134
+ // Lock is stale — safe to overwrite
135
+ }
136
+ else if (lockPid !== process.pid) {
137
+ // Check if holding process is alive
138
+ try {
139
+ process.kill(lockPid, 0);
140
+ return false; // Process alive, lock is valid
141
+ }
142
+ catch {
143
+ // Process dead — stale lock
144
+ }
145
+ }
146
+ // Lock is ours or stale — fall through to acquire
147
+ }
148
+ }
149
+ catch {
150
+ // No lock file — safe to create
151
+ }
152
+ try {
153
+ fs.writeFileSync(CAPTURE_LOCK_FILE, `${process.pid}:${Date.now()}`);
154
+ return true;
155
+ }
156
+ catch {
157
+ return false;
158
+ }
159
+ }
160
+ /**
161
+ * Release the capture lock (only if we hold it).
162
+ */
163
+ export function releaseCaptureLock() {
164
+ try {
165
+ const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
166
+ const lockPid = Number(existing.split(":")[0]);
167
+ if (lockPid === process.pid) {
168
+ fs.unlinkSync(CAPTURE_LOCK_FILE);
169
+ }
170
+ }
171
+ catch {
172
+ // No lock file or already cleaned up
173
+ }
174
+ }
175
+ /**
176
+ * Check if the capture lock is currently held (by any process).
177
+ */
178
+ export function isCaptureLocked() {
179
+ try {
180
+ const existing = fs.readFileSync(CAPTURE_LOCK_FILE, "utf-8").trim();
181
+ if (!existing)
182
+ return false;
183
+ const parts = existing.split(":");
184
+ const lockPid = Number(parts[0]);
185
+ const lockTime = Number(parts[1]);
186
+ if (Date.now() - lockTime > LOCK_STALE_MS)
187
+ return false;
188
+ try {
189
+ process.kill(lockPid, 0);
190
+ return true;
191
+ }
192
+ catch {
193
+ return false; // Process dead
194
+ }
195
+ }
196
+ catch {
197
+ return false;
198
+ }
199
+ }
@@ -0,0 +1,43 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /**
4
+ * Observer types — background app-level visual monitoring
5
+ *
6
+ * The observer daemon watches a single app window via CGWindowListCreateImage,
7
+ * runs OCR only when pixels change, and exposes state via a JSON file.
8
+ * The playbook engine reads this file — zero overhead on the hot path.
9
+ */
10
+ import os from "node:os";
11
+ import path from "node:path";
12
+ /** Default popup patterns covering common OS and app dialogs */
13
+ export const DEFAULT_POPUP_PATTERNS = [
14
+ // Save dialogs
15
+ { pattern: "Do you want to save", action: "click_cancel", buttonText: "Don't Save" },
16
+ { pattern: "Save changes", action: "click_cancel", buttonText: "Don't Save" },
17
+ { pattern: "would you like to save", action: "click_cancel", buttonText: "Don't Save" },
18
+ // Permission dialogs
19
+ { pattern: "would like to access", action: "click_allow", buttonText: "Allow" },
20
+ { pattern: "wants to access", action: "click_allow", buttonText: "Allow" },
21
+ { pattern: "requesting permission", action: "click_allow", buttonText: "Allow" },
22
+ // Cookie banners
23
+ { pattern: "Accept all cookies", action: "click_ok", buttonText: "Accept" },
24
+ { pattern: "cookie preferences", action: "click_ok", buttonText: "Accept All" },
25
+ // Update prompts
26
+ { pattern: "update is available", action: "click_cancel", buttonText: "Later" },
27
+ { pattern: "Remind Me Later", action: "click_cancel", buttonText: "Remind Me Later" },
28
+ { pattern: "Update Now", action: "click_cancel", buttonText: "Not Now" },
29
+ // Generic modals
30
+ { pattern: "Are you sure", action: "click_ok", buttonText: "OK" },
31
+ { pattern: "Close without saving", action: "click_ok", buttonText: "Close" },
32
+ // Chrome specific
33
+ { pattern: "Chrome is being controlled", action: "press_escape" },
34
+ { pattern: "Restore pages", action: "press_escape" },
35
+ // macOS specific
36
+ { pattern: "allow notifications", action: "click_deny", buttonText: "Don't Allow" },
37
+ ];
38
+ export const OBSERVER_DIR = path.join(os.homedir(), ".screenhand", "observer");
39
+ export const OBSERVER_STATE_FILE = path.join(OBSERVER_DIR, "state.json");
40
+ export const OBSERVER_COMMANDS_FILE = path.join(OBSERVER_DIR, "commands.json");
41
+ export const OBSERVER_PID_FILE = path.join(OBSERVER_DIR, "observer.pid");
42
+ export const OBSERVER_LOG_FILE = path.join(OBSERVER_DIR, "observer.log");
43
+ export const CAPTURE_LOCK_FILE = path.join(OBSERVER_DIR, "capture.lock");