screenhand 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/README.md +193 -109
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +5876 -0
  4. package/dist/scripts/codex-monitor-daemon.js +335 -0
  5. package/dist/scripts/export-help-center.js +112 -0
  6. package/dist/scripts/marketing-loop.js +117 -0
  7. package/dist/scripts/observer-daemon.js +288 -0
  8. package/dist/scripts/orchestrator-daemon.js +399 -0
  9. package/dist/scripts/supervisor-daemon.js +272 -0
  10. package/dist/scripts/threads-campaign.js +208 -0
  11. package/dist/scripts/worker-daemon.js +228 -0
  12. package/dist/src/agent/cli.js +82 -0
  13. package/dist/src/agent/loop.js +274 -0
  14. package/dist/src/community/fetcher.js +109 -0
  15. package/dist/src/community/index.js +6 -0
  16. package/dist/src/community/publisher.js +191 -0
  17. package/dist/src/community/remote-api.js +121 -0
  18. package/dist/src/community/types.js +3 -0
  19. package/dist/src/community/validator.js +95 -0
  20. package/{src/config.ts → dist/src/config.js} +5 -10
  21. package/dist/src/context-tracker.js +489 -0
  22. package/{src/index.ts → dist/src/index.js} +32 -52
  23. package/dist/src/ingestion/coverage-auditor.js +233 -0
  24. package/dist/src/ingestion/doc-parser.js +164 -0
  25. package/dist/src/ingestion/index.js +8 -0
  26. package/dist/src/ingestion/menu-scanner.js +152 -0
  27. package/dist/src/ingestion/reference-merger.js +186 -0
  28. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  29. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  30. package/dist/src/ingestion/types.js +3 -0
  31. package/dist/src/jobs/manager.js +305 -0
  32. package/dist/src/jobs/runner.js +806 -0
  33. package/dist/src/jobs/store.js +102 -0
  34. package/dist/src/jobs/types.js +30 -0
  35. package/dist/src/jobs/worker.js +97 -0
  36. package/dist/src/learning/engine.js +356 -0
  37. package/dist/src/learning/index.js +9 -0
  38. package/dist/src/learning/locator-policy.js +120 -0
  39. package/dist/src/learning/pattern-policy.js +89 -0
  40. package/dist/src/learning/recovery-policy.js +116 -0
  41. package/dist/src/learning/sensor-policy.js +115 -0
  42. package/dist/src/learning/timing-model.js +204 -0
  43. package/dist/src/learning/topology-policy.js +90 -0
  44. package/dist/src/learning/types.js +9 -0
  45. package/dist/src/logging/timeline-logger.js +48 -0
  46. package/dist/src/mcp/mcp-stdio-server.js +464 -0
  47. package/dist/src/mcp/server.js +363 -0
  48. package/dist/src/mcp-entry.js +60 -0
  49. package/dist/src/memory/playbook-seeds.js +200 -0
  50. package/dist/src/memory/recall.js +222 -0
  51. package/dist/src/memory/research.js +104 -0
  52. package/dist/src/memory/seeds.js +101 -0
  53. package/dist/src/memory/service.js +446 -0
  54. package/dist/src/memory/session.js +169 -0
  55. package/dist/src/memory/store.js +451 -0
  56. package/{src/runtime/locator-cache.ts → dist/src/memory/types.js} +1 -17
  57. package/dist/src/monitor/codex-monitor.js +382 -0
  58. package/dist/src/monitor/task-queue.js +97 -0
  59. package/dist/src/monitor/types.js +62 -0
  60. package/dist/src/native/bridge-client.js +412 -0
  61. package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
  62. package/dist/src/observer/state.js +199 -0
  63. package/dist/src/observer/types.js +43 -0
  64. package/dist/src/orchestrator/state.js +68 -0
  65. package/dist/src/orchestrator/types.js +22 -0
  66. package/dist/src/perception/ax-source.js +162 -0
  67. package/dist/src/perception/cdp-source.js +162 -0
  68. package/dist/src/perception/coordinator.js +771 -0
  69. package/dist/src/perception/frame-differ.js +287 -0
  70. package/dist/src/perception/index.js +22 -0
  71. package/dist/src/perception/manager.js +199 -0
  72. package/dist/src/perception/types.js +47 -0
  73. package/dist/src/perception/vision-source.js +399 -0
  74. package/dist/src/planner/deterministic.js +298 -0
  75. package/dist/src/planner/executor.js +870 -0
  76. package/dist/src/planner/goal-store.js +92 -0
  77. package/dist/src/planner/index.js +21 -0
  78. package/dist/src/planner/planner.js +520 -0
  79. package/dist/src/planner/tool-registry.js +71 -0
  80. package/dist/src/planner/types.js +22 -0
  81. package/dist/src/platform/explorer.js +213 -0
  82. package/dist/src/platform/help-center-markdown.js +527 -0
  83. package/dist/src/platform/learner.js +257 -0
  84. package/dist/src/playbook/engine.js +486 -0
  85. package/dist/src/playbook/index.js +20 -0
  86. package/dist/src/playbook/mcp-recorder.js +204 -0
  87. package/dist/src/playbook/recorder.js +536 -0
  88. package/dist/src/playbook/runner.js +408 -0
  89. package/dist/src/playbook/store.js +312 -0
  90. package/dist/src/playbook/types.js +17 -0
  91. package/dist/src/recovery/detectors.js +156 -0
  92. package/dist/src/recovery/engine.js +327 -0
  93. package/dist/src/recovery/index.js +20 -0
  94. package/dist/src/recovery/strategies.js +274 -0
  95. package/dist/src/recovery/types.js +20 -0
  96. package/dist/src/runtime/accessibility-adapter.js +430 -0
  97. package/dist/src/runtime/app-adapter.js +64 -0
  98. package/dist/src/runtime/applescript-adapter.js +305 -0
  99. package/dist/src/runtime/ax-role-map.js +96 -0
  100. package/dist/src/runtime/browser-adapter.js +52 -0
  101. package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
  102. package/dist/src/runtime/composite-adapter.js +221 -0
  103. package/dist/src/runtime/execution-contract.js +159 -0
  104. package/dist/src/runtime/executor.js +286 -0
  105. package/dist/src/runtime/locator-cache.js +50 -0
  106. package/dist/src/runtime/planning-loop.js +63 -0
  107. package/dist/src/runtime/service.js +432 -0
  108. package/dist/src/runtime/session-manager.js +63 -0
  109. package/dist/src/runtime/state-observer.js +121 -0
  110. package/dist/src/runtime/vision-adapter.js +225 -0
  111. package/dist/src/state/app-map-types.js +72 -0
  112. package/dist/src/state/app-map.js +1974 -0
  113. package/dist/src/state/entity-tracker.js +108 -0
  114. package/dist/src/state/fusion.js +96 -0
  115. package/dist/src/state/index.js +21 -0
  116. package/dist/src/state/ladder-generator.js +236 -0
  117. package/dist/src/state/persistence.js +156 -0
  118. package/dist/src/state/types.js +17 -0
  119. package/dist/src/state/world-model.js +1456 -0
  120. package/dist/src/supervisor/locks.js +186 -0
  121. package/dist/src/supervisor/supervisor.js +403 -0
  122. package/dist/src/supervisor/types.js +30 -0
  123. package/dist/src/test-mcp-protocol.js +154 -0
  124. package/dist/src/types.js +17 -0
  125. package/dist/src/util/atomic-write.js +133 -0
  126. package/dist/src/util/sanitize.js +146 -0
  127. package/dist-app-maps/com.figma.Desktop.json +959 -0
  128. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  129. package/dist-app-maps/notion.id.json +2831 -0
  130. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  131. package/dist-playbooks/codex-desktop.json +76 -0
  132. package/dist-playbooks/competitor-research-stack.json +122 -0
  133. package/dist-playbooks/davinci-color-grade.json +153 -0
  134. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  135. package/dist-playbooks/davinci-render.json +114 -0
  136. package/dist-playbooks/devto.json +52 -0
  137. package/dist-playbooks/discord.json +41 -0
  138. package/dist-playbooks/google-flow-create-project.json +59 -0
  139. package/dist-playbooks/google-flow-edit-image.json +90 -0
  140. package/dist-playbooks/google-flow-edit-video.json +90 -0
  141. package/dist-playbooks/google-flow-generate-image.json +68 -0
  142. package/dist-playbooks/google-flow-generate-video.json +191 -0
  143. package/dist-playbooks/google-flow-open-project.json +48 -0
  144. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  145. package/dist-playbooks/google-flow-search-assets.json +64 -0
  146. package/dist-playbooks/instagram.json +57 -0
  147. package/dist-playbooks/linkedin.json +52 -0
  148. package/dist-playbooks/n8n.json +43 -0
  149. package/dist-playbooks/reddit.json +52 -0
  150. package/dist-playbooks/threads.json +59 -0
  151. package/dist-playbooks/x-twitter.json +59 -0
  152. package/dist-playbooks/youtube.json +59 -0
  153. package/dist-references/canva.json +646 -0
  154. package/dist-references/codex-desktop.json +305 -0
  155. package/dist-references/davinci-resolve-keyboard.json +594 -0
  156. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  157. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  158. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  159. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  160. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  161. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  162. package/dist-references/devto.json +317 -0
  163. package/dist-references/discord.json +549 -0
  164. package/dist-references/figma.json +1186 -0
  165. package/dist-references/finder.json +146 -0
  166. package/dist-references/google-ads-transparency.json +95 -0
  167. package/dist-references/google-flow.json +649 -0
  168. package/dist-references/instagram.json +341 -0
  169. package/dist-references/linkedin.json +324 -0
  170. package/dist-references/meta-ad-library.json +86 -0
  171. package/dist-references/n8n.json +387 -0
  172. package/dist-references/notes.json +27 -0
  173. package/dist-references/notion.json +163 -0
  174. package/dist-references/reddit.json +341 -0
  175. package/dist-references/threads.json +337 -0
  176. package/dist-references/x-twitter.json +403 -0
  177. package/dist-references/youtube.json +373 -0
  178. package/native/macos-bridge/Package.swift +1 -0
  179. package/native/macos-bridge/Sources/AccessibilityBridge.swift +257 -36
  180. package/native/macos-bridge/Sources/AppManagement.swift +212 -2
  181. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +348 -53
  182. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  183. package/native/macos-bridge/Sources/VisionBridge.swift +165 -7
  184. package/native/macos-bridge/Sources/main.swift +169 -16
  185. package/native/windows-bridge/Program.cs +5 -0
  186. package/native/windows-bridge/ScreenCapture.cs +124 -0
  187. package/package.json +29 -4
  188. package/scripts/postinstall.cjs +127 -0
  189. package/.claude/commands/automate.md +0 -28
  190. package/.claude/commands/debug-ui.md +0 -19
  191. package/.claude/commands/screenshot.md +0 -15
  192. package/.github/FUNDING.yml +0 -1
  193. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
  194. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  195. package/.mcp.json +0 -8
  196. package/DESKTOP_MCP_GUIDE.md +0 -92
  197. package/SECURITY.md +0 -44
  198. package/docs/architecture.md +0 -47
  199. package/install-skills.sh +0 -19
  200. package/mcp-bridge.ts +0 -271
  201. package/mcp-desktop.ts +0 -1221
  202. package/playbooks/instagram.json +0 -41
  203. package/playbooks/instagram_v2.json +0 -201
  204. package/playbooks/x_v1.json +0 -211
  205. package/scripts/devpost-live-loop.mjs +0 -421
  206. package/src/logging/timeline-logger.ts +0 -55
  207. package/src/mcp/server.ts +0 -449
  208. package/src/memory/recall.ts +0 -191
  209. package/src/memory/research.ts +0 -146
  210. package/src/memory/seeds.ts +0 -123
  211. package/src/memory/session.ts +0 -201
  212. package/src/memory/store.ts +0 -434
  213. package/src/memory/types.ts +0 -69
  214. package/src/native/bridge-client.ts +0 -239
  215. package/src/runtime/accessibility-adapter.ts +0 -487
  216. package/src/runtime/app-adapter.ts +0 -169
  217. package/src/runtime/applescript-adapter.ts +0 -376
  218. package/src/runtime/ax-role-map.ts +0 -102
  219. package/src/runtime/browser-adapter.ts +0 -129
  220. package/src/runtime/cdp-chrome-adapter.ts +0 -676
  221. package/src/runtime/composite-adapter.ts +0 -274
  222. package/src/runtime/executor.ts +0 -396
  223. package/src/runtime/planning-loop.ts +0 -81
  224. package/src/runtime/service.ts +0 -448
  225. package/src/runtime/session-manager.ts +0 -50
  226. package/src/runtime/state-observer.ts +0 -136
  227. package/src/runtime/vision-adapter.ts +0 -297
  228. package/src/types.ts +0 -297
  229. package/tests/bridge-client.test.ts +0 -176
  230. package/tests/browser-stealth.test.ts +0 -210
  231. package/tests/composite-adapter.test.ts +0 -64
  232. package/tests/mcp-server.test.ts +0 -151
  233. package/tests/memory-recall.test.ts +0 -339
  234. package/tests/memory-research.test.ts +0 -159
  235. package/tests/memory-seeds.test.ts +0 -120
  236. package/tests/memory-store.test.ts +0 -392
  237. package/tests/types.test.ts +0 -92
  238. package/tsconfig.check.json +0 -17
  239. package/tsconfig.json +0 -19
  240. package/vitest.config.ts +0 -8
  241. /package/{playbooks → dist-references}/devpost.json +0 -0
@@ -0,0 +1,288 @@
1
+ #!/usr/bin/env npx tsx
2
+ /**
3
+ * Observer Daemon — background app-level visual monitor.
4
+ *
5
+ * Captures a single app window via cg.captureWindow (CGWindowListCreateImage).
6
+ * Uses pixel-hash frame diff to skip OCR when nothing changed.
7
+ * Persists state to ~/.screenhand/observer/state.json for the engine to read.
8
+ *
9
+ * Zero overhead on the main execution path — engine reads a JSON file, daemon
10
+ * does the heavy lifting in a separate process.
11
+ *
12
+ * Usage:
13
+ * npx tsx scripts/observer-daemon.ts --bundleId com.blackmagic-design.DaVinciResolve --windowId 1234
14
+ * npx tsx scripts/observer-daemon.ts --bundleId com.blackmagic-design.DaVinciResolve --windowId 1234 --interval 2000
15
+ *
16
+ * State files:
17
+ * ~/.screenhand/observer/state.json — observer state (latest OCR, popup detection)
18
+ * ~/.screenhand/observer/observer.pid — PID of this process
19
+ * ~/.screenhand/observer/observer.log — log output
20
+ */
21
+ import path from "node:path";
22
+ import fs from "node:fs";
23
+ import crypto from "node:crypto";
24
+ import { BridgeClient } from "../src/native/bridge-client.js";
25
+ import { writeObserverState, readObserverCommands, writeObserverCommands, acquireCaptureLock, releaseCaptureLock } from "../src/observer/state.js";
26
+ import { detectPopup } from "../src/observer/state.js";
27
+ import { OBSERVER_DIR, OBSERVER_PID_FILE, OBSERVER_LOG_FILE } from "../src/observer/types.js";
28
+ // ── Config from CLI args ──
29
+ const args = process.argv.slice(2);
30
+ function getArg(name, fallback) {
31
+ const idx = args.indexOf("--" + name);
32
+ if (idx === -1)
33
+ return fallback;
34
+ return args[idx + 1] ?? fallback;
35
+ }
36
+ const BUNDLE_ID = getArg("bundleId");
37
+ const WINDOW_ID = Number(getArg("windowId", "0"));
38
+ const INTERVAL_MS = Number(getArg("interval", "2000"));
39
+ if (!BUNDLE_ID || !WINDOW_ID) {
40
+ process.stderr.write("Usage: observer-daemon.ts --bundleId <id> --windowId <id> [--interval <ms>]\n");
41
+ process.exit(1);
42
+ }
43
+ // ── Logging ──
44
+ fs.mkdirSync(OBSERVER_DIR, { recursive: true });
45
+ const logStream = fs.createWriteStream(OBSERVER_LOG_FILE, { flags: "a" });
46
+ let daemonized = false;
47
+ function log(msg) {
48
+ const line = `[${new Date().toISOString()}] ${msg}`;
49
+ logStream.write(line + "\n");
50
+ if (!daemonized)
51
+ process.stderr.write(line + "\n");
52
+ }
53
+ // ── Bridge setup ──
54
+ const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
55
+ const projectRoot = scriptDir.includes("/dist/")
56
+ ? path.resolve(scriptDir, "../..")
57
+ : path.resolve(scriptDir, "..");
58
+ const bridgePath = process.platform === "win32"
59
+ ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
60
+ : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
61
+ const bridge = new BridgeClient(bridgePath);
62
+ let bridgeReady = false;
63
+ async function ensureBridge() {
64
+ if (!bridgeReady) {
65
+ await bridge.start();
66
+ bridgeReady = true;
67
+ }
68
+ }
69
+ // ── Frame diff via file hash ──
70
+ let lastFrameHash = null;
71
+ function hashFile(filePath) {
72
+ const data = fs.readFileSync(filePath);
73
+ return crypto.createHash("md5").update(data).digest("hex");
74
+ }
75
+ // ── State ──
76
+ let stopped = false;
77
+ let framesCaptured = 0;
78
+ let framesChanged = 0;
79
+ let ocrRuns = 0;
80
+ let lastFrame = null;
81
+ let lastPopup = null;
82
+ let lastError = null;
83
+ const startedAt = new Date().toISOString();
84
+ function buildState() {
85
+ return {
86
+ pid: process.pid,
87
+ running: !stopped,
88
+ startedAt,
89
+ bundleId: BUNDLE_ID,
90
+ windowId: WINDOW_ID,
91
+ intervalMs: INTERVAL_MS,
92
+ framesCaptured,
93
+ framesChanged,
94
+ ocrRuns,
95
+ lastFrame,
96
+ popup: lastPopup,
97
+ lastError,
98
+ };
99
+ }
100
+ function persistState() {
101
+ try {
102
+ writeObserverState(buildState());
103
+ }
104
+ catch {
105
+ // Non-fatal
106
+ }
107
+ }
108
+ // ── Capture loop ──
109
+ async function captureFrame() {
110
+ // Acquire capture lock to prevent concurrent captures with perception coordinator
111
+ if (!acquireCaptureLock()) {
112
+ log("Skipping capture — lock held by perception coordinator");
113
+ return;
114
+ }
115
+ try {
116
+ await ensureBridge();
117
+ // 1. Capture window (app-level, not full screen)
118
+ let shot;
119
+ try {
120
+ shot = await bridge.call("cg.captureWindow", { windowId: WINDOW_ID });
121
+ }
122
+ catch (err) {
123
+ lastError = `Capture failed: ${err instanceof Error ? err.message : String(err)}`;
124
+ return;
125
+ }
126
+ framesCaptured++;
127
+ // 2. Frame diff — hash the image file, skip OCR if identical
128
+ const currentHash = hashFile(shot.path);
129
+ const pixelsChanged = currentHash !== lastFrameHash;
130
+ lastFrameHash = currentHash;
131
+ if (!pixelsChanged) {
132
+ // Frame identical — update timestamp only, skip expensive OCR
133
+ if (lastFrame) {
134
+ lastFrame.capturedAt = new Date().toISOString();
135
+ lastFrame.changed = false;
136
+ }
137
+ return;
138
+ }
139
+ framesChanged++;
140
+ // 3. OCR only on changed frames
141
+ let ocrText = "";
142
+ try {
143
+ const ocr = await bridge.call("vision.ocr", {
144
+ imagePath: shot.path,
145
+ });
146
+ ocrText = ocr.text;
147
+ ocrRuns++;
148
+ }
149
+ catch (err) {
150
+ lastError = `OCR failed: ${err instanceof Error ? err.message : String(err)}`;
151
+ ocrText = lastFrame?.ocrText ?? "";
152
+ }
153
+ // 4. Update frame
154
+ lastFrame = {
155
+ capturedAt: new Date().toISOString(),
156
+ ocrText,
157
+ changed: true,
158
+ };
159
+ // 5. Popup detection on the new OCR text
160
+ lastPopup = detectPopup(ocrText);
161
+ if (lastPopup) {
162
+ log(`Popup detected: "${lastPopup.pattern}" → ${lastPopup.dismissAction}`);
163
+ }
164
+ lastError = null;
165
+ // Clean up temp screenshot
166
+ try {
167
+ fs.unlinkSync(shot.path);
168
+ }
169
+ catch { /* ignore */ }
170
+ }
171
+ finally {
172
+ releaseCaptureLock();
173
+ }
174
+ }
175
+ // ── Command processing ──
176
+ async function processCommands() {
177
+ let commands;
178
+ try {
179
+ commands = readObserverCommands();
180
+ }
181
+ catch {
182
+ return; // No commands file or corrupt — skip
183
+ }
184
+ const pending = commands.filter((c) => c.status === "pending");
185
+ if (pending.length === 0)
186
+ return;
187
+ let changed = false;
188
+ for (const cmd of pending) {
189
+ if (cmd.type !== "ocr_roi") {
190
+ cmd.status = "error";
191
+ cmd.error = `Unknown command type: ${cmd.type}`;
192
+ changed = true;
193
+ continue;
194
+ }
195
+ cmd.status = "running";
196
+ changed = true;
197
+ try {
198
+ await ensureBridge();
199
+ const targetWindowId = cmd.windowId ?? WINDOW_ID;
200
+ // Use vision.ocrRegion for targeted ROI OCR
201
+ const result = await bridge.call("vision.ocrRegion", {
202
+ windowId: targetWindowId,
203
+ region: cmd.roi,
204
+ });
205
+ cmd.status = "done";
206
+ cmd.result = {
207
+ text: result.text ?? "",
208
+ regions: result.regions ?? [],
209
+ completedAt: new Date().toISOString(),
210
+ };
211
+ ocrRuns++;
212
+ log(`Command ${cmd.id}: OCR ROI completed (${cmd.result.regions.length} regions)`);
213
+ }
214
+ catch (err) {
215
+ cmd.status = "error";
216
+ cmd.error = err instanceof Error ? err.message : String(err);
217
+ log(`Command ${cmd.id}: failed — ${cmd.error}`);
218
+ }
219
+ }
220
+ if (changed) {
221
+ writeObserverCommands(commands);
222
+ }
223
+ }
224
+ // ── Main loop ──
225
+ async function main() {
226
+ // Enforce single daemon
227
+ try {
228
+ const existingPid = fs.readFileSync(OBSERVER_PID_FILE, "utf-8").trim();
229
+ const pid = Number(existingPid);
230
+ if (!Number.isNaN(pid) && pid !== process.pid) {
231
+ try {
232
+ process.kill(pid, 0); // Check if alive
233
+ log(`Another observer daemon already running (pid=${pid}). Aborting.`);
234
+ process.exit(1);
235
+ }
236
+ catch {
237
+ // Stale PID — safe to continue
238
+ }
239
+ }
240
+ }
241
+ catch {
242
+ // No PID file — first run
243
+ }
244
+ fs.writeFileSync(OBSERVER_PID_FILE, String(process.pid));
245
+ daemonized = true;
246
+ log(`Observer daemon started (pid=${process.pid})`);
247
+ log(`Watching: bundleId=${BUNDLE_ID} windowId=${WINDOW_ID} interval=${INTERVAL_MS}ms`);
248
+ persistState();
249
+ while (!stopped) {
250
+ try {
251
+ await captureFrame();
252
+ await processCommands();
253
+ persistState();
254
+ }
255
+ catch (err) {
256
+ lastError = `Frame error: ${err instanceof Error ? err.message : String(err)}`;
257
+ log(lastError);
258
+ }
259
+ await sleep(INTERVAL_MS);
260
+ }
261
+ }
262
+ function sleep(ms) {
263
+ return new Promise((resolve) => setTimeout(resolve, ms));
264
+ }
265
+ // ── Graceful shutdown ──
266
+ process.on("SIGINT", shutdown);
267
+ process.on("SIGTERM", shutdown);
268
+ async function shutdown() {
269
+ if (stopped)
270
+ return;
271
+ stopped = true;
272
+ log("Shutting down...");
273
+ persistState();
274
+ try {
275
+ fs.unlinkSync(OBSERVER_PID_FILE);
276
+ }
277
+ catch { /* ignore */ }
278
+ try {
279
+ await bridge.stop();
280
+ }
281
+ catch { /* ignore */ }
282
+ logStream.end();
283
+ process.exit(0);
284
+ }
285
+ main().catch((err) => {
286
+ log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
287
+ process.exit(1);
288
+ });
@@ -0,0 +1,399 @@
1
+ #!/usr/bin/env npx tsx
2
+ /**
3
+ * Orchestrator Daemon — multi-agent task router and coordinator.
4
+ *
5
+ * Manages a pool of worker slots that process tasks in parallel:
6
+ * - Web slots (CDP-only): truly parallel, no mouse/keyboard conflict
7
+ * - Native slots: serialized per-app via lease locks
8
+ *
9
+ * Each worker slot runs a JobRunner independently. The orchestrator
10
+ * routes tasks to the right slot type and manages coordination.
11
+ *
12
+ * Usage:
13
+ * npx tsx scripts/orchestrator-daemon.ts
14
+ * npx tsx scripts/orchestrator-daemon.ts --web-slots 4 --native-slots 1 --poll 1000
15
+ *
16
+ * State files:
17
+ * ~/.screenhand/orchestrator/state.json — orchestrator state
18
+ * ~/.screenhand/orchestrator/orchestrator.pid — PID of this process
19
+ * ~/.screenhand/orchestrator/orchestrator.log — log output
20
+ */
21
+ import path from "node:path";
22
+ import fs from "node:fs";
23
+ import os from "node:os";
24
+ import { BridgeClient } from "../src/native/bridge-client.js";
25
+ import { SessionSupervisor, LeaseManager } from "../src/supervisor/supervisor.js";
26
+ import { JobManager } from "../src/jobs/manager.js";
27
+ import { JobRunner } from "../src/jobs/runner.js";
28
+ import { PlaybookEngine } from "../src/playbook/engine.js";
29
+ import { PlaybookStore } from "../src/playbook/store.js";
30
+ import { AccessibilityAdapter } from "../src/runtime/accessibility-adapter.js";
31
+ import { AutomationRuntimeService } from "../src/runtime/service.js";
32
+ import { TimelineLogger } from "../src/logging/timeline-logger.js";
33
+ import { MemoryService } from "../src/memory/service.js";
34
+ import { writeOrchestratorState, getOrchestratorDaemonPid } from "../src/orchestrator/state.js";
35
+ import { ORCHESTRATOR_DIR, ORCHESTRATOR_PID_FILE, ORCHESTRATOR_LOG_FILE } from "../src/orchestrator/types.js";
36
+ // ── Config from CLI args ──
37
+ const args = process.argv.slice(2);
38
+ function getArg(name, fallback) {
39
+ const idx = args.indexOf("--" + name);
40
+ if (idx === -1)
41
+ return fallback;
42
+ return args[idx + 1] ?? fallback;
43
+ }
44
+ const WEB_SLOTS = Number(getArg("web-slots", "4"));
45
+ const NATIVE_SLOTS = Number(getArg("native-slots", "1"));
46
+ const POLL_MS = Number(getArg("poll", "1000"));
47
+ // ── Directories ──
48
+ const JOB_DIR = path.join(os.homedir(), ".screenhand", "jobs");
49
+ const LOCK_DIR = path.join(os.homedir(), ".screenhand", "locks");
50
+ const PLAYBOOKS_DIR = path.join(os.homedir(), ".screenhand", "playbooks");
51
+ const SUPERVISOR_STATE_DIR = path.join(os.homedir(), ".screenhand", "supervisor");
52
+ fs.mkdirSync(ORCHESTRATOR_DIR, { recursive: true });
53
+ fs.mkdirSync(JOB_DIR, { recursive: true });
54
+ // ── Logging ──
55
+ const logStream = fs.createWriteStream(ORCHESTRATOR_LOG_FILE, { flags: "a" });
56
+ let daemonized = false;
57
+ function log(msg) {
58
+ const line = `[${new Date().toISOString()}] ${msg}`;
59
+ logStream.write(line + "\n");
60
+ if (!daemonized)
61
+ process.stderr.write(line + "\n");
62
+ }
63
+ // ── Bridge setup ──
64
+ const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
65
+ const projectRoot = scriptDir.includes("/dist/")
66
+ ? path.resolve(scriptDir, "../..")
67
+ : path.resolve(scriptDir, "..");
68
+ const bridgePath = process.platform === "win32"
69
+ ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
70
+ : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
71
+ // Each worker slot gets its own bridge for true parallelism
72
+ function createBridge() {
73
+ return new BridgeClient(bridgePath);
74
+ }
75
+ // ── Shared services ──
76
+ const leaseManager = new LeaseManager(LOCK_DIR);
77
+ const supervisor = new SessionSupervisor({
78
+ stateDir: SUPERVISOR_STATE_DIR,
79
+ lockDir: LOCK_DIR,
80
+ });
81
+ const memory = new MemoryService(os.homedir());
82
+ const jobManager = new JobManager({ jobDir: JOB_DIR, memory, supervisor });
83
+ jobManager.init();
84
+ const playbookStore = new PlaybookStore(PLAYBOOKS_DIR);
85
+ // ── State ──
86
+ let stopped = false;
87
+ const startedAt = new Date().toISOString();
88
+ let totalSubmitted = 0;
89
+ let totalCompleted = 0;
90
+ let totalFailed = 0;
91
+ // Task queue — loaded from / persisted to state.json
92
+ let taskQueue = [];
93
+ // Worker slots
94
+ const workers = [];
95
+ const workerRunners = new Map();
96
+ // App locks for native tasks — bundleId → worker slot ID
97
+ const nativeLocks = new Map();
98
+ // ── Worker slot initialization ──
99
+ async function initWorkerSlots() {
100
+ let slotId = 0;
101
+ // Web slots
102
+ for (let i = 0; i < WEB_SLOTS; i++) {
103
+ const slot = {
104
+ id: slotId,
105
+ type: "web",
106
+ busy: false,
107
+ tasksCompleted: 0,
108
+ tasksFailed: 0,
109
+ };
110
+ workers.push(slot);
111
+ const bridge = createBridge();
112
+ await bridge.start();
113
+ const adapter = new AccessibilityAdapter(bridge);
114
+ const logger = new TimelineLogger();
115
+ const runtimeService = new AutomationRuntimeService(adapter, logger);
116
+ const playbookEngine = new PlaybookEngine(runtimeService);
117
+ const runner = new JobRunner(bridge, jobManager, leaseManager, supervisor, {
118
+ playbookEngine,
119
+ playbookStore,
120
+ runtimeService,
121
+ onLog: (msg) => log(`[W${slotId}] ${msg}`),
122
+ });
123
+ workerRunners.set(slotId, { runner, bridge, busy: false });
124
+ slotId++;
125
+ }
126
+ // Native slots
127
+ for (let i = 0; i < NATIVE_SLOTS; i++) {
128
+ const slot = {
129
+ id: slotId,
130
+ type: "native",
131
+ busy: false,
132
+ tasksCompleted: 0,
133
+ tasksFailed: 0,
134
+ };
135
+ workers.push(slot);
136
+ const bridge = createBridge();
137
+ await bridge.start();
138
+ const adapter = new AccessibilityAdapter(bridge);
139
+ const logger = new TimelineLogger();
140
+ const runtimeService = new AutomationRuntimeService(adapter, logger);
141
+ const playbookEngine = new PlaybookEngine(runtimeService);
142
+ const runner = new JobRunner(bridge, jobManager, leaseManager, supervisor, {
143
+ playbookEngine,
144
+ playbookStore,
145
+ runtimeService,
146
+ onLog: (msg) => log(`[W${slotId}] ${msg}`),
147
+ });
148
+ workerRunners.set(slotId, { runner, bridge, busy: false });
149
+ slotId++;
150
+ }
151
+ log(`Initialized ${WEB_SLOTS} web slots + ${NATIVE_SLOTS} native slots = ${workers.length} total`);
152
+ }
153
+ // ── Task routing ──
154
+ function findAvailableSlot(task) {
155
+ if (task.mode === "web") {
156
+ // Any free web slot
157
+ return workers.find(w => w.type === "web" && !w.busy) ?? null;
158
+ }
159
+ if (task.mode === "native" || task.mode === "mixed") {
160
+ // Native tasks need a free native slot AND the app must not be locked by another slot
161
+ const bundleId = task.bundleId ?? "unknown";
162
+ const lockHolder = nativeLocks.get(bundleId);
163
+ if (lockHolder !== undefined) {
164
+ // App is locked — only the lock holder can work on it
165
+ const slot = workers.find(w => w.id === lockHolder && !w.busy);
166
+ return slot ?? null;
167
+ }
168
+ // No lock — find any free native slot
169
+ return workers.find(w => w.type === "native" && !w.busy) ?? null;
170
+ }
171
+ return null;
172
+ }
173
+ // ── Task → Job conversion ──
174
+ function taskToJobParams(task) {
175
+ return {
176
+ task: task.task,
177
+ ...(task.playbookId !== undefined ? { playbookId: task.playbookId } : {}),
178
+ ...(task.bundleId !== undefined ? { bundleId: task.bundleId } : {}),
179
+ ...(task.windowId !== undefined ? { windowId: task.windowId } : {}),
180
+ ...(task.vars ? { vars: task.vars } : {}),
181
+ priority: task.priority,
182
+ tags: ["orchestrator", `task_${task.id}`],
183
+ };
184
+ }
185
+ // ── Task dispatch ──
186
+ async function dispatchTask(task, slot) {
187
+ const worker = workerRunners.get(slot.id);
188
+ if (!worker)
189
+ return;
190
+ // Mark slot as busy
191
+ slot.busy = true;
192
+ slot.currentTaskId = task.id;
193
+ worker.busy = true;
194
+ // Lock native app
195
+ if ((task.mode === "native" || task.mode === "mixed") && task.bundleId) {
196
+ nativeLocks.set(task.bundleId, slot.id);
197
+ }
198
+ // Update task status
199
+ task.status = "assigned";
200
+ task.assignedWorker = slot.id;
201
+ task.startedAt = new Date().toISOString();
202
+ log(`Dispatching task ${task.id} ("${task.task.slice(0, 50)}") to slot ${slot.id} (${slot.type})`);
203
+ // Create a job for this task
204
+ const jobParams = taskToJobParams(task);
205
+ const job = jobManager.create(jobParams);
206
+ task.jobId = job.id;
207
+ task.status = "running";
208
+ persistState();
209
+ // Run the job asynchronously
210
+ try {
211
+ const result = await worker.runner.run();
212
+ if (result) {
213
+ task.result = `${result.finalState}: ${result.stepsCompleted}/${result.totalSteps} steps in ${result.durationMs}ms`;
214
+ if (result.finalState === "done") {
215
+ task.status = "done";
216
+ slot.tasksCompleted++;
217
+ totalCompleted++;
218
+ log(`Task ${task.id} completed (${result.durationMs}ms)`);
219
+ }
220
+ else if (result.finalState === "blocked" || result.finalState === "waiting_human") {
221
+ task.status = "blocked";
222
+ task.error = result.error ?? "Blocked";
223
+ log(`Task ${task.id} blocked: ${result.error ?? "unknown"}`);
224
+ }
225
+ else {
226
+ task.status = "failed";
227
+ task.error = result.error ?? "Failed";
228
+ slot.tasksFailed++;
229
+ totalFailed++;
230
+ log(`Task ${task.id} failed: ${result.error ?? "unknown"}`);
231
+ }
232
+ }
233
+ else {
234
+ // No job was dequeued — it may have been picked up already
235
+ task.status = "failed";
236
+ task.error = "No job to dequeue";
237
+ slot.tasksFailed++;
238
+ totalFailed++;
239
+ }
240
+ }
241
+ catch (err) {
242
+ task.status = "failed";
243
+ task.error = err instanceof Error ? err.message : String(err);
244
+ slot.tasksFailed++;
245
+ totalFailed++;
246
+ log(`Task ${task.id} error: ${task.error}`);
247
+ }
248
+ // Release slot
249
+ task.completedAt = new Date().toISOString();
250
+ slot.busy = false;
251
+ delete slot.currentTaskId;
252
+ worker.busy = false;
253
+ // Release native app lock
254
+ if ((task.mode === "native" || task.mode === "mixed") && task.bundleId) {
255
+ nativeLocks.delete(task.bundleId);
256
+ }
257
+ persistState();
258
+ }
259
+ // ── State persistence ──
260
+ function buildState() {
261
+ const nativeLocksObj = {};
262
+ for (const [k, v] of nativeLocks)
263
+ nativeLocksObj[k] = v;
264
+ return {
265
+ pid: process.pid,
266
+ running: !stopped,
267
+ startedAt,
268
+ workers: [...workers],
269
+ webSlots: WEB_SLOTS,
270
+ nativeSlots: NATIVE_SLOTS,
271
+ tasks: taskQueue,
272
+ totalSubmitted,
273
+ totalCompleted,
274
+ totalFailed,
275
+ nativeLocks: nativeLocksObj,
276
+ };
277
+ }
278
+ function persistState() {
279
+ try {
280
+ writeOrchestratorState(buildState());
281
+ }
282
+ catch {
283
+ // Non-fatal
284
+ }
285
+ }
286
+ // ── Load tasks from state (resume after restart) ──
287
+ function loadState() {
288
+ try {
289
+ const state = readExistingState();
290
+ if (state && state.tasks) {
291
+ // Resume queued/running tasks
292
+ for (const task of state.tasks) {
293
+ if (task.status === "queued" || task.status === "assigned" || task.status === "running") {
294
+ task.status = "queued"; // Re-queue interrupted tasks
295
+ delete task.assignedWorker;
296
+ taskQueue.push(task);
297
+ }
298
+ }
299
+ totalSubmitted = state.totalSubmitted ?? 0;
300
+ totalCompleted = state.totalCompleted ?? 0;
301
+ totalFailed = state.totalFailed ?? 0;
302
+ if (taskQueue.length > 0) {
303
+ log(`Resumed ${taskQueue.length} tasks from previous state`);
304
+ }
305
+ }
306
+ }
307
+ catch {
308
+ // Fresh start
309
+ }
310
+ }
311
+ function readExistingState() {
312
+ try {
313
+ const data = fs.readFileSync(path.join(ORCHESTRATOR_DIR, "state.json"), "utf-8");
314
+ return JSON.parse(data);
315
+ }
316
+ catch {
317
+ return null;
318
+ }
319
+ }
320
+ // ── Main poll loop ──
321
+ async function poll() {
322
+ // Also reload playbooks periodically
323
+ playbookStore.load();
324
+ // Sort queue by priority (lower = higher priority)
325
+ const queued = taskQueue
326
+ .filter(t => t.status === "queued")
327
+ .sort((a, b) => a.priority - b.priority);
328
+ // Dispatch tasks to available slots
329
+ const dispatches = [];
330
+ for (const task of queued) {
331
+ const slot = findAvailableSlot(task);
332
+ if (slot) {
333
+ // Dispatch in parallel — don't await here
334
+ dispatches.push(dispatchTask(task, slot));
335
+ }
336
+ }
337
+ // Wait for all dispatched tasks in this cycle
338
+ if (dispatches.length > 0) {
339
+ await Promise.allSettled(dispatches);
340
+ }
341
+ }
342
+ async function main() {
343
+ // Enforce single daemon
344
+ const existingPid = getOrchestratorDaemonPid();
345
+ if (existingPid !== null && existingPid !== process.pid) {
346
+ log(`Another orchestrator daemon already running (pid=${existingPid}). Aborting.`);
347
+ process.exit(1);
348
+ }
349
+ fs.writeFileSync(ORCHESTRATOR_PID_FILE, String(process.pid));
350
+ daemonized = true;
351
+ log(`Orchestrator daemon started (pid=${process.pid})`);
352
+ log(`Config: web-slots=${WEB_SLOTS} native-slots=${NATIVE_SLOTS} poll=${POLL_MS}ms`);
353
+ // Load previous state
354
+ loadState();
355
+ // Initialize worker slots (each with its own bridge)
356
+ await initWorkerSlots();
357
+ persistState();
358
+ // Poll loop
359
+ while (!stopped) {
360
+ try {
361
+ await poll();
362
+ }
363
+ catch (err) {
364
+ log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
365
+ }
366
+ await sleep(POLL_MS);
367
+ }
368
+ }
369
+ function sleep(ms) {
370
+ return new Promise((resolve) => setTimeout(resolve, ms));
371
+ }
372
+ // ── Graceful shutdown ──
373
+ process.on("SIGINT", shutdown);
374
+ process.on("SIGTERM", shutdown);
375
+ async function shutdown() {
376
+ if (stopped)
377
+ return;
378
+ stopped = true;
379
+ log("Shutting down...");
380
+ // Stop all worker bridges
381
+ for (const [, w] of workerRunners) {
382
+ try {
383
+ await w.bridge.stop();
384
+ }
385
+ catch { /* ignore */ }
386
+ }
387
+ persistState();
388
+ try {
389
+ fs.unlinkSync(ORCHESTRATOR_PID_FILE);
390
+ }
391
+ catch { /* ignore */ }
392
+ logStream.end();
393
+ log(`Orchestrator exiting (${totalCompleted} done, ${totalFailed} failed)`);
394
+ process.exit(0);
395
+ }
396
+ main().catch((err) => {
397
+ log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
398
+ process.exit(1);
399
+ });