screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -45,6 +45,8 @@ export class JobRunner {
45
45
  config;
46
46
  heartbeatTimer = null;
47
47
  stopped = false;
48
+ /** PID of the currently focused target app, resolved during focusTargetApp */
49
+ activePid = 0;
48
50
  constructor(bridge, jobs, leaseManager, supervisor, config) {
49
51
  this.bridge = bridge;
50
52
  this.jobs = jobs;
@@ -154,11 +156,12 @@ export class JobRunner {
154
156
  const engineSessionId = runtimeSessionId ?? sessionId;
155
157
  let stepsCompleted = 0;
156
158
  const result = await engine.run(engineSessionId, remainingPlaybook, {
159
+ ...(job.vars ? { vars: job.vars } : {}),
157
160
  onStep: (i, step, res) => {
158
161
  const globalIdx = resumeIdx + i;
159
- this.jobs.completeStep(job.id, globalIdx, { durationMs: 0 });
162
+ this.jobs.completeStep(job.id, globalIdx, { durationMs: 0, output: res });
160
163
  stepsCompleted++;
161
- this.log(` Step ${globalIdx}/${playbook.steps.length - 1}: ${step.description ?? step.action} → ${res}`);
164
+ this.log(` Step ${globalIdx}/${playbook.steps.length - 1}: ${step.description ?? step.action} → ${res.substring(0, 200)}`);
162
165
  },
163
166
  });
164
167
  if (result.success) {
@@ -211,7 +214,10 @@ export class JobRunner {
211
214
  const stepStart = Date.now();
212
215
  const result = await this.executeStep(step);
213
216
  if (result.ok) {
214
- this.jobs.completeStep(job.id, i, { durationMs: Date.now() - stepStart });
217
+ const stepOpts = { durationMs: Date.now() - stepStart };
218
+ if (result.target)
219
+ stepOpts.output = result.target;
220
+ this.jobs.completeStep(job.id, i, stepOpts);
215
221
  stepsCompleted++;
216
222
  consecutiveFailures = 0;
217
223
  this.log(` ✓ ${result.method} in ${result.durationMs}ms${result.fallbackFrom ? ` (fallback from ${result.fallbackFrom})` : ""}`);
@@ -235,11 +241,17 @@ export class JobRunner {
235
241
  this.log(` → blocked (transient): ${reason}`);
236
242
  return this.finalize(job, start, stepsCompleted, reason);
237
243
  }
238
- if (consecutiveFailures >= this.config.maxConsecutiveFailures) {
244
+ // L2-72 fix: Use job's maxRetries if set, otherwise use runner's maxConsecutiveFailures
245
+ const maxFails = job.maxRetries ?? this.config.maxConsecutiveFailures;
246
+ if (consecutiveFailures >= maxFails) {
239
247
  this.jobs.transition(job.id, "failed", { error: `${consecutiveFailures} consecutive step failures. Last: ${lastError}` });
240
248
  this.log(` → failed: ${consecutiveFailures} consecutive failures`);
241
249
  return this.finalize(job, start, stepsCompleted, lastError);
242
250
  }
251
+ // L2-72 fix: A failed step blocks subsequent steps — break out of the loop
252
+ // (the job will be marked as failed in the allAttempted check below)
253
+ this.log(` Step failed — stopping execution (${consecutiveFailures}/${maxFails} consecutive failures)`);
254
+ break;
243
255
  }
244
256
  // Delay between steps
245
257
  if (i < job.steps.length - 1) {
@@ -252,10 +264,23 @@ export class JobRunner {
252
264
  return this.finalize(job, start, stepsCompleted, "Job disappeared");
253
265
  }
254
266
  const allDone = updated.steps.every((s) => s.status === "done" || s.status === "skipped");
267
+ const allAttempted = updated.steps.every((s) => s.status === "done" || s.status === "skipped" || s.status === "failed");
255
268
  if (allDone && !this.stopped) {
256
269
  this.jobs.transition(job.id, "done");
257
270
  this.log(`Job ${job.id} → done (${stepsCompleted} steps in ${Date.now() - start}ms)`);
258
271
  }
272
+ else if (allAttempted && !this.stopped) {
273
+ // L2-72 fix: All steps attempted but some failed → mark as failed, not done
274
+ const failedCount = updated.steps.filter((s) => s.status === "failed").length;
275
+ this.jobs.transition(job.id, "failed", { error: `${failedCount} step(s) failed` });
276
+ this.log(`Job ${job.id} → failed with ${failedCount} failed step(s) (${stepsCompleted}/${updated.steps.length} in ${Date.now() - start}ms)`);
277
+ }
278
+ else if (!this.stopped && lastError) {
279
+ // L2-72 fix: Broke out of loop due to a failed step with pending steps remaining
280
+ const failedCount = updated.steps.filter((s) => s.status === "failed").length;
281
+ this.jobs.transition(job.id, "failed", { error: `Step failed, ${updated.steps.length - stepsCompleted - failedCount} step(s) skipped. Last: ${lastError}` });
282
+ this.log(`Job ${job.id} → failed at step (${stepsCompleted}/${updated.steps.length}, ${failedCount} failed)`);
283
+ }
259
284
  else if (this.stopped) {
260
285
  this.log(`Job ${job.id} paused at step ${updated.lastStep + 1}`);
261
286
  }
@@ -276,6 +301,8 @@ export class JobRunner {
276
301
  if (!target) {
277
302
  throw new Error(`Target app ${job.bundleId} is not running`);
278
303
  }
304
+ // Store resolved PID for use in execClick/execType AX calls
305
+ this.activePid = target.pid;
279
306
  // Focus the app
280
307
  await this.bridge.call("app.focus", { bundleId: job.bundleId });
281
308
  // If windowId specified, validate it exists
@@ -288,8 +315,14 @@ export class JobRunner {
288
315
  }
289
316
  }
290
317
  catch (err) {
291
- // Log but don't fail the step itself will fail if the target isn't right
292
- this.log(` Warning: focus target app failed: ${err instanceof Error ? err.message : String(err)}`);
318
+ const msg = err instanceof Error ? err.message : String(err);
319
+ // If the target app is dead, this is fatal don't silently continue
320
+ // sending keystrokes to the wrong app
321
+ if (msg.includes("is not running")) {
322
+ throw new Error(`Target app killed: ${msg}`);
323
+ }
324
+ // Other focus errors (e.g. window not found) are non-fatal
325
+ this.log(` Warning: focus target app failed: ${msg}`);
293
326
  }
294
327
  }
295
328
  /**
@@ -422,6 +455,13 @@ export class JobRunner {
422
455
  case "read":
423
456
  case "extract":
424
457
  return await this.execRead(method, target, start, attempt);
458
+ case "focus":
459
+ case "launch":
460
+ return await this.execFocus(target, start, attempt);
461
+ case "browser_js":
462
+ return await this.execBrowserJs(step.description ?? "", start, attempt);
463
+ case "cdp_key_event":
464
+ return await this.execCdpKeyEvent(step.keys ?? "", start, attempt);
425
465
  default:
426
466
  // Try as a generic click on the target text
427
467
  if (target)
@@ -439,8 +479,8 @@ export class JobRunner {
439
479
  return { ok: false, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: "Click requires a target", target };
440
480
  switch (method) {
441
481
  case "ax": {
442
- const found = await this.bridge.call("ax.findElement", { pid: 0, title: target, exact: false });
443
- await this.bridge.call("ax.performAction", { pid: 0, elementPath: found.elementPath, action: "AXPress" });
482
+ const found = await this.bridge.call("ax.findElement", { pid: this.activePid, title: target, exact: false });
483
+ await this.bridge.call("ax.performAction", { pid: this.activePid, elementPath: found.elementPath, action: "AXPress" });
444
484
  return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target };
445
485
  }
446
486
  case "cdp": {
@@ -482,12 +522,12 @@ export class JobRunner {
482
522
  switch (method) {
483
523
  case "ax": {
484
524
  if (target) {
485
- const found = await this.bridge.call("ax.findElement", { pid: 0, title: target, exact: false });
486
- await this.bridge.call("ax.setElementValue", { pid: 0, elementPath: found.elementPath, value: text });
525
+ const found = await this.bridge.call("ax.findElement", { pid: this.activePid, title: target, exact: false });
526
+ await this.bridge.call("ax.setElementValue", { pid: this.activePid, elementPath: found.elementPath, value: text });
487
527
  }
488
528
  else {
489
- // Type into focused element via key events
490
- await this.bridge.call("cg.typeText", { text });
529
+ // Type into focused element via key events — use pid for targeting
530
+ await this.bridge.call("cg.typeText", { text, pid: this.activePid });
491
531
  }
492
532
  return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target };
493
533
  }
@@ -520,6 +560,14 @@ export class JobRunner {
520
560
  async execNavigate(url, start, attempt) {
521
561
  if (!url)
522
562
  return { ok: false, method: "ax", durationMs: 0, fallbackFrom: null, retries: attempt, error: "Navigate requires a URL target", target: null };
563
+ // L2-74 fix: Block dangerous URL protocols (mirrors L2-71 fix in mcp-desktop.ts)
564
+ const BLOCKED_PROTOCOLS = ["javascript:", "data:", "blob:", "vbscript:"];
565
+ const urlLower = url.trim().toLowerCase();
566
+ for (const proto of BLOCKED_PROTOCOLS) {
567
+ if (urlLower.startsWith(proto)) {
568
+ return { ok: false, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: `Blocked: "${proto}" URLs are not allowed for security reasons`, target: url };
569
+ }
570
+ }
523
571
  if (this.config.cdpConnect) {
524
572
  const client = await this.config.cdpConnect();
525
573
  try {
@@ -530,9 +578,16 @@ export class JobRunner {
530
578
  await client.close();
531
579
  }
532
580
  }
533
- // Fallback: use AppleScript / open command
534
- await this.bridge.call("app.openURL", { url });
535
- return { ok: true, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: url };
581
+ // Fallback: try bridge openURL, then macOS `open` command
582
+ try {
583
+ await this.bridge.call("app.openURL", { url });
584
+ return { ok: true, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: url };
585
+ }
586
+ catch {
587
+ const { execSync } = await import("node:child_process");
588
+ execSync(`open ${JSON.stringify(url)}`, { timeout: 10_000 });
589
+ return { ok: true, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: url };
590
+ }
536
591
  }
537
592
  async execScreenshot(start, attempt) {
538
593
  const shot = await this.bridge.call("cg.captureScreen", {});
@@ -562,6 +617,24 @@ export class JobRunner {
562
617
  }
563
618
  throw new Error(`Method ${method} does not support scroll`);
564
619
  }
620
+ async execFocus(bundleId, start, attempt) {
621
+ if (!bundleId)
622
+ return { ok: false, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: "Focus/launch requires a bundleId target", target: null };
623
+ try {
624
+ await this.bridge.call("app.focus", { bundleId });
625
+ return { ok: true, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: bundleId };
626
+ }
627
+ catch {
628
+ // Fallback: try launch (app might not be running)
629
+ try {
630
+ await this.bridge.call("app.launch", { bundleId });
631
+ return { ok: true, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: bundleId };
632
+ }
633
+ catch (err) {
634
+ return { ok: false, method: "ax", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: err instanceof Error ? err.message : String(err), target: bundleId };
635
+ }
636
+ }
637
+ }
565
638
  async execKey(keys, start, attempt) {
566
639
  // keys is a "+" separated combo like "cmd+a"
567
640
  const parts = keys.split("+").map((k) => k.trim());
@@ -618,6 +691,56 @@ export class JobRunner {
618
691
  }
619
692
  throw new Error(`Method ${method} does not support read`);
620
693
  }
694
+ async execBrowserJs(code, start, attempt) {
695
+ if (!this.config.cdpConnect)
696
+ throw new Error("browser_js requires CDP");
697
+ const client = await this.config.cdpConnect();
698
+ try {
699
+ const result = await client.Runtime.evaluate({
700
+ expression: code,
701
+ awaitPromise: true,
702
+ returnByValue: true,
703
+ });
704
+ if (result.exceptionDetails) {
705
+ throw new Error(`JS Error: ${result.exceptionDetails.text ?? result.exceptionDetails.exception?.description ?? "unknown"}`);
706
+ }
707
+ return { ok: true, method: "cdp", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: String(result.result?.value ?? "") };
708
+ }
709
+ finally {
710
+ await client.close();
711
+ }
712
+ }
713
+ async execCdpKeyEvent(keys, start, attempt) {
714
+ if (!this.config.cdpConnect)
715
+ throw new Error("cdp_key_event requires CDP");
716
+ const client = await this.config.cdpConnect();
717
+ try {
718
+ // Parse "mod4+Enter" format or raw key
719
+ const parts = keys.split("+").map(k => k.trim());
720
+ let modifiers = 0;
721
+ let key = parts[parts.length - 1] ?? "";
722
+ for (const p of parts.slice(0, -1)) {
723
+ if (p.startsWith("mod"))
724
+ modifiers = parseInt(p.replace("mod", ""), 10) || 0;
725
+ if (p === "Meta" || p === "Cmd")
726
+ modifiers = 4;
727
+ if (p === "Shift")
728
+ modifiers |= 8;
729
+ if (p === "Ctrl")
730
+ modifiers |= 2;
731
+ if (p === "Alt")
732
+ modifiers |= 1;
733
+ }
734
+ const keyCode = key === "Enter" ? 13 : key === "Tab" ? 9 : key === "Escape" ? 27 : 0;
735
+ const baseParams = { key, code: key, modifiers, windowsVirtualKeyCode: keyCode, nativeVirtualKeyCode: keyCode };
736
+ await client.Input.dispatchKeyEvent({ type: "keyDown", ...baseParams });
737
+ await client.Input.dispatchKeyEvent({ type: "keyUp", ...baseParams });
738
+ return { ok: true, method: "cdp", durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: keys };
739
+ }
740
+ finally {
741
+ await client.close();
742
+ }
743
+ }
621
744
  // ── Blocker classification ──────────────────────
622
745
  /** Check a single error string for blocker patterns. */
623
746
  classifyBlocker(error) {
@@ -0,0 +1,356 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ import * as fs from "node:fs";
4
+ import * as path from "node:path";
5
+ import * as os from "node:os";
6
+ import { writeFileAtomicSync } from "../util/atomic-write.js";
7
+ import { LocatorPolicy } from "./locator-policy.js";
8
+ import { RecoveryPolicy } from "./recovery-policy.js";
9
+ import { TimingModel } from "./timing-model.js";
10
+ import { SensorPolicy } from "./sensor-policy.js";
11
+ import { PatternPolicy } from "./pattern-policy.js";
12
+ import { TopologyPolicy } from "./topology-policy.js";
13
+ import { DEFAULT_LEARNING_CONFIG } from "./types.js";
14
+ /**
15
+ * Prune an array to `max` entries, keeping the most recent by date field.
16
+ */
17
+ function pruneByDate(entries, max, getDate) {
18
+ return [...entries]
19
+ .sort((a, b) => getDate(b).localeCompare(getDate(a)))
20
+ .slice(0, max);
21
+ }
22
+ /**
23
+ * LearningEngine — the central coordinator for all learning policies.
24
+ *
25
+ * Observes outcomes from tool execution, recovery, and perception,
26
+ * updates the four sub-policies, and provides recommendations that
27
+ * make the system smarter over time.
28
+ *
29
+ * Persistence: each policy writes its own JSONL file in the data directory.
30
+ * Loading is eager (on init), saving is debounced.
31
+ */
32
+ export class LearningEngine {
33
+ locators;
34
+ recovery;
35
+ timing;
36
+ sensors;
37
+ patterns;
38
+ topology;
39
+ config;
40
+ dirty = false;
41
+ saveTimer = null;
42
+ constructor(config) {
43
+ this.config = {
44
+ ...DEFAULT_LEARNING_CONFIG,
45
+ dataDir: config?.dataDir ??
46
+ path.join(os.homedir(), ".screenhand", "learning"),
47
+ ...config,
48
+ };
49
+ this.locators = new LocatorPolicy(this.config.priorStrength);
50
+ this.recovery = new RecoveryPolicy(this.config.priorStrength);
51
+ this.timing = new TimingModel(this.config.maxTimingSamples);
52
+ this.sensors = new SensorPolicy(this.config.priorStrength);
53
+ this.patterns = new PatternPolicy(this.config.priorStrength);
54
+ this.topology = new TopologyPolicy(this.config.priorStrength);
55
+ }
56
+ /**
57
+ * Initialize: create data directory and load persisted data.
58
+ */
59
+ init() {
60
+ fs.mkdirSync(this.config.dataDir, { recursive: true });
61
+ this.load();
62
+ }
63
+ // ── Record Outcomes ─────────────────────────────────────────────
64
+ recordLocatorOutcome(outcome) {
65
+ this.locators.record(outcome);
66
+ this.scheduleSave();
67
+ }
68
+ recordRecoveryOutcome(outcome) {
69
+ this.recovery.record(outcome);
70
+ this.scheduleSave();
71
+ }
72
+ recordToolTiming(event) {
73
+ this.timing.record(event);
74
+ this.scheduleSave();
75
+ }
76
+ recordSensorOutcome(outcome) {
77
+ this.sensors.record(outcome);
78
+ this.scheduleSave();
79
+ }
80
+ recordPattern(outcome) {
81
+ this.patterns.record(outcome);
82
+ this.scheduleSave();
83
+ }
84
+ recordTopologyOutcome(outcome) {
85
+ this.topology.record(outcome);
86
+ this.scheduleSave();
87
+ }
88
+ // ── Recommendations ─────────────────────────────────────────────
89
+ /**
90
+ * Get the best locator for a given app×action.
91
+ */
92
+ recommendLocator(bundleId, actionKey) {
93
+ return this.locators.recommend(bundleId, actionKey, this.config.minSamplesForConfidence);
94
+ }
95
+ /**
96
+ * Get ranked recovery strategies for a blocker×app pair.
97
+ */
98
+ rankRecoveryStrategies(blockerType, bundleId) {
99
+ return this.recovery.rank(blockerType, bundleId);
100
+ }
101
+ /**
102
+ * Get adaptive timeouts for a given app.
103
+ */
104
+ getAdaptiveBudget(bundleId) {
105
+ return this.timing.getAdaptiveBudget(bundleId, this.config.minSamplesForConfidence);
106
+ }
107
+ /**
108
+ * Get per-app source confidence using Bayesian posterior from observed accuracy.
109
+ * Falls back to hardcoded defaults if insufficient data.
110
+ */
111
+ getSourceConfidence(bundleId, source) {
112
+ const DEFAULTS = { ax: 0.9, cdp: 0.85, ocr: 0.7, vision: 0.6 };
113
+ const ranked = this.sensors.rank(bundleId);
114
+ const match = ranked.find((r) => r.sourceType === source);
115
+ if (match && match.score > 0) {
116
+ // Bayesian posterior from sensor policy is already computed
117
+ return match.score;
118
+ }
119
+ return DEFAULTS[source] ?? 0.5;
120
+ }
121
+ /**
122
+ * Get ranked perception sources for a given app.
123
+ */
124
+ rankSensors(bundleId) {
125
+ return this.sensors.rank(bundleId);
126
+ }
127
+ /**
128
+ * Query verified UI patterns for a given app, optionally filtered by tool.
129
+ */
130
+ queryPatterns(bundleId, tool) {
131
+ return this.patterns.query(bundleId, tool);
132
+ }
133
+ /**
134
+ * Get the best verified pattern for a given app×tool.
135
+ */
136
+ recommendPattern(bundleId, tool) {
137
+ return this.patterns.recommend(bundleId, tool, this.config.minSamplesForConfidence);
138
+ }
139
+ /**
140
+ * Query all topology entries (navigation edges) for a given app.
141
+ */
142
+ queryTopology(bundleId) {
143
+ return this.topology.query(bundleId);
144
+ }
145
+ /**
146
+ * Get the best navigation edge from a given node.
147
+ */
148
+ recommendNextNavigation(bundleId, fromNode) {
149
+ return this.topology.recommend(bundleId, fromNode, this.config.minSamplesForConfidence);
150
+ }
151
+ /**
152
+ * Get a summary of learning stats for a given app.
153
+ */
154
+ getAppSummary(bundleId) {
155
+ const locPrefix = `${bundleId.length}:${bundleId}\0`;
156
+ const locEntries = this.locators
157
+ .getAllEntries()
158
+ .filter((e) => e.key.startsWith(locPrefix));
159
+ const recEntries = this.recovery
160
+ .getAllEntries()
161
+ .filter((e) => {
162
+ const parts = e.key.split("::");
163
+ return parts[parts.length - 1] === bundleId;
164
+ });
165
+ const timSamples = this.timing
166
+ .getAllSamples()
167
+ .filter((s) => s.bundleId === bundleId);
168
+ const senEntries = this.sensors
169
+ .getAllEntries()
170
+ .filter((e) => e.bundleId === bundleId);
171
+ const patEntries = this.patterns.query(bundleId);
172
+ const topoEntries = this.topology.query(bundleId);
173
+ const topSensor = this.sensors.recommend(bundleId, 1);
174
+ const topLoc = locEntries.sort((a, b) => b.score - a.score)[0];
175
+ return {
176
+ locatorEntries: locEntries.length,
177
+ recoveryEntries: recEntries.length,
178
+ timingSamples: timSamples.length,
179
+ sensorEntries: senEntries.length,
180
+ patternEntries: patEntries.length,
181
+ topologyEntries: topoEntries.length,
182
+ topLocatorMethod: topLoc?.method ?? null,
183
+ topSensor,
184
+ adaptiveBudget: this.getAdaptiveBudget(bundleId),
185
+ };
186
+ }
187
+ // ── Persistence ─────────────────────────────────────────────────
188
+ /**
189
+ * Clear all learning data and flush empty state to disk.
190
+ */
191
+ reset() {
192
+ this.locators.clear();
193
+ this.recovery.clear();
194
+ this.timing.clear();
195
+ this.sensors.clear();
196
+ this.patterns.clear();
197
+ this.topology.clear();
198
+ this.flush();
199
+ }
200
+ /**
201
+ * Force save all policies to disk.
202
+ */
203
+ flush() {
204
+ if (this.saveTimer) {
205
+ clearTimeout(this.saveTimer);
206
+ this.saveTimer = null;
207
+ }
208
+ this.save();
209
+ }
210
+ scheduleSave() {
211
+ this.dirty = true;
212
+ if (this.saveTimer)
213
+ return;
214
+ this.saveTimer = setTimeout(() => {
215
+ this.saveTimer = null;
216
+ if (this.dirty) {
217
+ this.save();
218
+ this.dirty = false;
219
+ }
220
+ }, 500);
221
+ }
222
+ save() {
223
+ try {
224
+ const dir = this.config.dataDir;
225
+ const max = this.config.maxEntriesPerFile;
226
+ // Locator entries — prune by lastUsed
227
+ let locatorEntries = this.locators.getAllEntries();
228
+ if (locatorEntries.length > max) {
229
+ locatorEntries = pruneByDate(locatorEntries, max, (e) => e.lastUsed);
230
+ this.locators.loadEntries(locatorEntries);
231
+ }
232
+ const locatorData = locatorEntries.map((e) => JSON.stringify(e)).join("\n");
233
+ if (locatorData) {
234
+ writeFileAtomicSync(path.join(dir, "locators.jsonl"), locatorData + "\n");
235
+ }
236
+ // Recovery entries — prune by lastUsed
237
+ let recoveryEntries = this.recovery.getAllEntries();
238
+ if (recoveryEntries.length > max) {
239
+ recoveryEntries = pruneByDate(recoveryEntries, max, (e) => e.lastUsed);
240
+ this.recovery.loadEntries(recoveryEntries);
241
+ }
242
+ const recoveryData = recoveryEntries.map((e) => JSON.stringify(e)).join("\n");
243
+ if (recoveryData) {
244
+ writeFileAtomicSync(path.join(dir, "recoveries.jsonl"), recoveryData + "\n");
245
+ }
246
+ // Timing samples — prune by timestamp
247
+ let timingSamples = this.timing.getAllSamples();
248
+ if (timingSamples.length > max) {
249
+ timingSamples = pruneByDate(timingSamples, max, (s) => s.timestamp);
250
+ this.timing.loadSamples(timingSamples);
251
+ }
252
+ const timingData = timingSamples.map((s) => JSON.stringify(s)).join("\n");
253
+ if (timingData) {
254
+ writeFileAtomicSync(path.join(dir, "timings.jsonl"), timingData + "\n");
255
+ }
256
+ // Sensor entries — prune by lastUsed
257
+ let sensorEntries = this.sensors.getAllEntries();
258
+ if (sensorEntries.length > max) {
259
+ sensorEntries = pruneByDate(sensorEntries, max, (e) => e.lastUsed);
260
+ this.sensors.loadEntries(sensorEntries);
261
+ }
262
+ const sensorData = sensorEntries.map((e) => JSON.stringify(e)).join("\n");
263
+ if (sensorData) {
264
+ writeFileAtomicSync(path.join(dir, "sensors.jsonl"), sensorData + "\n");
265
+ }
266
+ // Pattern entries — prune by lastSeen
267
+ let patternEntries = this.patterns.getAllEntries();
268
+ if (patternEntries.length > max) {
269
+ patternEntries = pruneByDate(patternEntries, max, (e) => e.lastSeen);
270
+ this.patterns.loadEntries(patternEntries);
271
+ }
272
+ const patternData = patternEntries.map((e) => JSON.stringify(e)).join("\n");
273
+ if (patternData) {
274
+ writeFileAtomicSync(path.join(dir, "patterns.jsonl"), patternData + "\n");
275
+ }
276
+ // Topology entries — prune by lastUsed
277
+ let topologyEntries = this.topology.getAllEntries();
278
+ if (topologyEntries.length > max) {
279
+ topologyEntries = pruneByDate(topologyEntries, max, (e) => e.lastUsed);
280
+ this.topology.loadEntries(topologyEntries);
281
+ }
282
+ const topologyData = topologyEntries.map((e) => JSON.stringify(e)).join("\n");
283
+ if (topologyData) {
284
+ writeFileAtomicSync(path.join(dir, "topology.jsonl"), topologyData + "\n");
285
+ }
286
+ }
287
+ catch {
288
+ // Persistence failure is non-fatal — data stays in memory
289
+ }
290
+ }
291
+ load() {
292
+ const dir = this.config.dataDir;
293
+ // Load locators
294
+ const locatorEntries = this.readJsonl(path.join(dir, "locators.jsonl"));
295
+ if (locatorEntries.length > 0) {
296
+ this.locators.loadEntries(locatorEntries);
297
+ }
298
+ // Load recoveries
299
+ const recoveryEntries = this.readJsonl(path.join(dir, "recoveries.jsonl"));
300
+ if (recoveryEntries.length > 0) {
301
+ this.recovery.loadEntries(recoveryEntries);
302
+ }
303
+ // Load timings
304
+ const timingSamples = this.readJsonl(path.join(dir, "timings.jsonl"));
305
+ if (timingSamples.length > 0) {
306
+ this.timing.loadSamples(timingSamples);
307
+ }
308
+ // Load sensors
309
+ const sensorEntries = this.readJsonl(path.join(dir, "sensors.jsonl"));
310
+ if (sensorEntries.length > 0) {
311
+ this.sensors.loadEntries(sensorEntries);
312
+ }
313
+ // Load patterns
314
+ const patternEntries = this.readJsonl(path.join(dir, "patterns.jsonl"));
315
+ if (patternEntries.length > 0) {
316
+ this.patterns.loadEntries(patternEntries);
317
+ }
318
+ // Load topology
319
+ const topologyEntries = this.readJsonl(path.join(dir, "topology.jsonl"));
320
+ if (topologyEntries.length > 0) {
321
+ this.topology.loadEntries(topologyEntries);
322
+ }
323
+ }
324
+ readJsonl(filePath) {
325
+ try {
326
+ if (!fs.existsSync(filePath))
327
+ return [];
328
+ // Guard against oversized files: skip if larger than 10MB
329
+ const stat = fs.statSync(filePath);
330
+ if (stat.size > 10 * 1024 * 1024) {
331
+ console.error(`[Learning] Skipping oversized file: ${filePath} (${stat.size} bytes)`);
332
+ return [];
333
+ }
334
+ const content = fs.readFileSync(filePath, "utf-8");
335
+ const results = [];
336
+ const maxEntries = this.config.maxEntriesPerFile;
337
+ for (const line of content.split("\n")) {
338
+ if (results.length >= maxEntries)
339
+ break;
340
+ const trimmed = line.trim();
341
+ if (!trimmed)
342
+ continue;
343
+ try {
344
+ results.push(JSON.parse(trimmed));
345
+ }
346
+ catch {
347
+ // Skip corrupt lines
348
+ }
349
+ }
350
+ return results;
351
+ }
352
+ catch {
353
+ return [];
354
+ }
355
+ }
356
+ }
@@ -0,0 +1,9 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ export { LearningEngine } from "./engine.js";
4
+ export { LocatorPolicy } from "./locator-policy.js";
5
+ export { RecoveryPolicy } from "./recovery-policy.js";
6
+ export { TimingModel } from "./timing-model.js";
7
+ export { SensorPolicy } from "./sensor-policy.js";
8
+ export { TopologyPolicy } from "./topology-policy.js";
9
+ export { DEFAULT_LEARNING_CONFIG } from "./types.js";