screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,92 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * GoalStore — atomic JSON persistence for planner goals.
19
+ *
20
+ * In-memory cache + sync atomic writes (same pattern as JobStore).
21
+ * File: ~/.screenhand/planner/goals.json
22
+ */
23
+ import fs from "node:fs";
24
+ import path from "node:path";
25
+ import { writeFileAtomicSync, readJsonWithRecovery } from "../util/atomic-write.js";
26
+ const MAX_GOALS = 100;
27
+ export class GoalStore {
28
+ filePath;
29
+ goals = [];
30
+ initialized = false;
31
+ constructor(dir) {
32
+ fs.mkdirSync(dir, { recursive: true });
33
+ this.filePath = path.join(dir, "goals.json");
34
+ }
35
+ init() {
36
+ if (this.initialized)
37
+ return;
38
+ this.initialized = true;
39
+ this.goals = readJsonWithRecovery(this.filePath) ?? [];
40
+ }
41
+ get(id) {
42
+ return this.goals.find((g) => g.id === id);
43
+ }
44
+ list(status) {
45
+ if (status)
46
+ return this.goals.filter((g) => g.status === status);
47
+ return [...this.goals];
48
+ }
49
+ add(goal) {
50
+ this.goals.push(goal);
51
+ this.persist();
52
+ }
53
+ update(id, goal) {
54
+ const idx = this.goals.findIndex((g) => g.id === id);
55
+ if (idx < 0) {
56
+ this.goals.push(goal);
57
+ }
58
+ else {
59
+ this.goals[idx] = goal;
60
+ }
61
+ this.persist();
62
+ }
63
+ remove(id) {
64
+ const before = this.goals.length;
65
+ this.goals = this.goals.filter((g) => g.id !== id);
66
+ if (this.goals.length < before) {
67
+ this.persist();
68
+ return true;
69
+ }
70
+ return false;
71
+ }
72
+ prune() {
73
+ const terminal = this.goals
74
+ .filter((g) => g.status === "completed" || g.status === "failed")
75
+ .sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
76
+ if (terminal.length <= MAX_GOALS)
77
+ return 0;
78
+ const evictCount = terminal.length - MAX_GOALS;
79
+ const evictIds = new Set(terminal.slice(0, evictCount).map((g) => g.id));
80
+ this.goals = this.goals.filter((g) => !evictIds.has(g.id));
81
+ this.persist();
82
+ return evictCount;
83
+ }
84
+ persist() {
85
+ try {
86
+ writeFileAtomicSync(this.filePath, JSON.stringify(this.goals, null, 2));
87
+ }
88
+ catch {
89
+ // Non-critical — in-memory cache is authoritative
90
+ }
91
+ }
92
+ }
@@ -0,0 +1,21 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export { Planner } from "./planner.js";
18
+ export { PlanExecutor } from "./executor.js";
19
+ export { GoalStore } from "./goal-store.js";
20
+ export { ToolRegistry } from "./tool-registry.js";
21
+ export { playbookToPlan, strategyToPlan, flowToPlan } from "./deterministic.js";
@@ -0,0 +1,520 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ import crypto from "node:crypto";
18
+ import { DEFAULT_PLANNER_CONFIG } from "./types.js";
19
+ import { playbookToPlan, strategyToPlan, flowToPlan } from "./deterministic.js";
20
+ function uid() {
21
+ return crypto.randomBytes(6).toString("hex");
22
+ }
23
+ /**
24
+ * Decompose a goal description into subgoal parts.
25
+ * Splits on: numbered steps ("1. ... 2. ..."), "and then", "then",
26
+ * ", and " (Oxford comma), or semicolons.
27
+ * Returns the original description as a single-element array if no split applies.
28
+ */
29
+ function decomposeGoal(description) {
30
+ // 1. Try numbered steps: "1. do X 2. do Y" or "1) do X 2) do Y"
31
+ const numberedPattern = /(?:^|\s)(\d+)[.)]\s+/g;
32
+ const numberedMatches = [...description.matchAll(numberedPattern)];
33
+ if (numberedMatches.length >= 2) {
34
+ const parts = [];
35
+ for (let i = 0; i < numberedMatches.length; i++) {
36
+ const start = numberedMatches[i].index + numberedMatches[i][0].indexOf(numberedMatches[i][1]);
37
+ const stepStart = start + numberedMatches[i][0].trimStart().length;
38
+ const end = i + 1 < numberedMatches.length
39
+ ? numberedMatches[i + 1].index
40
+ : description.length;
41
+ const text = description.slice(stepStart, end).trim();
42
+ if (text)
43
+ parts.push(text);
44
+ }
45
+ if (parts.length >= 2)
46
+ return parts;
47
+ }
48
+ // 2. Try semicolons
49
+ const semiParts = description.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
50
+ if (semiParts.length >= 2)
51
+ return semiParts;
52
+ // 3. Try "and then" or ", then"
53
+ const thenParts = description.split(/\s+and\s+then\s+|,\s*then\s+/i).map((s) => s.trim()).filter((s) => s.length > 0);
54
+ if (thenParts.length >= 2)
55
+ return thenParts;
56
+ // 4. Try ", and " (Oxford comma pattern — implies list of actions)
57
+ const andParts = description.split(/,\s+and\s+/i).map((s) => s.trim()).filter((s) => s.length > 0);
58
+ if (andParts.length >= 2)
59
+ return andParts;
60
+ // 5. No decomposition
61
+ return [description];
62
+ }
63
+ /**
64
+ * Planner — goal-oriented planning with deterministic fast-path.
65
+ *
66
+ * Priority:
67
+ * 1. Playbook match → deterministic plan (0 LLM calls)
68
+ * 2. Strategy recall → plan from memory (0 LLM calls)
69
+ * 3. Reference flow → semi-deterministic (LLM interprets steps)
70
+ * 4. LLM generation → full plan from scratch
71
+ */
72
+ export class Planner {
73
+ playbookStore;
74
+ memory;
75
+ contextTracker;
76
+ worldModel;
77
+ config;
78
+ constructor(playbookStore, memory, contextTracker, worldModel, configOrLearning, learningOrConfig) {
79
+ this.playbookStore = playbookStore;
80
+ this.memory = memory;
81
+ this.contextTracker = contextTracker;
82
+ this.worldModel = worldModel;
83
+ // Support both (config, learning) and (learning, config) orderings
84
+ let config;
85
+ let learning;
86
+ for (const arg of [configOrLearning, learningOrConfig]) {
87
+ if (!arg)
88
+ continue;
89
+ if (typeof arg.recommendLocator === "function") {
90
+ learning = arg;
91
+ }
92
+ else {
93
+ config = arg;
94
+ }
95
+ }
96
+ this.config = { ...DEFAULT_PLANNER_CONFIG, ...config };
97
+ this.learningEngine = learning ?? null;
98
+ }
99
+ learningEngine;
100
+ toolRegistry = null;
101
+ /**
102
+ * Set the tool registry for LLM plan generation.
103
+ */
104
+ setToolRegistry(registry) {
105
+ this.toolRegistry = registry;
106
+ }
107
+ /**
108
+ * Create a Goal from a description.
109
+ * Decomposes complex goals into multiple subgoals when the description
110
+ * contains "and then", "then", ", and", numbered steps, or semicolons.
111
+ */
112
+ createGoal(description) {
113
+ const parts = decomposeGoal(description);
114
+ const subgoals = parts.map((part) => ({
115
+ id: `sg_${uid()}`,
116
+ description: part,
117
+ status: "pending",
118
+ plan: null,
119
+ attempts: 0,
120
+ maxAttempts: this.config.defaultMaxAttempts,
121
+ lastError: null,
122
+ }));
123
+ const goal = {
124
+ id: `goal_${uid()}`,
125
+ description,
126
+ status: "pending",
127
+ subgoals,
128
+ createdAt: new Date().toISOString(),
129
+ completedAt: null,
130
+ };
131
+ return goal;
132
+ }
133
+ /**
134
+ * Plan a subgoal: find the best ActionPlan from available sources.
135
+ *
136
+ * Priority: playbook > strategy > reference flow > LLM
137
+ */
138
+ async planSubgoal(subgoal) {
139
+ // 1. Try playbook match
140
+ const playbookPlan = this.findPlaybookPlan(subgoal.description);
141
+ if (playbookPlan)
142
+ return playbookPlan;
143
+ // 2. Try strategy recall
144
+ const strategyPlan = this.findStrategyPlan(subgoal.description);
145
+ if (strategyPlan)
146
+ return strategyPlan;
147
+ // 3. Try reference flow
148
+ const flowPlan = this.findFlowPlan(subgoal.description);
149
+ if (flowPlan)
150
+ return flowPlan;
151
+ // 4. Fallback: LLM-generated plan (or stub if no API key)
152
+ return this.createLLMPlan(subgoal.description);
153
+ }
154
+ /**
155
+ * Plan all subgoals in a goal.
156
+ */
157
+ async planGoal(goal) {
158
+ goal.status = "active";
159
+ for (const sg of goal.subgoals) {
160
+ if (sg.status === "completed" || sg.status === "skipped")
161
+ continue;
162
+ // Don't re-plan subgoals that already have a plan (e.g. from plan_goal)
163
+ if (!sg.plan) {
164
+ sg.plan = await this.planSubgoal(sg);
165
+ }
166
+ sg.status = "pending";
167
+ }
168
+ }
169
+ /**
170
+ * Replan a subgoal after failure.
171
+ * Increments attempt count, resets plan, tries alternative sources.
172
+ */
173
+ async replan(subgoal, reason, errorMsg) {
174
+ subgoal.attempts++;
175
+ subgoal.lastError = errorMsg ?? reason;
176
+ if (subgoal.attempts >= subgoal.maxAttempts) {
177
+ subgoal.status = "failed";
178
+ return null;
179
+ }
180
+ subgoal.status = "pending";
181
+ // On replan, try alternative sources or adjust params
182
+ const currentSource = subgoal.plan?.source;
183
+ // If playbook failed, try strategy
184
+ if (currentSource === "playbook") {
185
+ const strategyPlan = this.findStrategyPlan(subgoal.description);
186
+ if (strategyPlan)
187
+ return strategyPlan;
188
+ }
189
+ // If strategy failed, try reference flow
190
+ if (currentSource === "playbook" || currentSource === "strategy") {
191
+ const flowPlan = this.findFlowPlan(subgoal.description);
192
+ if (flowPlan)
193
+ return flowPlan;
194
+ }
195
+ // Don't downgrade deterministic plans to LLM stubs.
196
+ // A 9-step reference_flow failing on step 1 (bridge crash) shouldn't
197
+ // be replaced with a 1-step "ask the human" stub. Better to retry
198
+ // the original plan or fail cleanly.
199
+ if (currentSource === "playbook" || currentSource === "strategy" || currentSource === "reference_flow") {
200
+ // Reset the current plan to retry from the failed step
201
+ if (subgoal.plan) {
202
+ subgoal.plan.currentStepIndex = 0;
203
+ // Reset step statuses so they can be re-executed
204
+ for (const step of subgoal.plan.steps) {
205
+ if (step.status === "failed")
206
+ step.status = "pending";
207
+ }
208
+ return subgoal.plan;
209
+ }
210
+ }
211
+ // Only fall back to LLM when no deterministic plan existed
212
+ return this.createLLMPlan(subgoal.description);
213
+ }
214
+ /**
215
+ * Check if a goal is complete (all subgoals done or failed).
216
+ */
217
+ evaluateGoal(goal) {
218
+ const allDone = goal.subgoals.every((sg) => sg.status === "completed" ||
219
+ sg.status === "failed" ||
220
+ sg.status === "skipped");
221
+ if (!allDone)
222
+ return;
223
+ const anyFailed = goal.subgoals.some((sg) => sg.status === "failed");
224
+ goal.status = anyFailed ? "failed" : "completed";
225
+ goal.completedAt = new Date().toISOString();
226
+ }
227
+ /**
228
+ * Serialize a goal to JSON (for persistence/transport).
229
+ */
230
+ static serializeGoal(goal) {
231
+ return JSON.stringify(goal);
232
+ }
233
+ /**
234
+ * Deserialize a goal from JSON.
235
+ */
236
+ static deserializeGoal(json) {
237
+ const obj = JSON.parse(json);
238
+ if (!obj.id || !Array.isArray(obj.subgoals)) {
239
+ throw new Error("Invalid Goal JSON: missing id or subgoals");
240
+ }
241
+ return obj;
242
+ }
243
+ // ── Private plan finding ──
244
+ getBundleId() {
245
+ return this.worldModel.getState().focusedApp?.bundleId ?? "";
246
+ }
247
+ findPlaybookPlan(description) {
248
+ // Try task-based match only — don't unconditionally use the active playbook
249
+ // here, because that would shadow findFlowPlan() which also uses the active
250
+ // playbook's flows. The active playbook's steps are only a good match if
251
+ // matchByTask explicitly selects it.
252
+ const playbook = this.playbookStore.matchByTask(description, this.getBundleId());
253
+ if (playbook && playbook.steps.length > 0) {
254
+ return playbookToPlan(playbook, this.config, this.learningEngine, this.getBundleId());
255
+ }
256
+ return null;
257
+ }
258
+ findStrategyPlan(description) {
259
+ const strategies = this.memory.recallStrategies(description, 3, this.getBundleId());
260
+ if (strategies.length === 0)
261
+ return null;
262
+ // Prefer strategies whose task description mentions the current app
263
+ const currentApp = this.worldModel?.getState()?.focusedApp?.appName?.toLowerCase() ?? "";
264
+ const currentBundle = this.worldModel?.getState()?.focusedApp?.bundleId?.toLowerCase() ?? "";
265
+ let best = strategies[0];
266
+ for (const s of strategies) {
267
+ const taskLower = s.task.toLowerCase();
268
+ if (currentApp && taskLower.includes(currentApp)) {
269
+ best = s;
270
+ break;
271
+ }
272
+ if (currentBundle && taskLower.includes(currentBundle)) {
273
+ best = s;
274
+ break;
275
+ }
276
+ }
277
+ if (best.score < 0.6)
278
+ return null;
279
+ // Reject strategies that only contain trivial steps (focus/screenshot/apps/windows)
280
+ // — these are artifacts from testing, not useful automation plans
281
+ const TRIVIAL_TOOLS = new Set(["focus", "screenshot", "apps", "windows", "screenshot_file"]);
282
+ const hasSubstantiveStep = best.steps.some((s) => !TRIVIAL_TOOLS.has(s.tool));
283
+ if (!hasSubstantiveStep)
284
+ return null;
285
+ return strategyToPlan(best, this.config, this.learningEngine, this.getBundleId());
286
+ }
287
+ findFlowPlan(description) {
288
+ // Collect all playbooks that have flows: active playbook first, then ALL loaded playbooks
289
+ const active = this.contextTracker.getActivePlaybook();
290
+ const allPlaybooks = this.playbookStore.getAll();
291
+ // Find best matching flow across ALL playbooks with flows
292
+ // Filter out common automation verbs/nouns that match almost any flow
293
+ const FLOW_STOPWORDS = new Set([
294
+ "open", "close", "click", "set", "get", "the", "and", "for", "from",
295
+ "into", "with", "then", "this", "that", "use", "run", "start", "stop",
296
+ "new", "add", "app", "settings", "window", "button", "text", "page",
297
+ "file", "menu", "tab", "navigate", "type", "select", "find", "wait",
298
+ "send", "save", "copy", "paste", "delete", "create", "edit", "view",
299
+ "show", "hide", "move", "drag", "drop", "enter", "press", "about",
300
+ "input", "form", "link", "image", "video", "upload", "download",
301
+ ]);
302
+ const tokens = description.toLowerCase().split(/\W+/).filter((w) => w.length >= 3 && !FLOW_STOPWORDS.has(w));
303
+ // If all tokens are stopwords, there's nothing meaningful to match against flows
304
+ if (tokens.length === 0)
305
+ return null;
306
+ let bestFlow = null;
307
+ let bestScore = 0;
308
+ // Platform-aware scoring: detect current app for flow preference
309
+ const state = this.worldModel.getState();
310
+ const focusedBundle = state.focusedApp?.bundleId?.toLowerCase() ?? "";
311
+ const focusedApp = state.focusedApp?.appName?.toLowerCase() ?? "";
312
+ // Map known apps to flow name prefixes they should prefer
313
+ const isSafari = focusedBundle.includes("safari") || focusedApp === "safari";
314
+ const isChrome = focusedBundle.includes("chrome") || focusedApp === "chrome";
315
+ const isBrowser = isSafari || isChrome;
316
+ // Search active playbook first (gets priority via +2 bonus)
317
+ const candidates = active?.flows ? [{ pb: active, bonus: 2 }] : [];
318
+ for (const pb of allPlaybooks) {
319
+ if (pb.flows && pb !== active) {
320
+ candidates.push({ pb, bonus: 0 });
321
+ }
322
+ }
323
+ for (const { pb, bonus } of candidates) {
324
+ if (!pb.flows)
325
+ continue;
326
+ for (const [name, flow] of Object.entries(pb.flows)) {
327
+ if (!Array.isArray(flow?.steps))
328
+ continue;
329
+ const flowNameLower = name.toLowerCase();
330
+ const flowTokens = flowNameLower.split(/[_\-\s]+/);
331
+ const allTokens = [
332
+ ...flowTokens,
333
+ ...flow.steps.join(" ").toLowerCase().split(/\W+/),
334
+ ];
335
+ let score = bonus;
336
+ for (const t of tokens) {
337
+ if (allTokens.some((ft) => ft.includes(t)))
338
+ score++;
339
+ }
340
+ // Platform-aware boost: prefer flows that match the focused app
341
+ if (isSafari && flowNameLower.includes("safari"))
342
+ score += 3;
343
+ if (isChrome && flowNameLower.includes("browser"))
344
+ score += 3;
345
+ // Penalize browser/CDP flows when in Safari (no CDP available)
346
+ if (isSafari && flowNameLower.includes("browser"))
347
+ score -= 2;
348
+ // Penalize safari-specific flows when in Chrome (use CDP instead)
349
+ if (isChrome && flowNameLower.includes("safari"))
350
+ score -= 2;
351
+ // Penalize desktop_automation generic flows when a specific flow exists
352
+ if (flowNameLower === "desktop_automation" && bestScore > 0)
353
+ score -= 1;
354
+ if (score > bestScore) {
355
+ bestScore = score;
356
+ bestFlow = { name, flow };
357
+ }
358
+ }
359
+ }
360
+ if (!bestFlow || bestScore <= 0)
361
+ return null;
362
+ // Require at least 40% of meaningful goal tokens to match flow tokens,
363
+ // with a minimum absolute score of 2, to avoid spurious matches from
364
+ // common verbs hitting unrelated flows.
365
+ // Subtract the active-playbook bonus before comparing — the bonus is a
366
+ // tiebreaker, not evidence that the goal text matches the flow.
367
+ const activeBonus = (active?.flows && bestFlow && active.flows[bestFlow.name]) ? 2 : 0;
368
+ const contentScore = bestScore - activeBonus;
369
+ const minScore = Math.max(2, Math.ceil(tokens.length * 0.4));
370
+ if (contentScore < minScore)
371
+ return null;
372
+ return flowToPlan(bestFlow.name, bestFlow.flow, this.config, this.getRuntimeContext());
373
+ }
374
+ getRuntimeContext() {
375
+ const state = this.worldModel.getState();
376
+ return {
377
+ pid: state.focusedApp?.pid,
378
+ windowId: state.focusedWindowId ?? undefined,
379
+ bundleId: state.focusedApp?.bundleId,
380
+ };
381
+ }
382
+ createLLMPlanStub(description) {
383
+ const step = {
384
+ tool: "",
385
+ params: {},
386
+ expectedPostcondition: null,
387
+ timeout: this.config.defaultStepTimeout,
388
+ fallbackTool: null,
389
+ requiresLLM: true,
390
+ status: "pending",
391
+ description,
392
+ };
393
+ return {
394
+ steps: [step],
395
+ currentStepIndex: 0,
396
+ confidence: 0.3,
397
+ source: "llm",
398
+ sourceId: null,
399
+ };
400
+ }
401
+ /**
402
+ * Build a runtime context summary for the LLM prompt.
403
+ * Includes focused app, window, and visible controls.
404
+ */
405
+ buildRuntimeContextPrompt() {
406
+ const state = this.worldModel.getState();
407
+ const lines = [];
408
+ if (state.focusedApp) {
409
+ lines.push(`Focused app: ${state.focusedApp.appName} (${state.focusedApp.bundleId}), PID: ${state.focusedApp.pid}`);
410
+ }
411
+ if (state.focusedWindowId !== null) {
412
+ lines.push(`Window ID: ${state.focusedWindowId}`);
413
+ }
414
+ // Include visible controls from world model (top 20)
415
+ if (state.focusedWindowId !== null) {
416
+ const win = state.windows.get(state.focusedWindowId);
417
+ if (win && win.controls.size > 0) {
418
+ const controls = [...win.controls.values()]
419
+ .slice(0, 20)
420
+ .map((c) => `${c.role}:${c.label.value ?? ""}`)
421
+ .filter((s) => s.length > 1);
422
+ if (controls.length > 0) {
423
+ lines.push(`Visible controls: ${controls.join(", ")}`);
424
+ }
425
+ }
426
+ }
427
+ // Include active reference context if available
428
+ const active = this.contextTracker.getActivePlaybook();
429
+ if (active) {
430
+ lines.push(`Platform reference loaded: ${active.platform ?? active.id}`);
431
+ }
432
+ return lines.length > 0 ? `\nRuntime context:\n${lines.join("\n")}` : "";
433
+ }
434
+ async createLLMPlan(description) {
435
+ const apiKey = process.env["ANTHROPIC_API_KEY"];
436
+ if (!apiKey || !this.toolRegistry) {
437
+ return this.createLLMPlanStub(description);
438
+ }
439
+ try {
440
+ const toolNames = this.toolRegistry.getToolNames();
441
+ const runtimeCtx = this.buildRuntimeContextPrompt();
442
+ const controller = new AbortController();
443
+ const timeout = setTimeout(() => controller.abort(), 15_000);
444
+ const prompt = [
445
+ "You are a desktop automation planner. Generate concrete tool steps with real parameter values.",
446
+ runtimeCtx,
447
+ "",
448
+ "Key tool signatures:",
449
+ "- screenshot() → captures current screen",
450
+ "- click_text(windowId: number, text: string, prefer?: 'first'|'largest'|'topmost'|'leftmost') → OCR-click",
451
+ "- ui_press(pid: number, title: string, role?: string) → AX button press",
452
+ "- type_text(text: string) → keyboard typing",
453
+ "- key(key: string) → keyboard shortcut (e.g. 'cmd+a', 'Return')",
454
+ "- browser_navigate(url: string) → navigate browser",
455
+ "- browser_click(selector: string) → click element in browser",
456
+ "- browser_type(selector: string, text: string) → type in browser input",
457
+ "- focus(appName: string) → focus app window",
458
+ "- launch(bundleId: string) → launch app",
459
+ "",
460
+ `All available tools: ${toolNames.join(", ")}`,
461
+ "",
462
+ `Goal: ${description}`,
463
+ "",
464
+ 'Return ONLY a JSON array of objects: [{"tool": "...", "params": {...}, "description": "..."}]',
465
+ "Use concrete values from the runtime context above (pid, windowId, etc).",
466
+ "No markdown, no explanation.",
467
+ ].join("\n");
468
+ const response = await fetch("https://api.anthropic.com/v1/messages", {
469
+ method: "POST",
470
+ headers: {
471
+ "Content-Type": "application/json",
472
+ "x-api-key": apiKey,
473
+ "anthropic-version": "2023-06-01",
474
+ },
475
+ body: JSON.stringify({
476
+ model: "claude-haiku-4-5-20251001",
477
+ max_tokens: 1000,
478
+ messages: [{ role: "user", content: prompt }],
479
+ }),
480
+ signal: controller.signal,
481
+ });
482
+ clearTimeout(timeout);
483
+ if (!response.ok) {
484
+ return this.createLLMPlanStub(description);
485
+ }
486
+ const data = await response.json();
487
+ const text = data.content?.[0]?.text?.trim();
488
+ if (!text)
489
+ return this.createLLMPlanStub(description);
490
+ // Extract JSON array from response (may be wrapped in markdown code blocks)
491
+ const jsonMatch = text.match(/\[[\s\S]*\]/);
492
+ if (!jsonMatch)
493
+ return this.createLLMPlanStub(description);
494
+ const parsed = JSON.parse(jsonMatch[0]);
495
+ if (!Array.isArray(parsed) || parsed.length === 0) {
496
+ return this.createLLMPlanStub(description);
497
+ }
498
+ const steps = parsed.map((s) => ({
499
+ tool: s.tool ?? "",
500
+ params: s.params ?? {},
501
+ expectedPostcondition: null,
502
+ timeout: this.config.defaultStepTimeout,
503
+ fallbackTool: null,
504
+ requiresLLM: !s.tool,
505
+ status: "pending",
506
+ description: s.description ?? s.tool ?? description,
507
+ }));
508
+ return {
509
+ steps,
510
+ currentStepIndex: 0,
511
+ confidence: 0.5,
512
+ source: "llm",
513
+ sourceId: null,
514
+ };
515
+ }
516
+ catch {
517
+ return this.createLLMPlanStub(description);
518
+ }
519
+ }
520
+ }