screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,327 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ import fs from "node:fs";
18
+ import path from "node:path";
19
+ import { detectBlockers } from "./detectors.js";
20
+ import { getBuiltinStrategies, parseReferenceStrategies, buildStrategyWithContext, } from "./strategies.js";
21
+ const DEFAULT_CONFIG = {
22
+ referencesDir: path.join(process.cwd(), "references"),
23
+ };
24
+ const STRATEGY_COOLDOWN_MS = 30_000; // 30 seconds
25
+ export class RecoveryEngine {
26
+ worldModel;
27
+ executeTool;
28
+ memory;
29
+ config;
30
+ referenceCache = new Map();
31
+ /** Map of "blockerType:strategyId" → cooldown entry */
32
+ strategyCooldowns = new Map();
33
+ learningEngine = null;
34
+ constructor(worldModel, executeTool, memory, config) {
35
+ this.worldModel = worldModel;
36
+ this.executeTool = executeTool;
37
+ this.memory = memory;
38
+ this.config = { ...DEFAULT_CONFIG, ...config };
39
+ }
40
+ /**
41
+ * Inject the learning engine for recording recovery outcomes.
42
+ * Called after both engines are constructed (avoids circular dependency).
43
+ */
44
+ setLearningEngine(engine) {
45
+ this.learningEngine = engine;
46
+ }
47
+ /**
48
+ * Get the current status of the recovery engine.
49
+ */
50
+ getStatus() {
51
+ return {
52
+ cooldownCount: this.strategyCooldowns.size,
53
+ referenceCacheSize: this.referenceCache.size,
54
+ learningEngineConnected: this.learningEngine !== null,
55
+ };
56
+ }
57
+ /**
58
+ * Update the default recovery budget configuration.
59
+ */
60
+ configure(partial) {
61
+ Object.assign(this.config, partial);
62
+ }
63
+ /**
64
+ * Attempt to recover from a step failure.
65
+ * Called by PlanExecutor after a step fails, before replanning.
66
+ */
67
+ async attemptRecovery(failedStepError, expectedBundleId, budget) {
68
+ const budgetStart = Date.now();
69
+ // Detect blockers
70
+ const blockers = detectBlockers(this.worldModel, failedStepError, expectedBundleId);
71
+ // Try strategies for each blocker in priority order
72
+ for (const blocker of blockers) {
73
+ if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
74
+ return { recovered: false, reason: "budget_exhausted" };
75
+ }
76
+ const strategies = this.selectStrategies(blocker, budget);
77
+ for (const strategy of strategies) {
78
+ if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
79
+ return { recovered: false, reason: "budget_exhausted" };
80
+ }
81
+ if (budget.usedStrategyIds.size >= budget.maxStrategies) {
82
+ return { recovered: false, reason: "budget_exhausted" };
83
+ }
84
+ budget.usedStrategyIds.add(strategy.id);
85
+ const outcome = await this.executeStrategy(strategy, blocker, budgetStart, budget);
86
+ if (outcome.recovered)
87
+ return outcome;
88
+ }
89
+ }
90
+ return { recovered: false, reason: "all_strategies_failed" };
91
+ }
92
+ /**
93
+ * Select strategies for a blocker: reference-based first, then built-in.
94
+ * Excludes already-used strategies.
95
+ */
96
+ selectStrategies(blocker, budget) {
97
+ const candidates = [];
98
+ // Reference strategies first (app-specific)
99
+ if (blocker.bundleId) {
100
+ const refErrors = this.loadReferenceErrors(blocker.bundleId);
101
+ candidates.push(...parseReferenceStrategies(refErrors, blocker.type));
102
+ }
103
+ // Then built-in
104
+ candidates.push(...getBuiltinStrategies(blocker.type));
105
+ const now = Date.now();
106
+ const available = candidates.filter((s) => {
107
+ if (budget.usedStrategyIds.has(s.id))
108
+ return false;
109
+ // Check cooldown — skip strategies that failed recently for this blocker type
110
+ const cooldownKey = `${blocker.type}:${s.id}`;
111
+ const entry = this.strategyCooldowns.get(cooldownKey);
112
+ if (entry && now - entry.failedAt < STRATEGY_COOLDOWN_MS)
113
+ return false;
114
+ return true;
115
+ });
116
+ // Re-order by learning engine ranking if available
117
+ if (this.learningEngine && blocker.bundleId) {
118
+ const ranked = this.learningEngine.rankRecoveryStrategies(blocker.type, blocker.bundleId);
119
+ if (ranked.length > 0) {
120
+ const rankMap = new Map(ranked.map((r, i) => [r.strategyId, i]));
121
+ available.sort((a, b) => {
122
+ const ra = rankMap.get(a.id) ?? 999;
123
+ const rb = rankMap.get(b.id) ?? 999;
124
+ return ra - rb;
125
+ });
126
+ }
127
+ }
128
+ return available;
129
+ }
130
+ /**
131
+ * Execute a strategy's steps and verify recovery.
132
+ */
133
+ async executeStrategy(rawStrategy, blocker, budgetStart, budget) {
134
+ const start = Date.now();
135
+ const strategy = buildStrategyWithContext(rawStrategy, blocker.bundleId, blocker.pid);
136
+ // Escalation strategies (empty steps) — cannot auto-recover
137
+ if (strategy.steps.length === 0) {
138
+ this.recordEvent({
139
+ timestamp: new Date().toISOString(),
140
+ blocker,
141
+ strategyId: strategy.id,
142
+ strategyLabel: strategy.label,
143
+ success: false,
144
+ durationMs: 0,
145
+ error: "escalation_required",
146
+ });
147
+ return { recovered: false, reason: "all_strategies_failed" };
148
+ }
149
+ // Execute each step
150
+ for (const step of strategy.steps) {
151
+ if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
152
+ return { recovered: false, reason: "budget_exhausted" };
153
+ }
154
+ try {
155
+ const result = await this.executeTool(step.tool, step.params);
156
+ if (!result.ok) {
157
+ this.recordEvent({
158
+ timestamp: new Date().toISOString(),
159
+ blocker,
160
+ strategyId: strategy.id,
161
+ strategyLabel: strategy.label,
162
+ success: false,
163
+ durationMs: Date.now() - start,
164
+ error: result.error ?? "tool failed",
165
+ });
166
+ this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
167
+ return { recovered: false, reason: "all_strategies_failed" };
168
+ }
169
+ }
170
+ catch (err) {
171
+ this.recordEvent({
172
+ timestamp: new Date().toISOString(),
173
+ blocker,
174
+ strategyId: strategy.id,
175
+ strategyLabel: strategy.label,
176
+ success: false,
177
+ durationMs: Date.now() - start,
178
+ error: err instanceof Error ? err.message : String(err),
179
+ });
180
+ this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
181
+ return { recovered: false, reason: "all_strategies_failed" };
182
+ }
183
+ }
184
+ // Verify recovery
185
+ await sleep(300);
186
+ const verified = this.verifyRecovery(blocker);
187
+ const durationMs = Date.now() - start;
188
+ this.recordEvent({
189
+ timestamp: new Date().toISOString(),
190
+ blocker,
191
+ strategyId: strategy.id,
192
+ strategyLabel: strategy.label,
193
+ success: verified,
194
+ durationMs,
195
+ error: verified ? null : "verification failed",
196
+ });
197
+ // Feed learning engine with recovery outcome
198
+ if (this.learningEngine && blocker.bundleId) {
199
+ this.learningEngine.recordRecoveryOutcome({
200
+ bundleId: blocker.bundleId,
201
+ blockerType: blocker.type,
202
+ strategyId: strategy.id,
203
+ success: verified,
204
+ durationMs,
205
+ });
206
+ }
207
+ if (verified) {
208
+ // Clear cooldown on success
209
+ this.strategyCooldowns.delete(`${blocker.type}:${strategy.id}`);
210
+ return { recovered: true, strategyId: strategy.id, durationMs };
211
+ }
212
+ // Record cooldown for failed strategy
213
+ this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
214
+ return { recovered: false, reason: "all_strategies_failed" };
215
+ }
216
+ /**
217
+ * Verify the blocker is resolved by re-checking world model state.
218
+ */
219
+ verifyRecovery(blocker) {
220
+ switch (blocker.type) {
221
+ case "unexpected_dialog":
222
+ case "permission_dialog":
223
+ case "login_required":
224
+ case "captcha": {
225
+ const dialogs = this.worldModel.getActiveDialogs();
226
+ if (blocker.dialogTitle) {
227
+ return !dialogs.some((d) => d.title === blocker.dialogTitle);
228
+ }
229
+ return dialogs.length === 0;
230
+ }
231
+ case "focus_lost": {
232
+ if (!blocker.bundleId)
233
+ return false;
234
+ return this.worldModel.getState().focusedApp?.bundleId === blocker.bundleId;
235
+ }
236
+ case "app_crashed": {
237
+ return this.worldModel.getState().windows.size > 0;
238
+ }
239
+ case "element_gone": {
240
+ // The element should be back — verify focused window has controls
241
+ const win = this.worldModel.getFocusedWindow();
242
+ if (!win)
243
+ return false;
244
+ return win.controls.size > 0;
245
+ }
246
+ case "selector_drift": {
247
+ // After recovery, controls should be findable — verify the focused window
248
+ // has recently updated controls (not all stale)
249
+ const win = this.worldModel.getFocusedWindow();
250
+ if (!win)
251
+ return false;
252
+ if (win.controls.size === 0)
253
+ return false;
254
+ const stale = this.worldModel.getStaleControls(5_000);
255
+ return stale.length < win.controls.size;
256
+ }
257
+ case "unknown_state": {
258
+ // State should be less stale after recovery — check stale count is low
259
+ const state = this.worldModel.getState();
260
+ if (state.windows.size === 0)
261
+ return false;
262
+ const stale = this.worldModel.getStaleControls(5_000);
263
+ let totalControls = 0;
264
+ for (const w of state.windows.values()) {
265
+ totalControls += w.controls.size;
266
+ }
267
+ // Pass if fewer than half of controls are stale
268
+ return totalControls > 0 && stale.length < totalControls / 2;
269
+ }
270
+ case "loading_stuck": {
271
+ // UI should have changed — verify state was updated recently (within 2s)
272
+ const state = this.worldModel.getState();
273
+ const ageMs = Date.now() - new Date(state.updatedAt).getTime();
274
+ return ageMs < 2_000;
275
+ }
276
+ case "network_error":
277
+ case "rate_limited": {
278
+ // Transient errors — verify state was refreshed recently (within 3s)
279
+ const state = this.worldModel.getState();
280
+ const ageMs = Date.now() - new Date(state.updatedAt).getTime();
281
+ return ageMs < 3_000;
282
+ }
283
+ }
284
+ }
285
+ /**
286
+ * Load and cache reference errors for a bundleId.
287
+ */
288
+ loadReferenceErrors(bundleId) {
289
+ const cached = this.referenceCache.get(bundleId);
290
+ if (cached !== undefined)
291
+ return cached;
292
+ let errors = [];
293
+ try {
294
+ const files = fs.readdirSync(this.config.referencesDir);
295
+ for (const file of files) {
296
+ if (!file.endsWith(".json"))
297
+ continue;
298
+ try {
299
+ // Guard against oversized files (same 10MB limit as LearningEngine)
300
+ const filePath = path.join(this.config.referencesDir, file);
301
+ const stat = fs.statSync(filePath);
302
+ if (stat.size > 10 * 1024 * 1024)
303
+ continue;
304
+ const raw = fs.readFileSync(filePath, "utf-8");
305
+ const ref = JSON.parse(raw);
306
+ if (ref.bundleId === bundleId && Array.isArray(ref.errors)) {
307
+ errors = ref.errors.filter((e) => typeof e.error === "string" && typeof e.solution === "string");
308
+ break;
309
+ }
310
+ }
311
+ catch { /* skip malformed */ }
312
+ }
313
+ }
314
+ catch { /* dir doesn't exist */ }
315
+ this.referenceCache.set(bundleId, errors);
316
+ return errors;
317
+ }
318
+ recordEvent(event) {
319
+ try {
320
+ this.memory.recordError(`recovery:${event.strategyId}`, event.error ?? "", event.success ? event.strategyLabel : null, event.blocker.bundleId ?? undefined);
321
+ }
322
+ catch { /* best-effort */ }
323
+ }
324
+ }
325
+ function sleep(ms) {
326
+ return new Promise((resolve) => setTimeout(resolve, ms));
327
+ }
@@ -0,0 +1,20 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export { RecoveryEngine } from "./engine.js";
18
+ export { detectBlockers } from "./detectors.js";
19
+ export { getBuiltinStrategies, parseReferenceStrategies } from "./strategies.js";
20
+ export { DEFAULT_RECOVERY_BUDGET } from "./types.js";
@@ -0,0 +1,274 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * Built-in recovery strategies keyed by BlockerType, in priority order.
19
+ * Strategies with empty steps are escalations — they signal that human
20
+ * intervention is needed.
21
+ */
22
+ const BUILTIN_STRATEGIES = [
23
+ // ── unexpected_dialog ──
24
+ {
25
+ id: "dismiss_dialog_cancel",
26
+ blockerType: "unexpected_dialog",
27
+ label: "Dismiss dialog via Cancel",
28
+ steps: [{ tool: "click_text", params: { text: "Cancel" }, description: "Click Cancel" }],
29
+ postcondition: null,
30
+ source: "builtin",
31
+ },
32
+ {
33
+ id: "dismiss_dialog_ok",
34
+ blockerType: "unexpected_dialog",
35
+ label: "Dismiss dialog via OK",
36
+ steps: [{ tool: "click_text", params: { text: "OK" }, description: "Click OK" }],
37
+ postcondition: null,
38
+ source: "builtin",
39
+ },
40
+ {
41
+ id: "dismiss_dialog_escape",
42
+ blockerType: "unexpected_dialog",
43
+ label: "Dismiss dialog via Escape",
44
+ steps: [{ tool: "key", params: { combo: "Escape" }, description: "Press Escape" }],
45
+ postcondition: null,
46
+ source: "builtin",
47
+ },
48
+ // ── permission_dialog ──
49
+ {
50
+ id: "grant_permission_allow",
51
+ blockerType: "permission_dialog",
52
+ label: "Grant permission via Allow",
53
+ steps: [{ tool: "click_text", params: { text: "Allow" }, description: "Click Allow" }],
54
+ postcondition: null,
55
+ source: "builtin",
56
+ },
57
+ {
58
+ id: "grant_permission_ok",
59
+ blockerType: "permission_dialog",
60
+ label: "Grant permission via OK",
61
+ steps: [{ tool: "click_text", params: { text: "OK" }, description: "Click OK" }],
62
+ postcondition: null,
63
+ source: "builtin",
64
+ },
65
+ {
66
+ id: "safari_allow_popup",
67
+ blockerType: "permission_dialog",
68
+ label: "Allow Safari popup via notification bar",
69
+ steps: [
70
+ { tool: "key", params: { combo: "cmd+z" }, description: "Undo popup block (shows notification)" },
71
+ { tool: "screenshot", params: {}, description: "Check for popup notification bar" },
72
+ ],
73
+ postcondition: null,
74
+ source: "builtin",
75
+ },
76
+ {
77
+ id: "safari_enable_popups_applescript",
78
+ blockerType: "permission_dialog",
79
+ label: "Enable popups for site via AppleScript",
80
+ steps: [
81
+ { tool: "applescript", params: { script: 'tell application "Safari" to activate' }, description: "Activate Safari" },
82
+ { tool: "key", params: { combo: "cmd+," }, description: "Open Safari preferences" },
83
+ { tool: "screenshot", params: {}, description: "Navigate to popup settings" },
84
+ ],
85
+ postcondition: null,
86
+ source: "builtin",
87
+ },
88
+ // ── focus_lost ──
89
+ {
90
+ id: "refocus_app",
91
+ blockerType: "focus_lost",
92
+ label: "Refocus target application",
93
+ steps: [{ tool: "focus", params: {}, description: "Focus target app" }],
94
+ postcondition: null,
95
+ source: "builtin",
96
+ },
97
+ // ── app_crashed ──
98
+ {
99
+ id: "relaunch_app",
100
+ blockerType: "app_crashed",
101
+ label: "Relaunch crashed application",
102
+ steps: [
103
+ { tool: "launch", params: {}, description: "Launch application" },
104
+ { tool: "screenshot", params: {}, description: "Wait for app ready" },
105
+ ],
106
+ postcondition: null,
107
+ source: "builtin",
108
+ },
109
+ // ── element_gone ──
110
+ {
111
+ id: "rescan_ax_tree",
112
+ blockerType: "element_gone",
113
+ label: "Rescan AX tree to relocate element",
114
+ steps: [{ tool: "ui_tree", params: {}, description: "Refresh AX tree" }],
115
+ postcondition: null,
116
+ source: "builtin",
117
+ },
118
+ // ── selector_drift ──
119
+ {
120
+ id: "rescan_for_drift",
121
+ blockerType: "selector_drift",
122
+ label: "Rescan AX tree for selector drift",
123
+ steps: [{ tool: "ui_tree", params: {}, description: "Rescan for drift" }],
124
+ postcondition: null,
125
+ source: "builtin",
126
+ },
127
+ // ── loading_stuck ──
128
+ {
129
+ id: "wait_for_load",
130
+ blockerType: "loading_stuck",
131
+ label: "Wait and recheck",
132
+ steps: [{ tool: "screenshot", params: {}, description: "Wait via screenshot" }],
133
+ postcondition: null,
134
+ source: "builtin",
135
+ },
136
+ // ── network_error ──
137
+ {
138
+ id: "reload_page",
139
+ blockerType: "network_error",
140
+ label: "Reload page",
141
+ steps: [{ tool: "key", params: { combo: "cmd+r" }, description: "Reload page" }],
142
+ postcondition: null,
143
+ source: "builtin",
144
+ },
145
+ // ── unknown_state ──
146
+ {
147
+ id: "full_perception_refresh",
148
+ blockerType: "unknown_state",
149
+ label: "Full perception refresh",
150
+ steps: [
151
+ { tool: "screenshot", params: {}, description: "Take screenshot" },
152
+ { tool: "ui_tree", params: {}, description: "Refresh AX tree" },
153
+ ],
154
+ postcondition: null,
155
+ source: "builtin",
156
+ },
157
+ // ── Escalation-only (no automated recovery possible) ──
158
+ {
159
+ id: "escalate_login",
160
+ blockerType: "login_required",
161
+ label: "Escalate: login required",
162
+ steps: [],
163
+ postcondition: null,
164
+ source: "builtin",
165
+ },
166
+ {
167
+ id: "escalate_captcha",
168
+ blockerType: "captcha",
169
+ label: "Escalate: captcha",
170
+ steps: [],
171
+ postcondition: null,
172
+ source: "builtin",
173
+ },
174
+ {
175
+ id: "escalate_rate_limited",
176
+ blockerType: "rate_limited",
177
+ label: "Escalate: rate limited",
178
+ steps: [],
179
+ postcondition: null,
180
+ source: "builtin",
181
+ },
182
+ ];
183
+ /** Return built-in strategies matching a blocker type, in priority order. */
184
+ export function getBuiltinStrategies(blockerType) {
185
+ return BUILTIN_STRATEGIES.filter((s) => s.blockerType === blockerType);
186
+ }
187
+ /**
188
+ * Parse a solution text into concrete recovery steps.
189
+ * Pattern-matches common instruction phrases to map to tool calls.
190
+ * Falls back to screenshot with full solution as description.
191
+ */
192
+ export function parseSolutionToSteps(solution) {
193
+ const steps = [];
194
+ // Split multi-sentence solutions into individual instructions
195
+ const sentences = solution.split(/[.;]\s+/).filter((s) => s.trim().length > 5);
196
+ if (sentences.length === 0)
197
+ sentences.push(solution);
198
+ for (const sentence of sentences) {
199
+ const lower = sentence.toLowerCase();
200
+ const step = matchSolutionSentence(lower, sentence);
201
+ steps.push(step);
202
+ }
203
+ return steps;
204
+ }
205
+ function matchSolutionSentence(lower, original) {
206
+ // Click/tap/select patterns
207
+ const clickMatch = lower.match(/(?:click|tap|select|choose|press)\s+(?:the\s+)?['"]?([^'",.]+)['"]?/i);
208
+ if (clickMatch && !/(?:cmd|ctrl|alt|shift|command)/i.test(clickMatch[1])) {
209
+ // Trim trailing filler words like "button and wait", "and then", "to confirm"
210
+ let clickText = clickMatch[1].trim()
211
+ .replace(/\s+(?:button|link|icon|tab|menu|option)(?:\s+.*)?$/i, "")
212
+ .replace(/\s+(?:and\s+.*|to\s+.*|in\s+.*|on\s+.*|from\s+.*|into\s+.*)$/i, "")
213
+ .trim();
214
+ if (!clickText)
215
+ clickText = clickMatch[1].trim();
216
+ return { tool: "click_text", params: { text: clickText }, description: original };
217
+ }
218
+ // Keyboard shortcut patterns
219
+ const keyMatch = lower.match(/(?:press|use|hit)\s+(?:the\s+)?(?:shortcut\s+)?((?:cmd|ctrl|alt|shift|command|control|option)[\s+]+\w+)/i);
220
+ if (keyMatch) {
221
+ const key = keyMatch[1].replace(/\s+/g, "+").replace(/command/i, "Cmd").replace(/control/i, "Ctrl");
222
+ return { tool: "key", params: { combo: key }, description: original };
223
+ }
224
+ // Navigate/go to/open URL patterns
225
+ const navMatch = lower.match(/(?:navigate\s+to|go\s+to|open|visit)\s+(?:the\s+)?(?:url\s+)?(https?:\/\/\S+)/i);
226
+ if (navMatch) {
227
+ const url = navMatch[1];
228
+ // Reject javascript: and data: URLs
229
+ if (/^javascript:/i.test(url) || /^data:/i.test(url)) {
230
+ return { tool: "screenshot", params: {}, description: `Blocked unsafe URL: ${original}` };
231
+ }
232
+ return { tool: "browser_navigate", params: { url }, description: original };
233
+ }
234
+ // Type/enter/input patterns
235
+ const typeMatch = lower.match(/(?:type|enter|input)\s+['"]?([^'",.]+)['"]?/i);
236
+ if (typeMatch) {
237
+ return { tool: "type_text", params: { text: typeMatch[1].trim() }, description: original };
238
+ }
239
+ // Fallback: screenshot with full solution as description
240
+ return { tool: "screenshot", params: {}, description: `Reference solution: ${original}` };
241
+ }
242
+ /**
243
+ * Parse app-specific recovery strategies from reference JSON errors.
244
+ */
245
+ export function parseReferenceStrategies(errors, blockerType) {
246
+ return errors.map((e, idx) => ({
247
+ id: `ref_${blockerType}_${idx}`,
248
+ blockerType,
249
+ label: `Reference: ${e.error}`,
250
+ steps: parseSolutionToSteps(e.solution),
251
+ postcondition: null,
252
+ source: "reference",
253
+ }));
254
+ }
255
+ /**
256
+ * Inject bundleId into strategy steps that need it (focus, launch).
257
+ * Returns a shallow clone.
258
+ */
259
+ export function buildStrategyWithContext(strategy, bundleId, pid) {
260
+ if (!bundleId && !pid)
261
+ return strategy;
262
+ return {
263
+ ...strategy,
264
+ steps: strategy.steps.map((step) => {
265
+ if (bundleId && (step.tool === "focus" || step.tool === "launch")) {
266
+ return { ...step, params: { ...step.params, bundleId } };
267
+ }
268
+ if (pid != null && step.tool === "ui_tree") {
269
+ return { ...step, params: { ...step.params, pid } };
270
+ }
271
+ return step;
272
+ }),
273
+ };
274
+ }
@@ -0,0 +1,20 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export const DEFAULT_RECOVERY_BUDGET = {
18
+ maxRecoveryTimeMs: 30_000,
19
+ maxStrategies: 3,
20
+ };