screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { detectBlockers } from "./detectors.js";
|
|
20
|
+
import { getBuiltinStrategies, parseReferenceStrategies, buildStrategyWithContext, } from "./strategies.js";
|
|
21
|
+
const DEFAULT_CONFIG = {
|
|
22
|
+
referencesDir: path.join(process.cwd(), "references"),
|
|
23
|
+
};
|
|
24
|
+
const STRATEGY_COOLDOWN_MS = 30_000; // 30 seconds
|
|
25
|
+
export class RecoveryEngine {
|
|
26
|
+
worldModel;
|
|
27
|
+
executeTool;
|
|
28
|
+
memory;
|
|
29
|
+
config;
|
|
30
|
+
referenceCache = new Map();
|
|
31
|
+
/** Map of "blockerType:strategyId" → cooldown entry */
|
|
32
|
+
strategyCooldowns = new Map();
|
|
33
|
+
learningEngine = null;
|
|
34
|
+
constructor(worldModel, executeTool, memory, config) {
|
|
35
|
+
this.worldModel = worldModel;
|
|
36
|
+
this.executeTool = executeTool;
|
|
37
|
+
this.memory = memory;
|
|
38
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Inject the learning engine for recording recovery outcomes.
|
|
42
|
+
* Called after both engines are constructed (avoids circular dependency).
|
|
43
|
+
*/
|
|
44
|
+
setLearningEngine(engine) {
|
|
45
|
+
this.learningEngine = engine;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Get the current status of the recovery engine.
|
|
49
|
+
*/
|
|
50
|
+
getStatus() {
|
|
51
|
+
return {
|
|
52
|
+
cooldownCount: this.strategyCooldowns.size,
|
|
53
|
+
referenceCacheSize: this.referenceCache.size,
|
|
54
|
+
learningEngineConnected: this.learningEngine !== null,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Update the default recovery budget configuration.
|
|
59
|
+
*/
|
|
60
|
+
configure(partial) {
|
|
61
|
+
Object.assign(this.config, partial);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Attempt to recover from a step failure.
|
|
65
|
+
* Called by PlanExecutor after a step fails, before replanning.
|
|
66
|
+
*/
|
|
67
|
+
async attemptRecovery(failedStepError, expectedBundleId, budget) {
|
|
68
|
+
const budgetStart = Date.now();
|
|
69
|
+
// Detect blockers
|
|
70
|
+
const blockers = detectBlockers(this.worldModel, failedStepError, expectedBundleId);
|
|
71
|
+
// Try strategies for each blocker in priority order
|
|
72
|
+
for (const blocker of blockers) {
|
|
73
|
+
if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
|
|
74
|
+
return { recovered: false, reason: "budget_exhausted" };
|
|
75
|
+
}
|
|
76
|
+
const strategies = this.selectStrategies(blocker, budget);
|
|
77
|
+
for (const strategy of strategies) {
|
|
78
|
+
if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
|
|
79
|
+
return { recovered: false, reason: "budget_exhausted" };
|
|
80
|
+
}
|
|
81
|
+
if (budget.usedStrategyIds.size >= budget.maxStrategies) {
|
|
82
|
+
return { recovered: false, reason: "budget_exhausted" };
|
|
83
|
+
}
|
|
84
|
+
budget.usedStrategyIds.add(strategy.id);
|
|
85
|
+
const outcome = await this.executeStrategy(strategy, blocker, budgetStart, budget);
|
|
86
|
+
if (outcome.recovered)
|
|
87
|
+
return outcome;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return { recovered: false, reason: "all_strategies_failed" };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Select strategies for a blocker: reference-based first, then built-in.
|
|
94
|
+
* Excludes already-used strategies.
|
|
95
|
+
*/
|
|
96
|
+
selectStrategies(blocker, budget) {
|
|
97
|
+
const candidates = [];
|
|
98
|
+
// Reference strategies first (app-specific)
|
|
99
|
+
if (blocker.bundleId) {
|
|
100
|
+
const refErrors = this.loadReferenceErrors(blocker.bundleId);
|
|
101
|
+
candidates.push(...parseReferenceStrategies(refErrors, blocker.type));
|
|
102
|
+
}
|
|
103
|
+
// Then built-in
|
|
104
|
+
candidates.push(...getBuiltinStrategies(blocker.type));
|
|
105
|
+
const now = Date.now();
|
|
106
|
+
const available = candidates.filter((s) => {
|
|
107
|
+
if (budget.usedStrategyIds.has(s.id))
|
|
108
|
+
return false;
|
|
109
|
+
// Check cooldown — skip strategies that failed recently for this blocker type
|
|
110
|
+
const cooldownKey = `${blocker.type}:${s.id}`;
|
|
111
|
+
const entry = this.strategyCooldowns.get(cooldownKey);
|
|
112
|
+
if (entry && now - entry.failedAt < STRATEGY_COOLDOWN_MS)
|
|
113
|
+
return false;
|
|
114
|
+
return true;
|
|
115
|
+
});
|
|
116
|
+
// Re-order by learning engine ranking if available
|
|
117
|
+
if (this.learningEngine && blocker.bundleId) {
|
|
118
|
+
const ranked = this.learningEngine.rankRecoveryStrategies(blocker.type, blocker.bundleId);
|
|
119
|
+
if (ranked.length > 0) {
|
|
120
|
+
const rankMap = new Map(ranked.map((r, i) => [r.strategyId, i]));
|
|
121
|
+
available.sort((a, b) => {
|
|
122
|
+
const ra = rankMap.get(a.id) ?? 999;
|
|
123
|
+
const rb = rankMap.get(b.id) ?? 999;
|
|
124
|
+
return ra - rb;
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return available;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Execute a strategy's steps and verify recovery.
|
|
132
|
+
*/
|
|
133
|
+
async executeStrategy(rawStrategy, blocker, budgetStart, budget) {
|
|
134
|
+
const start = Date.now();
|
|
135
|
+
const strategy = buildStrategyWithContext(rawStrategy, blocker.bundleId, blocker.pid);
|
|
136
|
+
// Escalation strategies (empty steps) — cannot auto-recover
|
|
137
|
+
if (strategy.steps.length === 0) {
|
|
138
|
+
this.recordEvent({
|
|
139
|
+
timestamp: new Date().toISOString(),
|
|
140
|
+
blocker,
|
|
141
|
+
strategyId: strategy.id,
|
|
142
|
+
strategyLabel: strategy.label,
|
|
143
|
+
success: false,
|
|
144
|
+
durationMs: 0,
|
|
145
|
+
error: "escalation_required",
|
|
146
|
+
});
|
|
147
|
+
return { recovered: false, reason: "all_strategies_failed" };
|
|
148
|
+
}
|
|
149
|
+
// Execute each step
|
|
150
|
+
for (const step of strategy.steps) {
|
|
151
|
+
if (Date.now() - budgetStart >= budget.maxRecoveryTimeMs) {
|
|
152
|
+
return { recovered: false, reason: "budget_exhausted" };
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
const result = await this.executeTool(step.tool, step.params);
|
|
156
|
+
if (!result.ok) {
|
|
157
|
+
this.recordEvent({
|
|
158
|
+
timestamp: new Date().toISOString(),
|
|
159
|
+
blocker,
|
|
160
|
+
strategyId: strategy.id,
|
|
161
|
+
strategyLabel: strategy.label,
|
|
162
|
+
success: false,
|
|
163
|
+
durationMs: Date.now() - start,
|
|
164
|
+
error: result.error ?? "tool failed",
|
|
165
|
+
});
|
|
166
|
+
this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
|
|
167
|
+
return { recovered: false, reason: "all_strategies_failed" };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
this.recordEvent({
|
|
172
|
+
timestamp: new Date().toISOString(),
|
|
173
|
+
blocker,
|
|
174
|
+
strategyId: strategy.id,
|
|
175
|
+
strategyLabel: strategy.label,
|
|
176
|
+
success: false,
|
|
177
|
+
durationMs: Date.now() - start,
|
|
178
|
+
error: err instanceof Error ? err.message : String(err),
|
|
179
|
+
});
|
|
180
|
+
this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
|
|
181
|
+
return { recovered: false, reason: "all_strategies_failed" };
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
// Verify recovery
|
|
185
|
+
await sleep(300);
|
|
186
|
+
const verified = this.verifyRecovery(blocker);
|
|
187
|
+
const durationMs = Date.now() - start;
|
|
188
|
+
this.recordEvent({
|
|
189
|
+
timestamp: new Date().toISOString(),
|
|
190
|
+
blocker,
|
|
191
|
+
strategyId: strategy.id,
|
|
192
|
+
strategyLabel: strategy.label,
|
|
193
|
+
success: verified,
|
|
194
|
+
durationMs,
|
|
195
|
+
error: verified ? null : "verification failed",
|
|
196
|
+
});
|
|
197
|
+
// Feed learning engine with recovery outcome
|
|
198
|
+
if (this.learningEngine && blocker.bundleId) {
|
|
199
|
+
this.learningEngine.recordRecoveryOutcome({
|
|
200
|
+
bundleId: blocker.bundleId,
|
|
201
|
+
blockerType: blocker.type,
|
|
202
|
+
strategyId: strategy.id,
|
|
203
|
+
success: verified,
|
|
204
|
+
durationMs,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
if (verified) {
|
|
208
|
+
// Clear cooldown on success
|
|
209
|
+
this.strategyCooldowns.delete(`${blocker.type}:${strategy.id}`);
|
|
210
|
+
return { recovered: true, strategyId: strategy.id, durationMs };
|
|
211
|
+
}
|
|
212
|
+
// Record cooldown for failed strategy
|
|
213
|
+
this.strategyCooldowns.set(`${blocker.type}:${strategy.id}`, { failedAt: Date.now() });
|
|
214
|
+
return { recovered: false, reason: "all_strategies_failed" };
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Verify the blocker is resolved by re-checking world model state.
|
|
218
|
+
*/
|
|
219
|
+
verifyRecovery(blocker) {
|
|
220
|
+
switch (blocker.type) {
|
|
221
|
+
case "unexpected_dialog":
|
|
222
|
+
case "permission_dialog":
|
|
223
|
+
case "login_required":
|
|
224
|
+
case "captcha": {
|
|
225
|
+
const dialogs = this.worldModel.getActiveDialogs();
|
|
226
|
+
if (blocker.dialogTitle) {
|
|
227
|
+
return !dialogs.some((d) => d.title === blocker.dialogTitle);
|
|
228
|
+
}
|
|
229
|
+
return dialogs.length === 0;
|
|
230
|
+
}
|
|
231
|
+
case "focus_lost": {
|
|
232
|
+
if (!blocker.bundleId)
|
|
233
|
+
return false;
|
|
234
|
+
return this.worldModel.getState().focusedApp?.bundleId === blocker.bundleId;
|
|
235
|
+
}
|
|
236
|
+
case "app_crashed": {
|
|
237
|
+
return this.worldModel.getState().windows.size > 0;
|
|
238
|
+
}
|
|
239
|
+
case "element_gone": {
|
|
240
|
+
// The element should be back — verify focused window has controls
|
|
241
|
+
const win = this.worldModel.getFocusedWindow();
|
|
242
|
+
if (!win)
|
|
243
|
+
return false;
|
|
244
|
+
return win.controls.size > 0;
|
|
245
|
+
}
|
|
246
|
+
case "selector_drift": {
|
|
247
|
+
// After recovery, controls should be findable — verify the focused window
|
|
248
|
+
// has recently updated controls (not all stale)
|
|
249
|
+
const win = this.worldModel.getFocusedWindow();
|
|
250
|
+
if (!win)
|
|
251
|
+
return false;
|
|
252
|
+
if (win.controls.size === 0)
|
|
253
|
+
return false;
|
|
254
|
+
const stale = this.worldModel.getStaleControls(5_000);
|
|
255
|
+
return stale.length < win.controls.size;
|
|
256
|
+
}
|
|
257
|
+
case "unknown_state": {
|
|
258
|
+
// State should be less stale after recovery — check stale count is low
|
|
259
|
+
const state = this.worldModel.getState();
|
|
260
|
+
if (state.windows.size === 0)
|
|
261
|
+
return false;
|
|
262
|
+
const stale = this.worldModel.getStaleControls(5_000);
|
|
263
|
+
let totalControls = 0;
|
|
264
|
+
for (const w of state.windows.values()) {
|
|
265
|
+
totalControls += w.controls.size;
|
|
266
|
+
}
|
|
267
|
+
// Pass if fewer than half of controls are stale
|
|
268
|
+
return totalControls > 0 && stale.length < totalControls / 2;
|
|
269
|
+
}
|
|
270
|
+
case "loading_stuck": {
|
|
271
|
+
// UI should have changed — verify state was updated recently (within 2s)
|
|
272
|
+
const state = this.worldModel.getState();
|
|
273
|
+
const ageMs = Date.now() - new Date(state.updatedAt).getTime();
|
|
274
|
+
return ageMs < 2_000;
|
|
275
|
+
}
|
|
276
|
+
case "network_error":
|
|
277
|
+
case "rate_limited": {
|
|
278
|
+
// Transient errors — verify state was refreshed recently (within 3s)
|
|
279
|
+
const state = this.worldModel.getState();
|
|
280
|
+
const ageMs = Date.now() - new Date(state.updatedAt).getTime();
|
|
281
|
+
return ageMs < 3_000;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Load and cache reference errors for a bundleId.
|
|
287
|
+
*/
|
|
288
|
+
loadReferenceErrors(bundleId) {
|
|
289
|
+
const cached = this.referenceCache.get(bundleId);
|
|
290
|
+
if (cached !== undefined)
|
|
291
|
+
return cached;
|
|
292
|
+
let errors = [];
|
|
293
|
+
try {
|
|
294
|
+
const files = fs.readdirSync(this.config.referencesDir);
|
|
295
|
+
for (const file of files) {
|
|
296
|
+
if (!file.endsWith(".json"))
|
|
297
|
+
continue;
|
|
298
|
+
try {
|
|
299
|
+
// Guard against oversized files (same 10MB limit as LearningEngine)
|
|
300
|
+
const filePath = path.join(this.config.referencesDir, file);
|
|
301
|
+
const stat = fs.statSync(filePath);
|
|
302
|
+
if (stat.size > 10 * 1024 * 1024)
|
|
303
|
+
continue;
|
|
304
|
+
const raw = fs.readFileSync(filePath, "utf-8");
|
|
305
|
+
const ref = JSON.parse(raw);
|
|
306
|
+
if (ref.bundleId === bundleId && Array.isArray(ref.errors)) {
|
|
307
|
+
errors = ref.errors.filter((e) => typeof e.error === "string" && typeof e.solution === "string");
|
|
308
|
+
break;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
catch { /* skip malformed */ }
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
catch { /* dir doesn't exist */ }
|
|
315
|
+
this.referenceCache.set(bundleId, errors);
|
|
316
|
+
return errors;
|
|
317
|
+
}
|
|
318
|
+
recordEvent(event) {
|
|
319
|
+
try {
|
|
320
|
+
this.memory.recordError(`recovery:${event.strategyId}`, event.error ?? "", event.success ? event.strategyLabel : null, event.blocker.bundleId ?? undefined);
|
|
321
|
+
}
|
|
322
|
+
catch { /* best-effort */ }
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
function sleep(ms) {
|
|
326
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
327
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
export { RecoveryEngine } from "./engine.js";
|
|
18
|
+
export { detectBlockers } from "./detectors.js";
|
|
19
|
+
export { getBuiltinStrategies, parseReferenceStrategies } from "./strategies.js";
|
|
20
|
+
export { DEFAULT_RECOVERY_BUDGET } from "./types.js";
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
/**
|
|
18
|
+
* Built-in recovery strategies keyed by BlockerType, in priority order.
|
|
19
|
+
* Strategies with empty steps are escalations — they signal that human
|
|
20
|
+
* intervention is needed.
|
|
21
|
+
*/
|
|
22
|
+
const BUILTIN_STRATEGIES = [
|
|
23
|
+
// ── unexpected_dialog ──
|
|
24
|
+
{
|
|
25
|
+
id: "dismiss_dialog_cancel",
|
|
26
|
+
blockerType: "unexpected_dialog",
|
|
27
|
+
label: "Dismiss dialog via Cancel",
|
|
28
|
+
steps: [{ tool: "click_text", params: { text: "Cancel" }, description: "Click Cancel" }],
|
|
29
|
+
postcondition: null,
|
|
30
|
+
source: "builtin",
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: "dismiss_dialog_ok",
|
|
34
|
+
blockerType: "unexpected_dialog",
|
|
35
|
+
label: "Dismiss dialog via OK",
|
|
36
|
+
steps: [{ tool: "click_text", params: { text: "OK" }, description: "Click OK" }],
|
|
37
|
+
postcondition: null,
|
|
38
|
+
source: "builtin",
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
id: "dismiss_dialog_escape",
|
|
42
|
+
blockerType: "unexpected_dialog",
|
|
43
|
+
label: "Dismiss dialog via Escape",
|
|
44
|
+
steps: [{ tool: "key", params: { combo: "Escape" }, description: "Press Escape" }],
|
|
45
|
+
postcondition: null,
|
|
46
|
+
source: "builtin",
|
|
47
|
+
},
|
|
48
|
+
// ── permission_dialog ──
|
|
49
|
+
{
|
|
50
|
+
id: "grant_permission_allow",
|
|
51
|
+
blockerType: "permission_dialog",
|
|
52
|
+
label: "Grant permission via Allow",
|
|
53
|
+
steps: [{ tool: "click_text", params: { text: "Allow" }, description: "Click Allow" }],
|
|
54
|
+
postcondition: null,
|
|
55
|
+
source: "builtin",
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
id: "grant_permission_ok",
|
|
59
|
+
blockerType: "permission_dialog",
|
|
60
|
+
label: "Grant permission via OK",
|
|
61
|
+
steps: [{ tool: "click_text", params: { text: "OK" }, description: "Click OK" }],
|
|
62
|
+
postcondition: null,
|
|
63
|
+
source: "builtin",
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
id: "safari_allow_popup",
|
|
67
|
+
blockerType: "permission_dialog",
|
|
68
|
+
label: "Allow Safari popup via notification bar",
|
|
69
|
+
steps: [
|
|
70
|
+
{ tool: "key", params: { combo: "cmd+z" }, description: "Undo popup block (shows notification)" },
|
|
71
|
+
{ tool: "screenshot", params: {}, description: "Check for popup notification bar" },
|
|
72
|
+
],
|
|
73
|
+
postcondition: null,
|
|
74
|
+
source: "builtin",
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
id: "safari_enable_popups_applescript",
|
|
78
|
+
blockerType: "permission_dialog",
|
|
79
|
+
label: "Enable popups for site via AppleScript",
|
|
80
|
+
steps: [
|
|
81
|
+
{ tool: "applescript", params: { script: 'tell application "Safari" to activate' }, description: "Activate Safari" },
|
|
82
|
+
{ tool: "key", params: { combo: "cmd+," }, description: "Open Safari preferences" },
|
|
83
|
+
{ tool: "screenshot", params: {}, description: "Navigate to popup settings" },
|
|
84
|
+
],
|
|
85
|
+
postcondition: null,
|
|
86
|
+
source: "builtin",
|
|
87
|
+
},
|
|
88
|
+
// ── focus_lost ──
|
|
89
|
+
{
|
|
90
|
+
id: "refocus_app",
|
|
91
|
+
blockerType: "focus_lost",
|
|
92
|
+
label: "Refocus target application",
|
|
93
|
+
steps: [{ tool: "focus", params: {}, description: "Focus target app" }],
|
|
94
|
+
postcondition: null,
|
|
95
|
+
source: "builtin",
|
|
96
|
+
},
|
|
97
|
+
// ── app_crashed ──
|
|
98
|
+
{
|
|
99
|
+
id: "relaunch_app",
|
|
100
|
+
blockerType: "app_crashed",
|
|
101
|
+
label: "Relaunch crashed application",
|
|
102
|
+
steps: [
|
|
103
|
+
{ tool: "launch", params: {}, description: "Launch application" },
|
|
104
|
+
{ tool: "screenshot", params: {}, description: "Wait for app ready" },
|
|
105
|
+
],
|
|
106
|
+
postcondition: null,
|
|
107
|
+
source: "builtin",
|
|
108
|
+
},
|
|
109
|
+
// ── element_gone ──
|
|
110
|
+
{
|
|
111
|
+
id: "rescan_ax_tree",
|
|
112
|
+
blockerType: "element_gone",
|
|
113
|
+
label: "Rescan AX tree to relocate element",
|
|
114
|
+
steps: [{ tool: "ui_tree", params: {}, description: "Refresh AX tree" }],
|
|
115
|
+
postcondition: null,
|
|
116
|
+
source: "builtin",
|
|
117
|
+
},
|
|
118
|
+
// ── selector_drift ──
|
|
119
|
+
{
|
|
120
|
+
id: "rescan_for_drift",
|
|
121
|
+
blockerType: "selector_drift",
|
|
122
|
+
label: "Rescan AX tree for selector drift",
|
|
123
|
+
steps: [{ tool: "ui_tree", params: {}, description: "Rescan for drift" }],
|
|
124
|
+
postcondition: null,
|
|
125
|
+
source: "builtin",
|
|
126
|
+
},
|
|
127
|
+
// ── loading_stuck ──
|
|
128
|
+
{
|
|
129
|
+
id: "wait_for_load",
|
|
130
|
+
blockerType: "loading_stuck",
|
|
131
|
+
label: "Wait and recheck",
|
|
132
|
+
steps: [{ tool: "screenshot", params: {}, description: "Wait via screenshot" }],
|
|
133
|
+
postcondition: null,
|
|
134
|
+
source: "builtin",
|
|
135
|
+
},
|
|
136
|
+
// ── network_error ──
|
|
137
|
+
{
|
|
138
|
+
id: "reload_page",
|
|
139
|
+
blockerType: "network_error",
|
|
140
|
+
label: "Reload page",
|
|
141
|
+
steps: [{ tool: "key", params: { combo: "cmd+r" }, description: "Reload page" }],
|
|
142
|
+
postcondition: null,
|
|
143
|
+
source: "builtin",
|
|
144
|
+
},
|
|
145
|
+
// ── unknown_state ──
|
|
146
|
+
{
|
|
147
|
+
id: "full_perception_refresh",
|
|
148
|
+
blockerType: "unknown_state",
|
|
149
|
+
label: "Full perception refresh",
|
|
150
|
+
steps: [
|
|
151
|
+
{ tool: "screenshot", params: {}, description: "Take screenshot" },
|
|
152
|
+
{ tool: "ui_tree", params: {}, description: "Refresh AX tree" },
|
|
153
|
+
],
|
|
154
|
+
postcondition: null,
|
|
155
|
+
source: "builtin",
|
|
156
|
+
},
|
|
157
|
+
// ── Escalation-only (no automated recovery possible) ──
|
|
158
|
+
{
|
|
159
|
+
id: "escalate_login",
|
|
160
|
+
blockerType: "login_required",
|
|
161
|
+
label: "Escalate: login required",
|
|
162
|
+
steps: [],
|
|
163
|
+
postcondition: null,
|
|
164
|
+
source: "builtin",
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
id: "escalate_captcha",
|
|
168
|
+
blockerType: "captcha",
|
|
169
|
+
label: "Escalate: captcha",
|
|
170
|
+
steps: [],
|
|
171
|
+
postcondition: null,
|
|
172
|
+
source: "builtin",
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
id: "escalate_rate_limited",
|
|
176
|
+
blockerType: "rate_limited",
|
|
177
|
+
label: "Escalate: rate limited",
|
|
178
|
+
steps: [],
|
|
179
|
+
postcondition: null,
|
|
180
|
+
source: "builtin",
|
|
181
|
+
},
|
|
182
|
+
];
|
|
183
|
+
/** Return built-in strategies matching a blocker type, in priority order. */
|
|
184
|
+
export function getBuiltinStrategies(blockerType) {
|
|
185
|
+
return BUILTIN_STRATEGIES.filter((s) => s.blockerType === blockerType);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Parse a solution text into concrete recovery steps.
|
|
189
|
+
* Pattern-matches common instruction phrases to map to tool calls.
|
|
190
|
+
* Falls back to screenshot with full solution as description.
|
|
191
|
+
*/
|
|
192
|
+
export function parseSolutionToSteps(solution) {
|
|
193
|
+
const steps = [];
|
|
194
|
+
// Split multi-sentence solutions into individual instructions
|
|
195
|
+
const sentences = solution.split(/[.;]\s+/).filter((s) => s.trim().length > 5);
|
|
196
|
+
if (sentences.length === 0)
|
|
197
|
+
sentences.push(solution);
|
|
198
|
+
for (const sentence of sentences) {
|
|
199
|
+
const lower = sentence.toLowerCase();
|
|
200
|
+
const step = matchSolutionSentence(lower, sentence);
|
|
201
|
+
steps.push(step);
|
|
202
|
+
}
|
|
203
|
+
return steps;
|
|
204
|
+
}
|
|
205
|
+
function matchSolutionSentence(lower, original) {
|
|
206
|
+
// Click/tap/select patterns
|
|
207
|
+
const clickMatch = lower.match(/(?:click|tap|select|choose|press)\s+(?:the\s+)?['"]?([^'",.]+)['"]?/i);
|
|
208
|
+
if (clickMatch && !/(?:cmd|ctrl|alt|shift|command)/i.test(clickMatch[1])) {
|
|
209
|
+
// Trim trailing filler words like "button and wait", "and then", "to confirm"
|
|
210
|
+
let clickText = clickMatch[1].trim()
|
|
211
|
+
.replace(/\s+(?:button|link|icon|tab|menu|option)(?:\s+.*)?$/i, "")
|
|
212
|
+
.replace(/\s+(?:and\s+.*|to\s+.*|in\s+.*|on\s+.*|from\s+.*|into\s+.*)$/i, "")
|
|
213
|
+
.trim();
|
|
214
|
+
if (!clickText)
|
|
215
|
+
clickText = clickMatch[1].trim();
|
|
216
|
+
return { tool: "click_text", params: { text: clickText }, description: original };
|
|
217
|
+
}
|
|
218
|
+
// Keyboard shortcut patterns
|
|
219
|
+
const keyMatch = lower.match(/(?:press|use|hit)\s+(?:the\s+)?(?:shortcut\s+)?((?:cmd|ctrl|alt|shift|command|control|option)[\s+]+\w+)/i);
|
|
220
|
+
if (keyMatch) {
|
|
221
|
+
const key = keyMatch[1].replace(/\s+/g, "+").replace(/command/i, "Cmd").replace(/control/i, "Ctrl");
|
|
222
|
+
return { tool: "key", params: { combo: key }, description: original };
|
|
223
|
+
}
|
|
224
|
+
// Navigate/go to/open URL patterns
|
|
225
|
+
const navMatch = lower.match(/(?:navigate\s+to|go\s+to|open|visit)\s+(?:the\s+)?(?:url\s+)?(https?:\/\/\S+)/i);
|
|
226
|
+
if (navMatch) {
|
|
227
|
+
const url = navMatch[1];
|
|
228
|
+
// Reject javascript: and data: URLs
|
|
229
|
+
if (/^javascript:/i.test(url) || /^data:/i.test(url)) {
|
|
230
|
+
return { tool: "screenshot", params: {}, description: `Blocked unsafe URL: ${original}` };
|
|
231
|
+
}
|
|
232
|
+
return { tool: "browser_navigate", params: { url }, description: original };
|
|
233
|
+
}
|
|
234
|
+
// Type/enter/input patterns
|
|
235
|
+
const typeMatch = lower.match(/(?:type|enter|input)\s+['"]?([^'",.]+)['"]?/i);
|
|
236
|
+
if (typeMatch) {
|
|
237
|
+
return { tool: "type_text", params: { text: typeMatch[1].trim() }, description: original };
|
|
238
|
+
}
|
|
239
|
+
// Fallback: screenshot with full solution as description
|
|
240
|
+
return { tool: "screenshot", params: {}, description: `Reference solution: ${original}` };
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Parse app-specific recovery strategies from reference JSON errors.
|
|
244
|
+
*/
|
|
245
|
+
export function parseReferenceStrategies(errors, blockerType) {
|
|
246
|
+
return errors.map((e, idx) => ({
|
|
247
|
+
id: `ref_${blockerType}_${idx}`,
|
|
248
|
+
blockerType,
|
|
249
|
+
label: `Reference: ${e.error}`,
|
|
250
|
+
steps: parseSolutionToSteps(e.solution),
|
|
251
|
+
postcondition: null,
|
|
252
|
+
source: "reference",
|
|
253
|
+
}));
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Inject bundleId into strategy steps that need it (focus, launch).
|
|
257
|
+
* Returns a shallow clone.
|
|
258
|
+
*/
|
|
259
|
+
export function buildStrategyWithContext(strategy, bundleId, pid) {
|
|
260
|
+
if (!bundleId && !pid)
|
|
261
|
+
return strategy;
|
|
262
|
+
return {
|
|
263
|
+
...strategy,
|
|
264
|
+
steps: strategy.steps.map((step) => {
|
|
265
|
+
if (bundleId && (step.tool === "focus" || step.tool === "launch")) {
|
|
266
|
+
return { ...step, params: { ...step.params, bundleId } };
|
|
267
|
+
}
|
|
268
|
+
if (pid != null && step.tool === "ui_tree") {
|
|
269
|
+
return { ...step, params: { ...step.params, pid } };
|
|
270
|
+
}
|
|
271
|
+
return step;
|
|
272
|
+
}),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
export const DEFAULT_RECOVERY_BUDGET = {
|
|
18
|
+
maxRecoveryTimeMs: 30_000,
|
|
19
|
+
maxStrategies: 3,
|
|
20
|
+
};
|