screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
/**
|
|
18
|
+
* GoalStore — atomic JSON persistence for planner goals.
|
|
19
|
+
*
|
|
20
|
+
* In-memory cache + sync atomic writes (same pattern as JobStore).
|
|
21
|
+
* File: ~/.screenhand/planner/goals.json
|
|
22
|
+
*/
|
|
23
|
+
import fs from "node:fs";
|
|
24
|
+
import path from "node:path";
|
|
25
|
+
import { writeFileAtomicSync, readJsonWithRecovery } from "../util/atomic-write.js";
|
|
26
|
+
const MAX_GOALS = 100;
|
|
27
|
+
export class GoalStore {
|
|
28
|
+
filePath;
|
|
29
|
+
goals = [];
|
|
30
|
+
initialized = false;
|
|
31
|
+
constructor(dir) {
|
|
32
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
33
|
+
this.filePath = path.join(dir, "goals.json");
|
|
34
|
+
}
|
|
35
|
+
init() {
|
|
36
|
+
if (this.initialized)
|
|
37
|
+
return;
|
|
38
|
+
this.initialized = true;
|
|
39
|
+
this.goals = readJsonWithRecovery(this.filePath) ?? [];
|
|
40
|
+
}
|
|
41
|
+
get(id) {
|
|
42
|
+
return this.goals.find((g) => g.id === id);
|
|
43
|
+
}
|
|
44
|
+
list(status) {
|
|
45
|
+
if (status)
|
|
46
|
+
return this.goals.filter((g) => g.status === status);
|
|
47
|
+
return [...this.goals];
|
|
48
|
+
}
|
|
49
|
+
add(goal) {
|
|
50
|
+
this.goals.push(goal);
|
|
51
|
+
this.persist();
|
|
52
|
+
}
|
|
53
|
+
update(id, goal) {
|
|
54
|
+
const idx = this.goals.findIndex((g) => g.id === id);
|
|
55
|
+
if (idx < 0) {
|
|
56
|
+
this.goals.push(goal);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.goals[idx] = goal;
|
|
60
|
+
}
|
|
61
|
+
this.persist();
|
|
62
|
+
}
|
|
63
|
+
remove(id) {
|
|
64
|
+
const before = this.goals.length;
|
|
65
|
+
this.goals = this.goals.filter((g) => g.id !== id);
|
|
66
|
+
if (this.goals.length < before) {
|
|
67
|
+
this.persist();
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
prune() {
|
|
73
|
+
const terminal = this.goals
|
|
74
|
+
.filter((g) => g.status === "completed" || g.status === "failed")
|
|
75
|
+
.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
76
|
+
if (terminal.length <= MAX_GOALS)
|
|
77
|
+
return 0;
|
|
78
|
+
const evictCount = terminal.length - MAX_GOALS;
|
|
79
|
+
const evictIds = new Set(terminal.slice(0, evictCount).map((g) => g.id));
|
|
80
|
+
this.goals = this.goals.filter((g) => !evictIds.has(g.id));
|
|
81
|
+
this.persist();
|
|
82
|
+
return evictCount;
|
|
83
|
+
}
|
|
84
|
+
persist() {
|
|
85
|
+
try {
|
|
86
|
+
writeFileAtomicSync(this.filePath, JSON.stringify(this.goals, null, 2));
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// Non-critical — in-memory cache is authoritative
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
export { Planner } from "./planner.js";
|
|
18
|
+
export { PlanExecutor } from "./executor.js";
|
|
19
|
+
export { GoalStore } from "./goal-store.js";
|
|
20
|
+
export { ToolRegistry } from "./tool-registry.js";
|
|
21
|
+
export { playbookToPlan, strategyToPlan, flowToPlan } from "./deterministic.js";
|
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
import crypto from "node:crypto";
|
|
18
|
+
import { DEFAULT_PLANNER_CONFIG } from "./types.js";
|
|
19
|
+
import { playbookToPlan, strategyToPlan, flowToPlan } from "./deterministic.js";
|
|
20
|
+
function uid() {
|
|
21
|
+
return crypto.randomBytes(6).toString("hex");
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Decompose a goal description into subgoal parts.
|
|
25
|
+
* Splits on: numbered steps ("1. ... 2. ..."), "and then", "then",
|
|
26
|
+
* ", and " (Oxford comma), or semicolons.
|
|
27
|
+
* Returns the original description as a single-element array if no split applies.
|
|
28
|
+
*/
|
|
29
|
+
function decomposeGoal(description) {
|
|
30
|
+
// 1. Try numbered steps: "1. do X 2. do Y" or "1) do X 2) do Y"
|
|
31
|
+
const numberedPattern = /(?:^|\s)(\d+)[.)]\s+/g;
|
|
32
|
+
const numberedMatches = [...description.matchAll(numberedPattern)];
|
|
33
|
+
if (numberedMatches.length >= 2) {
|
|
34
|
+
const parts = [];
|
|
35
|
+
for (let i = 0; i < numberedMatches.length; i++) {
|
|
36
|
+
const start = numberedMatches[i].index + numberedMatches[i][0].indexOf(numberedMatches[i][1]);
|
|
37
|
+
const stepStart = start + numberedMatches[i][0].trimStart().length;
|
|
38
|
+
const end = i + 1 < numberedMatches.length
|
|
39
|
+
? numberedMatches[i + 1].index
|
|
40
|
+
: description.length;
|
|
41
|
+
const text = description.slice(stepStart, end).trim();
|
|
42
|
+
if (text)
|
|
43
|
+
parts.push(text);
|
|
44
|
+
}
|
|
45
|
+
if (parts.length >= 2)
|
|
46
|
+
return parts;
|
|
47
|
+
}
|
|
48
|
+
// 2. Try semicolons
|
|
49
|
+
const semiParts = description.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
50
|
+
if (semiParts.length >= 2)
|
|
51
|
+
return semiParts;
|
|
52
|
+
// 3. Try "and then" or ", then"
|
|
53
|
+
const thenParts = description.split(/\s+and\s+then\s+|,\s*then\s+/i).map((s) => s.trim()).filter((s) => s.length > 0);
|
|
54
|
+
if (thenParts.length >= 2)
|
|
55
|
+
return thenParts;
|
|
56
|
+
// 4. Try ", and " (Oxford comma pattern — implies list of actions)
|
|
57
|
+
const andParts = description.split(/,\s+and\s+/i).map((s) => s.trim()).filter((s) => s.length > 0);
|
|
58
|
+
if (andParts.length >= 2)
|
|
59
|
+
return andParts;
|
|
60
|
+
// 5. No decomposition
|
|
61
|
+
return [description];
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Planner — goal-oriented planning with deterministic fast-path.
|
|
65
|
+
*
|
|
66
|
+
* Priority:
|
|
67
|
+
* 1. Playbook match → deterministic plan (0 LLM calls)
|
|
68
|
+
* 2. Strategy recall → plan from memory (0 LLM calls)
|
|
69
|
+
* 3. Reference flow → semi-deterministic (LLM interprets steps)
|
|
70
|
+
* 4. LLM generation → full plan from scratch
|
|
71
|
+
*/
|
|
72
|
+
export class Planner {
|
|
73
|
+
playbookStore;
|
|
74
|
+
memory;
|
|
75
|
+
contextTracker;
|
|
76
|
+
worldModel;
|
|
77
|
+
config;
|
|
78
|
+
constructor(playbookStore, memory, contextTracker, worldModel, configOrLearning, learningOrConfig) {
|
|
79
|
+
this.playbookStore = playbookStore;
|
|
80
|
+
this.memory = memory;
|
|
81
|
+
this.contextTracker = contextTracker;
|
|
82
|
+
this.worldModel = worldModel;
|
|
83
|
+
// Support both (config, learning) and (learning, config) orderings
|
|
84
|
+
let config;
|
|
85
|
+
let learning;
|
|
86
|
+
for (const arg of [configOrLearning, learningOrConfig]) {
|
|
87
|
+
if (!arg)
|
|
88
|
+
continue;
|
|
89
|
+
if (typeof arg.recommendLocator === "function") {
|
|
90
|
+
learning = arg;
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
config = arg;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
this.config = { ...DEFAULT_PLANNER_CONFIG, ...config };
|
|
97
|
+
this.learningEngine = learning ?? null;
|
|
98
|
+
}
|
|
99
|
+
learningEngine;
|
|
100
|
+
toolRegistry = null;
|
|
101
|
+
/**
|
|
102
|
+
* Set the tool registry for LLM plan generation.
|
|
103
|
+
*/
|
|
104
|
+
setToolRegistry(registry) {
|
|
105
|
+
this.toolRegistry = registry;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Create a Goal from a description.
|
|
109
|
+
* Decomposes complex goals into multiple subgoals when the description
|
|
110
|
+
* contains "and then", "then", ", and", numbered steps, or semicolons.
|
|
111
|
+
*/
|
|
112
|
+
createGoal(description) {
|
|
113
|
+
const parts = decomposeGoal(description);
|
|
114
|
+
const subgoals = parts.map((part) => ({
|
|
115
|
+
id: `sg_${uid()}`,
|
|
116
|
+
description: part,
|
|
117
|
+
status: "pending",
|
|
118
|
+
plan: null,
|
|
119
|
+
attempts: 0,
|
|
120
|
+
maxAttempts: this.config.defaultMaxAttempts,
|
|
121
|
+
lastError: null,
|
|
122
|
+
}));
|
|
123
|
+
const goal = {
|
|
124
|
+
id: `goal_${uid()}`,
|
|
125
|
+
description,
|
|
126
|
+
status: "pending",
|
|
127
|
+
subgoals,
|
|
128
|
+
createdAt: new Date().toISOString(),
|
|
129
|
+
completedAt: null,
|
|
130
|
+
};
|
|
131
|
+
return goal;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Plan a subgoal: find the best ActionPlan from available sources.
|
|
135
|
+
*
|
|
136
|
+
* Priority: playbook > strategy > reference flow > LLM
|
|
137
|
+
*/
|
|
138
|
+
async planSubgoal(subgoal) {
|
|
139
|
+
// 1. Try playbook match
|
|
140
|
+
const playbookPlan = this.findPlaybookPlan(subgoal.description);
|
|
141
|
+
if (playbookPlan)
|
|
142
|
+
return playbookPlan;
|
|
143
|
+
// 2. Try strategy recall
|
|
144
|
+
const strategyPlan = this.findStrategyPlan(subgoal.description);
|
|
145
|
+
if (strategyPlan)
|
|
146
|
+
return strategyPlan;
|
|
147
|
+
// 3. Try reference flow
|
|
148
|
+
const flowPlan = this.findFlowPlan(subgoal.description);
|
|
149
|
+
if (flowPlan)
|
|
150
|
+
return flowPlan;
|
|
151
|
+
// 4. Fallback: LLM-generated plan (or stub if no API key)
|
|
152
|
+
return this.createLLMPlan(subgoal.description);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Plan all subgoals in a goal.
|
|
156
|
+
*/
|
|
157
|
+
async planGoal(goal) {
|
|
158
|
+
goal.status = "active";
|
|
159
|
+
for (const sg of goal.subgoals) {
|
|
160
|
+
if (sg.status === "completed" || sg.status === "skipped")
|
|
161
|
+
continue;
|
|
162
|
+
// Don't re-plan subgoals that already have a plan (e.g. from plan_goal)
|
|
163
|
+
if (!sg.plan) {
|
|
164
|
+
sg.plan = await this.planSubgoal(sg);
|
|
165
|
+
}
|
|
166
|
+
sg.status = "pending";
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Replan a subgoal after failure.
|
|
171
|
+
* Increments attempt count, resets plan, tries alternative sources.
|
|
172
|
+
*/
|
|
173
|
+
async replan(subgoal, reason, errorMsg) {
|
|
174
|
+
subgoal.attempts++;
|
|
175
|
+
subgoal.lastError = errorMsg ?? reason;
|
|
176
|
+
if (subgoal.attempts >= subgoal.maxAttempts) {
|
|
177
|
+
subgoal.status = "failed";
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
subgoal.status = "pending";
|
|
181
|
+
// On replan, try alternative sources or adjust params
|
|
182
|
+
const currentSource = subgoal.plan?.source;
|
|
183
|
+
// If playbook failed, try strategy
|
|
184
|
+
if (currentSource === "playbook") {
|
|
185
|
+
const strategyPlan = this.findStrategyPlan(subgoal.description);
|
|
186
|
+
if (strategyPlan)
|
|
187
|
+
return strategyPlan;
|
|
188
|
+
}
|
|
189
|
+
// If strategy failed, try reference flow
|
|
190
|
+
if (currentSource === "playbook" || currentSource === "strategy") {
|
|
191
|
+
const flowPlan = this.findFlowPlan(subgoal.description);
|
|
192
|
+
if (flowPlan)
|
|
193
|
+
return flowPlan;
|
|
194
|
+
}
|
|
195
|
+
// Don't downgrade deterministic plans to LLM stubs.
|
|
196
|
+
// A 9-step reference_flow failing on step 1 (bridge crash) shouldn't
|
|
197
|
+
// be replaced with a 1-step "ask the human" stub. Better to retry
|
|
198
|
+
// the original plan or fail cleanly.
|
|
199
|
+
if (currentSource === "playbook" || currentSource === "strategy" || currentSource === "reference_flow") {
|
|
200
|
+
// Reset the current plan to retry from the failed step
|
|
201
|
+
if (subgoal.plan) {
|
|
202
|
+
subgoal.plan.currentStepIndex = 0;
|
|
203
|
+
// Reset step statuses so they can be re-executed
|
|
204
|
+
for (const step of subgoal.plan.steps) {
|
|
205
|
+
if (step.status === "failed")
|
|
206
|
+
step.status = "pending";
|
|
207
|
+
}
|
|
208
|
+
return subgoal.plan;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// Only fall back to LLM when no deterministic plan existed
|
|
212
|
+
return this.createLLMPlan(subgoal.description);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Check if a goal is complete (all subgoals done or failed).
|
|
216
|
+
*/
|
|
217
|
+
evaluateGoal(goal) {
|
|
218
|
+
const allDone = goal.subgoals.every((sg) => sg.status === "completed" ||
|
|
219
|
+
sg.status === "failed" ||
|
|
220
|
+
sg.status === "skipped");
|
|
221
|
+
if (!allDone)
|
|
222
|
+
return;
|
|
223
|
+
const anyFailed = goal.subgoals.some((sg) => sg.status === "failed");
|
|
224
|
+
goal.status = anyFailed ? "failed" : "completed";
|
|
225
|
+
goal.completedAt = new Date().toISOString();
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Serialize a goal to JSON (for persistence/transport).
|
|
229
|
+
*/
|
|
230
|
+
static serializeGoal(goal) {
|
|
231
|
+
return JSON.stringify(goal);
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Deserialize a goal from JSON.
|
|
235
|
+
*/
|
|
236
|
+
static deserializeGoal(json) {
|
|
237
|
+
const obj = JSON.parse(json);
|
|
238
|
+
if (!obj.id || !Array.isArray(obj.subgoals)) {
|
|
239
|
+
throw new Error("Invalid Goal JSON: missing id or subgoals");
|
|
240
|
+
}
|
|
241
|
+
return obj;
|
|
242
|
+
}
|
|
243
|
+
// ── Private plan finding ──
|
|
244
|
+
getBundleId() {
|
|
245
|
+
return this.worldModel.getState().focusedApp?.bundleId ?? "";
|
|
246
|
+
}
|
|
247
|
+
findPlaybookPlan(description) {
|
|
248
|
+
// Try task-based match only — don't unconditionally use the active playbook
|
|
249
|
+
// here, because that would shadow findFlowPlan() which also uses the active
|
|
250
|
+
// playbook's flows. The active playbook's steps are only a good match if
|
|
251
|
+
// matchByTask explicitly selects it.
|
|
252
|
+
const playbook = this.playbookStore.matchByTask(description, this.getBundleId());
|
|
253
|
+
if (playbook && playbook.steps.length > 0) {
|
|
254
|
+
return playbookToPlan(playbook, this.config, this.learningEngine, this.getBundleId());
|
|
255
|
+
}
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
findStrategyPlan(description) {
|
|
259
|
+
const strategies = this.memory.recallStrategies(description, 3, this.getBundleId());
|
|
260
|
+
if (strategies.length === 0)
|
|
261
|
+
return null;
|
|
262
|
+
// Prefer strategies whose task description mentions the current app
|
|
263
|
+
const currentApp = this.worldModel?.getState()?.focusedApp?.appName?.toLowerCase() ?? "";
|
|
264
|
+
const currentBundle = this.worldModel?.getState()?.focusedApp?.bundleId?.toLowerCase() ?? "";
|
|
265
|
+
let best = strategies[0];
|
|
266
|
+
for (const s of strategies) {
|
|
267
|
+
const taskLower = s.task.toLowerCase();
|
|
268
|
+
if (currentApp && taskLower.includes(currentApp)) {
|
|
269
|
+
best = s;
|
|
270
|
+
break;
|
|
271
|
+
}
|
|
272
|
+
if (currentBundle && taskLower.includes(currentBundle)) {
|
|
273
|
+
best = s;
|
|
274
|
+
break;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (best.score < 0.6)
|
|
278
|
+
return null;
|
|
279
|
+
// Reject strategies that only contain trivial steps (focus/screenshot/apps/windows)
|
|
280
|
+
// — these are artifacts from testing, not useful automation plans
|
|
281
|
+
const TRIVIAL_TOOLS = new Set(["focus", "screenshot", "apps", "windows", "screenshot_file"]);
|
|
282
|
+
const hasSubstantiveStep = best.steps.some((s) => !TRIVIAL_TOOLS.has(s.tool));
|
|
283
|
+
if (!hasSubstantiveStep)
|
|
284
|
+
return null;
|
|
285
|
+
return strategyToPlan(best, this.config, this.learningEngine, this.getBundleId());
|
|
286
|
+
}
|
|
287
|
+
findFlowPlan(description) {
|
|
288
|
+
// Collect all playbooks that have flows: active playbook first, then ALL loaded playbooks
|
|
289
|
+
const active = this.contextTracker.getActivePlaybook();
|
|
290
|
+
const allPlaybooks = this.playbookStore.getAll();
|
|
291
|
+
// Find best matching flow across ALL playbooks with flows
|
|
292
|
+
// Filter out common automation verbs/nouns that match almost any flow
|
|
293
|
+
const FLOW_STOPWORDS = new Set([
|
|
294
|
+
"open", "close", "click", "set", "get", "the", "and", "for", "from",
|
|
295
|
+
"into", "with", "then", "this", "that", "use", "run", "start", "stop",
|
|
296
|
+
"new", "add", "app", "settings", "window", "button", "text", "page",
|
|
297
|
+
"file", "menu", "tab", "navigate", "type", "select", "find", "wait",
|
|
298
|
+
"send", "save", "copy", "paste", "delete", "create", "edit", "view",
|
|
299
|
+
"show", "hide", "move", "drag", "drop", "enter", "press", "about",
|
|
300
|
+
"input", "form", "link", "image", "video", "upload", "download",
|
|
301
|
+
]);
|
|
302
|
+
const tokens = description.toLowerCase().split(/\W+/).filter((w) => w.length >= 3 && !FLOW_STOPWORDS.has(w));
|
|
303
|
+
// If all tokens are stopwords, there's nothing meaningful to match against flows
|
|
304
|
+
if (tokens.length === 0)
|
|
305
|
+
return null;
|
|
306
|
+
let bestFlow = null;
|
|
307
|
+
let bestScore = 0;
|
|
308
|
+
// Platform-aware scoring: detect current app for flow preference
|
|
309
|
+
const state = this.worldModel.getState();
|
|
310
|
+
const focusedBundle = state.focusedApp?.bundleId?.toLowerCase() ?? "";
|
|
311
|
+
const focusedApp = state.focusedApp?.appName?.toLowerCase() ?? "";
|
|
312
|
+
// Map known apps to flow name prefixes they should prefer
|
|
313
|
+
const isSafari = focusedBundle.includes("safari") || focusedApp === "safari";
|
|
314
|
+
const isChrome = focusedBundle.includes("chrome") || focusedApp === "chrome";
|
|
315
|
+
const isBrowser = isSafari || isChrome;
|
|
316
|
+
// Search active playbook first (gets priority via +2 bonus)
|
|
317
|
+
const candidates = active?.flows ? [{ pb: active, bonus: 2 }] : [];
|
|
318
|
+
for (const pb of allPlaybooks) {
|
|
319
|
+
if (pb.flows && pb !== active) {
|
|
320
|
+
candidates.push({ pb, bonus: 0 });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
for (const { pb, bonus } of candidates) {
|
|
324
|
+
if (!pb.flows)
|
|
325
|
+
continue;
|
|
326
|
+
for (const [name, flow] of Object.entries(pb.flows)) {
|
|
327
|
+
if (!Array.isArray(flow?.steps))
|
|
328
|
+
continue;
|
|
329
|
+
const flowNameLower = name.toLowerCase();
|
|
330
|
+
const flowTokens = flowNameLower.split(/[_\-\s]+/);
|
|
331
|
+
const allTokens = [
|
|
332
|
+
...flowTokens,
|
|
333
|
+
...flow.steps.join(" ").toLowerCase().split(/\W+/),
|
|
334
|
+
];
|
|
335
|
+
let score = bonus;
|
|
336
|
+
for (const t of tokens) {
|
|
337
|
+
if (allTokens.some((ft) => ft.includes(t)))
|
|
338
|
+
score++;
|
|
339
|
+
}
|
|
340
|
+
// Platform-aware boost: prefer flows that match the focused app
|
|
341
|
+
if (isSafari && flowNameLower.includes("safari"))
|
|
342
|
+
score += 3;
|
|
343
|
+
if (isChrome && flowNameLower.includes("browser"))
|
|
344
|
+
score += 3;
|
|
345
|
+
// Penalize browser/CDP flows when in Safari (no CDP available)
|
|
346
|
+
if (isSafari && flowNameLower.includes("browser"))
|
|
347
|
+
score -= 2;
|
|
348
|
+
// Penalize safari-specific flows when in Chrome (use CDP instead)
|
|
349
|
+
if (isChrome && flowNameLower.includes("safari"))
|
|
350
|
+
score -= 2;
|
|
351
|
+
// Penalize desktop_automation generic flows when a specific flow exists
|
|
352
|
+
if (flowNameLower === "desktop_automation" && bestScore > 0)
|
|
353
|
+
score -= 1;
|
|
354
|
+
if (score > bestScore) {
|
|
355
|
+
bestScore = score;
|
|
356
|
+
bestFlow = { name, flow };
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (!bestFlow || bestScore <= 0)
|
|
361
|
+
return null;
|
|
362
|
+
// Require at least 40% of meaningful goal tokens to match flow tokens,
|
|
363
|
+
// with a minimum absolute score of 2, to avoid spurious matches from
|
|
364
|
+
// common verbs hitting unrelated flows.
|
|
365
|
+
// Subtract the active-playbook bonus before comparing — the bonus is a
|
|
366
|
+
// tiebreaker, not evidence that the goal text matches the flow.
|
|
367
|
+
const activeBonus = (active?.flows && bestFlow && active.flows[bestFlow.name]) ? 2 : 0;
|
|
368
|
+
const contentScore = bestScore - activeBonus;
|
|
369
|
+
const minScore = Math.max(2, Math.ceil(tokens.length * 0.4));
|
|
370
|
+
if (contentScore < minScore)
|
|
371
|
+
return null;
|
|
372
|
+
return flowToPlan(bestFlow.name, bestFlow.flow, this.config, this.getRuntimeContext());
|
|
373
|
+
}
|
|
374
|
+
getRuntimeContext() {
|
|
375
|
+
const state = this.worldModel.getState();
|
|
376
|
+
return {
|
|
377
|
+
pid: state.focusedApp?.pid,
|
|
378
|
+
windowId: state.focusedWindowId ?? undefined,
|
|
379
|
+
bundleId: state.focusedApp?.bundleId,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
createLLMPlanStub(description) {
|
|
383
|
+
const step = {
|
|
384
|
+
tool: "",
|
|
385
|
+
params: {},
|
|
386
|
+
expectedPostcondition: null,
|
|
387
|
+
timeout: this.config.defaultStepTimeout,
|
|
388
|
+
fallbackTool: null,
|
|
389
|
+
requiresLLM: true,
|
|
390
|
+
status: "pending",
|
|
391
|
+
description,
|
|
392
|
+
};
|
|
393
|
+
return {
|
|
394
|
+
steps: [step],
|
|
395
|
+
currentStepIndex: 0,
|
|
396
|
+
confidence: 0.3,
|
|
397
|
+
source: "llm",
|
|
398
|
+
sourceId: null,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Build a runtime context summary for the LLM prompt.
|
|
403
|
+
* Includes focused app, window, and visible controls.
|
|
404
|
+
*/
|
|
405
|
+
buildRuntimeContextPrompt() {
|
|
406
|
+
const state = this.worldModel.getState();
|
|
407
|
+
const lines = [];
|
|
408
|
+
if (state.focusedApp) {
|
|
409
|
+
lines.push(`Focused app: ${state.focusedApp.appName} (${state.focusedApp.bundleId}), PID: ${state.focusedApp.pid}`);
|
|
410
|
+
}
|
|
411
|
+
if (state.focusedWindowId !== null) {
|
|
412
|
+
lines.push(`Window ID: ${state.focusedWindowId}`);
|
|
413
|
+
}
|
|
414
|
+
// Include visible controls from world model (top 20)
|
|
415
|
+
if (state.focusedWindowId !== null) {
|
|
416
|
+
const win = state.windows.get(state.focusedWindowId);
|
|
417
|
+
if (win && win.controls.size > 0) {
|
|
418
|
+
const controls = [...win.controls.values()]
|
|
419
|
+
.slice(0, 20)
|
|
420
|
+
.map((c) => `${c.role}:${c.label.value ?? ""}`)
|
|
421
|
+
.filter((s) => s.length > 1);
|
|
422
|
+
if (controls.length > 0) {
|
|
423
|
+
lines.push(`Visible controls: ${controls.join(", ")}`);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
// Include active reference context if available
|
|
428
|
+
const active = this.contextTracker.getActivePlaybook();
|
|
429
|
+
if (active) {
|
|
430
|
+
lines.push(`Platform reference loaded: ${active.platform ?? active.id}`);
|
|
431
|
+
}
|
|
432
|
+
return lines.length > 0 ? `\nRuntime context:\n${lines.join("\n")}` : "";
|
|
433
|
+
}
|
|
434
|
+
async createLLMPlan(description) {
|
|
435
|
+
const apiKey = process.env["ANTHROPIC_API_KEY"];
|
|
436
|
+
if (!apiKey || !this.toolRegistry) {
|
|
437
|
+
return this.createLLMPlanStub(description);
|
|
438
|
+
}
|
|
439
|
+
try {
|
|
440
|
+
const toolNames = this.toolRegistry.getToolNames();
|
|
441
|
+
const runtimeCtx = this.buildRuntimeContextPrompt();
|
|
442
|
+
const controller = new AbortController();
|
|
443
|
+
const timeout = setTimeout(() => controller.abort(), 15_000);
|
|
444
|
+
const prompt = [
|
|
445
|
+
"You are a desktop automation planner. Generate concrete tool steps with real parameter values.",
|
|
446
|
+
runtimeCtx,
|
|
447
|
+
"",
|
|
448
|
+
"Key tool signatures:",
|
|
449
|
+
"- screenshot() → captures current screen",
|
|
450
|
+
"- click_text(windowId: number, text: string, prefer?: 'first'|'largest'|'topmost'|'leftmost') → OCR-click",
|
|
451
|
+
"- ui_press(pid: number, title: string, role?: string) → AX button press",
|
|
452
|
+
"- type_text(text: string) → keyboard typing",
|
|
453
|
+
"- key(key: string) → keyboard shortcut (e.g. 'cmd+a', 'Return')",
|
|
454
|
+
"- browser_navigate(url: string) → navigate browser",
|
|
455
|
+
"- browser_click(selector: string) → click element in browser",
|
|
456
|
+
"- browser_type(selector: string, text: string) → type in browser input",
|
|
457
|
+
"- focus(appName: string) → focus app window",
|
|
458
|
+
"- launch(bundleId: string) → launch app",
|
|
459
|
+
"",
|
|
460
|
+
`All available tools: ${toolNames.join(", ")}`,
|
|
461
|
+
"",
|
|
462
|
+
`Goal: ${description}`,
|
|
463
|
+
"",
|
|
464
|
+
'Return ONLY a JSON array of objects: [{"tool": "...", "params": {...}, "description": "..."}]',
|
|
465
|
+
"Use concrete values from the runtime context above (pid, windowId, etc).",
|
|
466
|
+
"No markdown, no explanation.",
|
|
467
|
+
].join("\n");
|
|
468
|
+
const response = await fetch("https://api.anthropic.com/v1/messages", {
|
|
469
|
+
method: "POST",
|
|
470
|
+
headers: {
|
|
471
|
+
"Content-Type": "application/json",
|
|
472
|
+
"x-api-key": apiKey,
|
|
473
|
+
"anthropic-version": "2023-06-01",
|
|
474
|
+
},
|
|
475
|
+
body: JSON.stringify({
|
|
476
|
+
model: "claude-haiku-4-5-20251001",
|
|
477
|
+
max_tokens: 1000,
|
|
478
|
+
messages: [{ role: "user", content: prompt }],
|
|
479
|
+
}),
|
|
480
|
+
signal: controller.signal,
|
|
481
|
+
});
|
|
482
|
+
clearTimeout(timeout);
|
|
483
|
+
if (!response.ok) {
|
|
484
|
+
return this.createLLMPlanStub(description);
|
|
485
|
+
}
|
|
486
|
+
const data = await response.json();
|
|
487
|
+
const text = data.content?.[0]?.text?.trim();
|
|
488
|
+
if (!text)
|
|
489
|
+
return this.createLLMPlanStub(description);
|
|
490
|
+
// Extract JSON array from response (may be wrapped in markdown code blocks)
|
|
491
|
+
const jsonMatch = text.match(/\[[\s\S]*\]/);
|
|
492
|
+
if (!jsonMatch)
|
|
493
|
+
return this.createLLMPlanStub(description);
|
|
494
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
495
|
+
if (!Array.isArray(parsed) || parsed.length === 0) {
|
|
496
|
+
return this.createLLMPlanStub(description);
|
|
497
|
+
}
|
|
498
|
+
const steps = parsed.map((s) => ({
|
|
499
|
+
tool: s.tool ?? "",
|
|
500
|
+
params: s.params ?? {},
|
|
501
|
+
expectedPostcondition: null,
|
|
502
|
+
timeout: this.config.defaultStepTimeout,
|
|
503
|
+
fallbackTool: null,
|
|
504
|
+
requiresLLM: !s.tool,
|
|
505
|
+
status: "pending",
|
|
506
|
+
description: s.description ?? s.tool ?? description,
|
|
507
|
+
}));
|
|
508
|
+
return {
|
|
509
|
+
steps,
|
|
510
|
+
currentStepIndex: 0,
|
|
511
|
+
confidence: 0.5,
|
|
512
|
+
source: "llm",
|
|
513
|
+
sourceId: null,
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
catch {
|
|
517
|
+
return this.createLLMPlanStub(description);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|