screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
/**
|
|
4
|
+
* TopologyPolicy — learns which navigation edges are reliable per app.
|
|
5
|
+
*
|
|
6
|
+
* Persisted to `topology.jsonl`. Each entry tracks success/fail counts
|
|
7
|
+
* for a specific bundleId×fromNode×action×toNode quad, scored with
|
|
8
|
+
* Bayesian averaging.
|
|
9
|
+
*
|
|
10
|
+
* Used alongside the AppMap to determine which navigation paths
|
|
11
|
+
* through an application are verified and reliable.
|
|
12
|
+
*/
|
|
13
|
+
export class TopologyPolicy {
|
|
14
|
+
entries = new Map();
|
|
15
|
+
priorStrength;
|
|
16
|
+
constructor(priorStrength = 2) {
|
|
17
|
+
this.priorStrength = priorStrength;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Record a navigation edge outcome (success/failure for an app).
|
|
21
|
+
*/
|
|
22
|
+
record(outcome) {
|
|
23
|
+
const key = `${outcome.bundleId}::${outcome.fromNode}::${outcome.action}::${outcome.toNode}`;
|
|
24
|
+
let entry = this.entries.get(key);
|
|
25
|
+
if (!entry) {
|
|
26
|
+
entry = {
|
|
27
|
+
key,
|
|
28
|
+
bundleId: outcome.bundleId,
|
|
29
|
+
fromNode: outcome.fromNode,
|
|
30
|
+
action: outcome.action,
|
|
31
|
+
toNode: outcome.toNode,
|
|
32
|
+
successCount: 0,
|
|
33
|
+
failCount: 0,
|
|
34
|
+
score: 0.5,
|
|
35
|
+
lastUsed: new Date().toISOString(),
|
|
36
|
+
};
|
|
37
|
+
this.entries.set(key, entry);
|
|
38
|
+
}
|
|
39
|
+
if (outcome.success) {
|
|
40
|
+
entry.successCount++;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
entry.failCount++;
|
|
44
|
+
}
|
|
45
|
+
entry.score = this.bayesianScore(entry.successCount, entry.failCount);
|
|
46
|
+
entry.lastUsed = new Date().toISOString();
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Query all navigation edges for a given app, sorted by score descending.
|
|
50
|
+
* Optionally filter by source node.
|
|
51
|
+
*/
|
|
52
|
+
query(bundleId, fromNode) {
|
|
53
|
+
const results = [];
|
|
54
|
+
for (const entry of this.entries.values()) {
|
|
55
|
+
if (entry.bundleId !== bundleId)
|
|
56
|
+
continue;
|
|
57
|
+
if (fromNode && entry.fromNode !== fromNode)
|
|
58
|
+
continue;
|
|
59
|
+
results.push(entry);
|
|
60
|
+
}
|
|
61
|
+
return results.sort((a, b) => b.score - a.score);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Get the best navigation edge from a given node, or null if insufficient data.
|
|
65
|
+
*/
|
|
66
|
+
recommend(bundleId, fromNode, minSamples = 3) {
|
|
67
|
+
const candidates = this.query(bundleId, fromNode);
|
|
68
|
+
for (const entry of candidates) {
|
|
69
|
+
if (entry.successCount + entry.failCount >= minSamples && entry.score > 0.5) {
|
|
70
|
+
return entry;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
clear() {
|
|
76
|
+
this.entries.clear();
|
|
77
|
+
}
|
|
78
|
+
getAllEntries() {
|
|
79
|
+
return [...this.entries.values()];
|
|
80
|
+
}
|
|
81
|
+
loadEntries(entries) {
|
|
82
|
+
for (const entry of entries) {
|
|
83
|
+
this.entries.set(entry.key, { ...entry });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
bayesianScore(successes, failures) {
|
|
87
|
+
return ((successes + this.priorStrength) /
|
|
88
|
+
(successes + failures + 2 * this.priorStrength));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
export const DEFAULT_LEARNING_CONFIG = {
|
|
4
|
+
dataDir: "",
|
|
5
|
+
minSamplesForConfidence: 5,
|
|
6
|
+
priorStrength: 2,
|
|
7
|
+
maxEntriesPerFile: 5000,
|
|
8
|
+
maxTimingSamples: 100,
|
|
9
|
+
};
|
|
@@ -29,7 +29,10 @@ export class TimelineLogger {
|
|
|
29
29
|
}
|
|
30
30
|
finish(telemetry, status) {
|
|
31
31
|
const finishedAt = new Date().toISOString();
|
|
32
|
-
const
|
|
32
|
+
const startTime = new Date(telemetry.startedAt).getTime();
|
|
33
|
+
const totalMs = Number.isFinite(startTime)
|
|
34
|
+
? new Date(finishedAt).getTime() - startTime
|
|
35
|
+
: 0;
|
|
33
36
|
const finalized = {
|
|
34
37
|
...telemetry,
|
|
35
38
|
finishedAt,
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
/**
|
|
6
|
+
* Playbook Seeds — converts playbook reference knowledge into memory-compatible formats.
|
|
7
|
+
*
|
|
8
|
+
* Reads all playbooks from disk and extracts:
|
|
9
|
+
* - errors[] → ErrorPattern[] (for quickErrorCheck auto-warnings)
|
|
10
|
+
* - flows{} → Learning[] (for pattern recall)
|
|
11
|
+
* - selectors{} → Learning[] (for verified selector patterns)
|
|
12
|
+
* - policyNotes{} → Learning[] (for rate limits and safety)
|
|
13
|
+
*
|
|
14
|
+
* Called once during MemoryStore.init() to seed the memory system with
|
|
15
|
+
* months of team-curated platform knowledge.
|
|
16
|
+
*/
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
// ── Tool name inference from error/solution text ──
|
|
20
|
+
const TOOL_KEYWORDS = {
|
|
21
|
+
browser_click: ["click", "el.click", ".click()", "button"],
|
|
22
|
+
browser_human_click: ["human_click", "dispatchMouseEvent", "CDP Input"],
|
|
23
|
+
browser_fill_form: ["fill_form", "browser_fill_form", "form"],
|
|
24
|
+
browser_type: ["browser_type", "type into", "typing"],
|
|
25
|
+
browser_js: ["browser_js", "evaluate", "script", "execCommand"],
|
|
26
|
+
browser_navigate: ["navigate", "navigation", "url"],
|
|
27
|
+
browser_dom: ["querySelector", "selector", "DOM"],
|
|
28
|
+
browser_wait: ["wait", "timeout", "load"],
|
|
29
|
+
click: ["native click", "coordinates", "screen click"],
|
|
30
|
+
type_text: ["type_text", "native typing"],
|
|
31
|
+
scroll: ["scroll"],
|
|
32
|
+
};
|
|
33
|
+
function inferTool(text) {
|
|
34
|
+
const lower = text.toLowerCase();
|
|
35
|
+
for (const [tool, keywords] of Object.entries(TOOL_KEYWORDS)) {
|
|
36
|
+
if (keywords.some(kw => lower.includes(kw.toLowerCase())))
|
|
37
|
+
return tool;
|
|
38
|
+
}
|
|
39
|
+
return "browser_click"; // default — most errors are click-related
|
|
40
|
+
}
|
|
41
|
+
// ── Main seed functions ──
|
|
42
|
+
/**
|
|
43
|
+
* Read all playbooks from a directory and extract error patterns.
|
|
44
|
+
* These get loaded into memory's errorsCache so quickErrorCheck() catches them.
|
|
45
|
+
*/
|
|
46
|
+
export function seedErrorsFromPlaybooks(playbooksDir) {
|
|
47
|
+
const playbooks = loadPlaybooks(playbooksDir);
|
|
48
|
+
const errors = [];
|
|
49
|
+
const seen = new Set(); // deduplicate by error text
|
|
50
|
+
for (const pb of playbooks) {
|
|
51
|
+
const platform = pb.platform ?? pb.id ?? "unknown";
|
|
52
|
+
// Extract from errors[]
|
|
53
|
+
if (pb.errors) {
|
|
54
|
+
for (const err of pb.errors) {
|
|
55
|
+
const key = `${platform}::${err.error}`;
|
|
56
|
+
if (seen.has(key))
|
|
57
|
+
continue;
|
|
58
|
+
seen.add(key);
|
|
59
|
+
errors.push({
|
|
60
|
+
id: `pb_err_${platform}_${errors.length}`,
|
|
61
|
+
tool: inferTool(`${err.error} ${err.context} ${err.solution}`),
|
|
62
|
+
params: { _source: "playbook", _platform: platform },
|
|
63
|
+
error: `[${platform}] ${err.error}`,
|
|
64
|
+
resolution: err.solution,
|
|
65
|
+
occurrences: err.severity === "high" ? 10 : err.severity === "medium" ? 5 : 2,
|
|
66
|
+
lastSeen: new Date().toISOString(),
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Extract from flows — tips and why fields often contain error knowledge
|
|
71
|
+
if (pb.flows) {
|
|
72
|
+
for (const [flowName, flow] of Object.entries(pb.flows)) {
|
|
73
|
+
if (flow.why && flow.why.includes("doesn't") || flow.why?.includes("don't") || flow.why?.includes("NOT")) {
|
|
74
|
+
const key = `${platform}::${flowName}::why`;
|
|
75
|
+
if (seen.has(key))
|
|
76
|
+
continue;
|
|
77
|
+
seen.add(key);
|
|
78
|
+
errors.push({
|
|
79
|
+
id: `pb_err_${platform}_flow_${errors.length}`,
|
|
80
|
+
tool: inferTool(flow.why ?? ""),
|
|
81
|
+
params: { _source: "playbook", _platform: platform, _flow: flowName },
|
|
82
|
+
error: `[${platform}/${flowName}] ${flow.why.slice(0, 200)}`,
|
|
83
|
+
resolution: flow.steps?.join(" → ") ?? null,
|
|
84
|
+
occurrences: 5,
|
|
85
|
+
lastSeen: new Date().toISOString(),
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return errors;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Read all playbooks and extract learnings (selectors, patterns, policy notes).
|
|
95
|
+
* These get loaded into memory's learningsCache so queryPatterns() finds them.
|
|
96
|
+
*/
|
|
97
|
+
export function seedLearningsFromPlaybooks(playbooksDir) {
|
|
98
|
+
const playbooks = loadPlaybooks(playbooksDir);
|
|
99
|
+
const learnings = [];
|
|
100
|
+
for (const pb of playbooks) {
|
|
101
|
+
const platform = pb.platform ?? pb.id ?? "unknown";
|
|
102
|
+
const reliability = (pb.successCount ?? 0) + (pb.failCount ?? 0) > 0
|
|
103
|
+
? (pb.successCount ?? 0) / ((pb.successCount ?? 0) + (pb.failCount ?? 0))
|
|
104
|
+
: 0.7;
|
|
105
|
+
// Selectors → learnings (verified working CSS selectors)
|
|
106
|
+
if (pb.selectors) {
|
|
107
|
+
for (const [group, sels] of Object.entries(pb.selectors)) {
|
|
108
|
+
for (const [name, selector] of Object.entries(sels)) {
|
|
109
|
+
// Skip notes/annotations (keys starting with _)
|
|
110
|
+
if (name.startsWith("_"))
|
|
111
|
+
continue;
|
|
112
|
+
learnings.push({
|
|
113
|
+
scope: `chrome/${platform}`,
|
|
114
|
+
pattern: `${group}.${name}: ${selector}`,
|
|
115
|
+
method: "cdp",
|
|
116
|
+
confidence: reliability,
|
|
117
|
+
successCount: Math.max(1, Math.round(reliability * 10)),
|
|
118
|
+
failCount: Math.round((1 - reliability) * 10),
|
|
119
|
+
lastSeen: new Date().toISOString(),
|
|
120
|
+
fix: null,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Flow selectors → learnings
|
|
126
|
+
if (pb.flows) {
|
|
127
|
+
for (const [flowName, flow] of Object.entries(pb.flows)) {
|
|
128
|
+
if (flow.selectors) {
|
|
129
|
+
for (const [name, selector] of Object.entries(flow.selectors)) {
|
|
130
|
+
learnings.push({
|
|
131
|
+
scope: `chrome/${platform}/${flowName}`,
|
|
132
|
+
pattern: `${name}: ${selector}`,
|
|
133
|
+
method: "cdp",
|
|
134
|
+
confidence: reliability,
|
|
135
|
+
successCount: Math.max(1, Math.round(reliability * 10)),
|
|
136
|
+
failCount: Math.round((1 - reliability) * 10),
|
|
137
|
+
lastSeen: new Date().toISOString(),
|
|
138
|
+
fix: null,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// Policy notes → learnings (rate limits, safety rules)
|
|
145
|
+
if (pb.policyNotes) {
|
|
146
|
+
for (const [category, notes] of Object.entries(pb.policyNotes)) {
|
|
147
|
+
for (const note of notes) {
|
|
148
|
+
learnings.push({
|
|
149
|
+
scope: `policy/${platform}`,
|
|
150
|
+
pattern: `[${category}] ${note}`,
|
|
151
|
+
method: "cdp",
|
|
152
|
+
confidence: 1.0,
|
|
153
|
+
successCount: 10,
|
|
154
|
+
failCount: 0,
|
|
155
|
+
lastSeen: new Date().toISOString(),
|
|
156
|
+
fix: null,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Detection expressions → learnings
|
|
162
|
+
if (pb.detection) {
|
|
163
|
+
for (const [name, expr] of Object.entries(pb.detection)) {
|
|
164
|
+
learnings.push({
|
|
165
|
+
scope: `chrome/${platform}/detection`,
|
|
166
|
+
pattern: `${name}: ${expr}`,
|
|
167
|
+
method: "cdp",
|
|
168
|
+
confidence: reliability,
|
|
169
|
+
successCount: Math.max(1, Math.round(reliability * 10)),
|
|
170
|
+
failCount: 0,
|
|
171
|
+
lastSeen: new Date().toISOString(),
|
|
172
|
+
fix: null,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return learnings;
|
|
178
|
+
}
|
|
179
|
+
// ── Helpers ──
|
|
180
|
+
function loadPlaybooks(dir) {
|
|
181
|
+
if (!fs.existsSync(dir))
|
|
182
|
+
return [];
|
|
183
|
+
const files = fs.readdirSync(dir).filter(f => f.endsWith(".json"));
|
|
184
|
+
const playbooks = [];
|
|
185
|
+
for (const file of files) {
|
|
186
|
+
try {
|
|
187
|
+
const raw = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8"));
|
|
188
|
+
// Ensure it has an id
|
|
189
|
+
if (!raw.id)
|
|
190
|
+
raw.id = file.replace(".json", "");
|
|
191
|
+
if (!raw.platform)
|
|
192
|
+
raw.platform = raw.id;
|
|
193
|
+
playbooks.push(raw);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
// Skip unparseable files
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return playbooks;
|
|
200
|
+
}
|
|
@@ -24,7 +24,7 @@ export class RecallEngine {
|
|
|
24
24
|
* Find strategies matching a task description (~0ms, in-memory).
|
|
25
25
|
* Strategies with high fail rates are penalized.
|
|
26
26
|
*/
|
|
27
|
-
recallStrategies(query, limit = 5) {
|
|
27
|
+
recallStrategies(query, limit = 5, currentBundleId) {
|
|
28
28
|
const strategies = this.store.readStrategies();
|
|
29
29
|
if (strategies.length === 0)
|
|
30
30
|
return [];
|
|
@@ -32,13 +32,13 @@ export class RecallEngine {
|
|
|
32
32
|
if (queryTokens.length === 0)
|
|
33
33
|
return [];
|
|
34
34
|
const scored = strategies.map((s) => {
|
|
35
|
+
// Only match against task description, tags, and tool names.
|
|
36
|
+
// Step params (JS code, URLs, selectors) contain too many generic words
|
|
37
|
+
// and cause false positives against unrelated strategies.
|
|
35
38
|
const targetTokens = new Set([
|
|
36
39
|
...tokenize(s.task),
|
|
37
40
|
...s.tags,
|
|
38
41
|
...s.steps.map((step) => step.tool),
|
|
39
|
-
...s.steps.flatMap((step) => Object.values(step.params)
|
|
40
|
-
.filter((v) => typeof v === "string")
|
|
41
|
-
.flatMap(tokenize)),
|
|
42
42
|
]);
|
|
43
43
|
let matches = 0;
|
|
44
44
|
for (const qt of queryTokens) {
|
|
@@ -51,7 +51,7 @@ export class RecallEngine {
|
|
|
51
51
|
}
|
|
52
52
|
const relevance = matches / queryTokens.length;
|
|
53
53
|
const ageMs = Date.now() - new Date(s.lastUsed).getTime();
|
|
54
|
-
const ageDays = ageMs / (1000 * 60 * 60 * 24);
|
|
54
|
+
const ageDays = Number.isFinite(ageMs) ? ageMs / (1000 * 60 * 60 * 24) : 0;
|
|
55
55
|
const recency = Math.max(0.5, 1.0 - ageDays / 365);
|
|
56
56
|
const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
|
|
57
57
|
// Penalty for strategies that have failed — reduces score proportionally
|
|
@@ -60,7 +60,32 @@ export class RecallEngine {
|
|
|
60
60
|
const reliabilityPenalty = totalAttempts > 0
|
|
61
61
|
? s.successCount / totalAttempts
|
|
62
62
|
: 1;
|
|
63
|
-
|
|
63
|
+
// App-context filtering: penalize strategies whose steps target a different app
|
|
64
|
+
let appContextFactor = 1.0;
|
|
65
|
+
if (currentBundleId) {
|
|
66
|
+
const stepsStr = s.steps.map((step) => JSON.stringify(step.params)).join(" ").toLowerCase();
|
|
67
|
+
const taskStr = s.task.toLowerCase();
|
|
68
|
+
const bundleLower = currentBundleId.toLowerCase();
|
|
69
|
+
// Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
|
|
70
|
+
const appName = bundleLower.split(".").pop() ?? "";
|
|
71
|
+
const mentionsCurrentApp = taskStr.includes(bundleLower) || taskStr.includes(appName)
|
|
72
|
+
|| stepsStr.includes(bundleLower);
|
|
73
|
+
// Check if strategy targets a DIFFERENT app via focus/launch steps
|
|
74
|
+
const hasFocusStep = s.steps.some((step) => (step.tool === "focus" || step.tool === "launch") &&
|
|
75
|
+
step.params && "bundleId" in step.params &&
|
|
76
|
+
typeof step.params.bundleId === "string" &&
|
|
77
|
+
step.params.bundleId.toLowerCase() !== bundleLower);
|
|
78
|
+
if (hasFocusStep && !mentionsCurrentApp) {
|
|
79
|
+
appContextFactor = 0.1; // Heavy penalty for wrong-app strategies
|
|
80
|
+
}
|
|
81
|
+
else if (mentionsCurrentApp) {
|
|
82
|
+
appContextFactor = 1.5; // Boost for matching strategies
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// Require at least 50% token overlap to be considered relevant
|
|
86
|
+
if (relevance < 0.5)
|
|
87
|
+
return { ...s, score: 0 };
|
|
88
|
+
const score = relevance * recency * successBoost * reliabilityPenalty * appContextFactor;
|
|
64
89
|
return { ...s, score };
|
|
65
90
|
});
|
|
66
91
|
return scored
|
|
@@ -103,9 +128,13 @@ export class RecallEngine {
|
|
|
103
128
|
* Tries fingerprint prefix match first (O(1)), then falls back to scan.
|
|
104
129
|
* Skips unreliable strategies (failCount > successCount).
|
|
105
130
|
*/
|
|
106
|
-
quickStrategyHint(recentTools) {
|
|
131
|
+
quickStrategyHint(recentTools, currentBundleId) {
|
|
107
132
|
if (recentTools.length === 0)
|
|
108
133
|
return null;
|
|
134
|
+
// Require at least 2 tools in the sequence to reduce false positives
|
|
135
|
+
// from single-tool matches (e.g. just "focus" matching every strategy)
|
|
136
|
+
if (recentTools.length < 2)
|
|
137
|
+
return null;
|
|
109
138
|
const strategies = this.store.readStrategies();
|
|
110
139
|
for (const s of strategies) {
|
|
111
140
|
if (s.steps.length <= recentTools.length)
|
|
@@ -114,6 +143,18 @@ export class RecallEngine {
|
|
|
114
143
|
const failCount = s.failCount ?? 0;
|
|
115
144
|
if (failCount > s.successCount)
|
|
116
145
|
continue;
|
|
146
|
+
// If we know the current app, prefer strategies that mention it
|
|
147
|
+
// and skip strategies clearly for a different app
|
|
148
|
+
if (currentBundleId) {
|
|
149
|
+
const taskLower = s.task.toLowerCase();
|
|
150
|
+
const bundleLower = currentBundleId.toLowerCase();
|
|
151
|
+
// Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
|
|
152
|
+
const appName = bundleLower.split(".").pop() ?? "";
|
|
153
|
+
const mentionsCurrentApp = taskLower.includes(appName) || taskLower.includes(bundleLower);
|
|
154
|
+
const mentionsOtherApp = !mentionsCurrentApp && /com\.\w+\.\w+/.test(s.task);
|
|
155
|
+
if (mentionsOtherApp)
|
|
156
|
+
continue; // strategy is for a different app
|
|
157
|
+
}
|
|
117
158
|
const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
|
|
118
159
|
const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
|
|
119
160
|
if (matches) {
|
|
@@ -144,11 +185,22 @@ export class RecallEngine {
|
|
|
144
185
|
}
|
|
145
186
|
}
|
|
146
187
|
/** Tokenize a string into lowercase keywords (3+ chars) */
|
|
188
|
+
/** Common automation verbs/nouns that match almost any strategy — filter them out */
|
|
189
|
+
const RECALL_STOPWORDS = new Set([
|
|
190
|
+
"open", "close", "click", "set", "get", "the", "and", "for", "from",
|
|
191
|
+
"into", "with", "then", "this", "that", "use", "run", "start", "stop",
|
|
192
|
+
"new", "add", "app", "settings", "window", "button", "text", "page",
|
|
193
|
+
"file", "menu", "tab", "navigate", "type", "select", "find", "wait",
|
|
194
|
+
"send", "save", "copy", "paste", "delete", "create", "edit", "view",
|
|
195
|
+
"show", "hide", "move", "drag", "drop", "enter", "press", "about",
|
|
196
|
+
"input", "form", "link", "image", "video", "upload", "download",
|
|
197
|
+
"first", "last", "next", "take", "result", "search",
|
|
198
|
+
]);
|
|
147
199
|
function tokenize(text) {
|
|
148
200
|
return text
|
|
149
201
|
.toLowerCase()
|
|
150
202
|
.split(/[\W_]+/)
|
|
151
|
-
.filter((w) => w.length >=
|
|
203
|
+
.filter((w) => w.length >= 2 && !RECALL_STOPWORDS.has(w));
|
|
152
204
|
}
|
|
153
205
|
/** Simple string similarity: shared character bigrams / total bigrams */
|
|
154
206
|
function stringSimilarity(a, b) {
|
|
@@ -29,6 +29,7 @@ import { writeFileAtomicSync } from "../util/atomic-write.js";
|
|
|
29
29
|
import { MemoryStore } from "./store.js";
|
|
30
30
|
import { SessionTracker } from "./session.js";
|
|
31
31
|
import { RecallEngine } from "./recall.js";
|
|
32
|
+
import { seedLearningsFromPlaybooks } from "./playbook-seeds.js";
|
|
32
33
|
// ── Defaults ─────────────────────────────────────
|
|
33
34
|
const DEFAULT_POLICY = {
|
|
34
35
|
maxConsecutiveErrors: 5,
|
|
@@ -90,6 +91,26 @@ export class MemoryService {
|
|
|
90
91
|
this.ensureMemDir();
|
|
91
92
|
// Load learnings
|
|
92
93
|
this.learningsCache = this.readJsonlSafe("learnings.jsonl");
|
|
94
|
+
// Seed learnings from playbooks — selectors, detection, policy notes.
|
|
95
|
+
// These have scope prefixes like "chrome/figma" so they merge cleanly
|
|
96
|
+
// with runtime-discovered learnings without duplicating.
|
|
97
|
+
const playbooksDir = path.join(this.baseDir, "references");
|
|
98
|
+
const pbLearnings = seedLearningsFromPlaybooks(playbooksDir);
|
|
99
|
+
if (pbLearnings.length > 0) {
|
|
100
|
+
const existingKeys = new Set(this.learningsCache.map(l => `${l.scope}::${l.pattern}::${l.method}`));
|
|
101
|
+
let added = 0;
|
|
102
|
+
for (const pl of pbLearnings) {
|
|
103
|
+
const key = `${pl.scope}::${pl.pattern}::${pl.method}`;
|
|
104
|
+
if (!existingKeys.has(key)) {
|
|
105
|
+
this.learningsCache.push({
|
|
106
|
+
id: `pb_lrn_${added}`,
|
|
107
|
+
...pl,
|
|
108
|
+
});
|
|
109
|
+
existingKeys.add(key);
|
|
110
|
+
added++;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
93
114
|
// Load existing snapshot if present (to restore mission/policy across restarts)
|
|
94
115
|
const snapPath = this.filePath("state.json");
|
|
95
116
|
if (fs.existsSync(snapPath)) {
|
|
@@ -285,17 +306,17 @@ export class MemoryService {
|
|
|
285
306
|
queryErrors(tool) {
|
|
286
307
|
return this.recall.recallErrors(tool);
|
|
287
308
|
}
|
|
288
|
-
/** Fuzzy-match strategies by query string. */
|
|
289
|
-
recallStrategies(query, limit) {
|
|
290
|
-
return this.recall.recallStrategies(query, limit);
|
|
309
|
+
/** Fuzzy-match strategies by query string. Optionally filter by current app. */
|
|
310
|
+
recallStrategies(query, limit, currentBundleId) {
|
|
311
|
+
return this.recall.recallStrategies(query, limit, currentBundleId);
|
|
291
312
|
}
|
|
292
313
|
/** Quick error check for interceptor (~0ms). */
|
|
293
314
|
quickErrorCheck(tool) {
|
|
294
315
|
return this.recall.quickErrorCheck(tool);
|
|
295
316
|
}
|
|
296
317
|
/** Quick strategy hint for interceptor (~0ms). */
|
|
297
|
-
quickStrategyHint(recentTools) {
|
|
298
|
-
return this.recall.quickStrategyHint(recentTools);
|
|
318
|
+
quickStrategyHint(recentTools, currentBundleId) {
|
|
319
|
+
return this.recall.quickStrategyHint(recentTools, currentBundleId);
|
|
299
320
|
}
|
|
300
321
|
/** Record strategy outcome for feedback loop. */
|
|
301
322
|
recordStrategyOutcome(fingerprint, success) {
|
|
@@ -397,6 +418,10 @@ export class MemoryService {
|
|
|
397
418
|
return [];
|
|
398
419
|
let text;
|
|
399
420
|
try {
|
|
421
|
+
// Guard against oversized files (same 10MB limit as LearningEngine)
|
|
422
|
+
const stat = fs.statSync(fp);
|
|
423
|
+
if (stat.size > 10 * 1024 * 1024)
|
|
424
|
+
return [];
|
|
400
425
|
text = fs.readFileSync(fp, "utf-8").trim();
|
|
401
426
|
}
|
|
402
427
|
catch {
|
package/dist/src/memory/store.js
CHANGED
|
@@ -26,7 +26,9 @@
|
|
|
26
26
|
import fs from "node:fs";
|
|
27
27
|
import path from "node:path";
|
|
28
28
|
import { writeFileAtomicSync } from "../util/atomic-write.js";
|
|
29
|
+
import { redactPII } from "../util/sanitize.js";
|
|
29
30
|
import { SEED_STRATEGIES } from "./seeds.js";
|
|
31
|
+
import { seedErrorsFromPlaybooks } from "./playbook-seeds.js";
|
|
30
32
|
const MAX_ACTION_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
|
|
31
33
|
const MAX_STRATEGIES = 500;
|
|
32
34
|
const MAX_ERRORS = 200;
|
|
@@ -52,7 +54,9 @@ export class MemoryStore {
|
|
|
52
54
|
// Global flag — only register exit handlers once across all instances
|
|
53
55
|
static exitHandlerRegistered = false;
|
|
54
56
|
static activeInstance = null;
|
|
57
|
+
baseDir;
|
|
55
58
|
constructor(baseDir) {
|
|
59
|
+
this.baseDir = baseDir;
|
|
56
60
|
this.dir = path.join(baseDir, ".screenhand", "memory");
|
|
57
61
|
this.lockPath = path.join(this.dir, ".lock");
|
|
58
62
|
}
|
|
@@ -70,11 +74,23 @@ export class MemoryStore {
|
|
|
70
74
|
if (this.strategiesCache.length === 0 && isFirstBoot) {
|
|
71
75
|
for (const s of SEED_STRATEGIES)
|
|
72
76
|
this.strategiesCache.push(s);
|
|
73
|
-
this.
|
|
77
|
+
this.writeStrategiesRedacted();
|
|
74
78
|
}
|
|
75
79
|
this.enforceStrategyLimit();
|
|
76
80
|
this.rebuildFingerprintIndex();
|
|
77
81
|
this.errorsCache = this.readLinesSafe("errors.jsonl");
|
|
82
|
+
// Seed errors from playbooks — merge curated platform knowledge into memory.
|
|
83
|
+
// Uses pb_err_ prefix IDs so we can identify and refresh them on each boot.
|
|
84
|
+
const playbooksDir = path.join(this.baseDir, "references");
|
|
85
|
+
const pbErrors = seedErrorsFromPlaybooks(playbooksDir);
|
|
86
|
+
if (pbErrors.length > 0) {
|
|
87
|
+
// Remove stale playbook-seeded errors (they'll be re-added with fresh data)
|
|
88
|
+
this.errorsCache = this.errorsCache.filter(e => !e.id.startsWith("pb_err_"));
|
|
89
|
+
// Merge fresh playbook errors
|
|
90
|
+
for (const e of pbErrors) {
|
|
91
|
+
this.errorsCache.push(e);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
78
94
|
this.enforceErrorLimit();
|
|
79
95
|
// Build action stats without caching all entries
|
|
80
96
|
const actions = this.readLinesSafe("actions.jsonl");
|
|
@@ -246,8 +262,9 @@ export class MemoryStore {
|
|
|
246
262
|
if (!this.hasLock)
|
|
247
263
|
return;
|
|
248
264
|
this.rotateActionsIfNeeded();
|
|
249
|
-
//
|
|
250
|
-
|
|
265
|
+
// S75 Option C: Redact PII before persisting to disk (not in live responses)
|
|
266
|
+
const redacted = { ...entry, result: entry.result ? redactPII(entry.result) : entry.result };
|
|
267
|
+
this.pendingActionWrites.push(JSON.stringify(redacted) + "\n");
|
|
251
268
|
// Schedule batch flush (debounced 100ms)
|
|
252
269
|
if (!this.flushTimer) {
|
|
253
270
|
this.flushTimer = setTimeout(() => {
|
|
@@ -321,7 +338,19 @@ export class MemoryStore {
|
|
|
321
338
|
this.rebuildFingerprintIndex();
|
|
322
339
|
}
|
|
323
340
|
}
|
|
324
|
-
this.
|
|
341
|
+
this.writeStrategiesRedacted();
|
|
342
|
+
}
|
|
343
|
+
/** S75 Option C: Redact PII from strategies before any disk write */
|
|
344
|
+
writeStrategiesRedacted() {
|
|
345
|
+
const redacted = this.strategiesCache.map(s => ({
|
|
346
|
+
...s,
|
|
347
|
+
task: redactPII(s.task),
|
|
348
|
+
steps: s.steps.map(st => ({
|
|
349
|
+
...st,
|
|
350
|
+
params: Object.fromEntries(Object.entries(st.params).map(([k, v]) => [k, typeof v === "string" ? redactPII(v) : v])),
|
|
351
|
+
})),
|
|
352
|
+
}));
|
|
353
|
+
this.writeLinesSync("strategies.jsonl", redacted);
|
|
325
354
|
}
|
|
326
355
|
/** O(1) exact lookup by tool sequence fingerprint */
|
|
327
356
|
lookupByFingerprint(fingerprint) {
|
|
@@ -339,7 +368,7 @@ export class MemoryStore {
|
|
|
339
368
|
else {
|
|
340
369
|
strategy.failCount = (strategy.failCount ?? 0) + 1;
|
|
341
370
|
}
|
|
342
|
-
this.
|
|
371
|
+
this.writeStrategiesRedacted();
|
|
343
372
|
}
|
|
344
373
|
/** Read from cache — ~0ms */
|
|
345
374
|
readStrategies() {
|