screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,90 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /**
4
+ * TopologyPolicy — learns which navigation edges are reliable per app.
5
+ *
6
+ * Persisted to `topology.jsonl`. Each entry tracks success/fail counts
7
+ * for a specific bundleId×fromNode×action×toNode quad, scored with
8
+ * Bayesian averaging.
9
+ *
10
+ * Used alongside the AppMap to determine which navigation paths
11
+ * through an application are verified and reliable.
12
+ */
13
+ export class TopologyPolicy {
14
+ entries = new Map();
15
+ priorStrength;
16
+ constructor(priorStrength = 2) {
17
+ this.priorStrength = priorStrength;
18
+ }
19
+ /**
20
+ * Record a navigation edge outcome (success/failure for an app).
21
+ */
22
+ record(outcome) {
23
+ const key = `${outcome.bundleId}::${outcome.fromNode}::${outcome.action}::${outcome.toNode}`;
24
+ let entry = this.entries.get(key);
25
+ if (!entry) {
26
+ entry = {
27
+ key,
28
+ bundleId: outcome.bundleId,
29
+ fromNode: outcome.fromNode,
30
+ action: outcome.action,
31
+ toNode: outcome.toNode,
32
+ successCount: 0,
33
+ failCount: 0,
34
+ score: 0.5,
35
+ lastUsed: new Date().toISOString(),
36
+ };
37
+ this.entries.set(key, entry);
38
+ }
39
+ if (outcome.success) {
40
+ entry.successCount++;
41
+ }
42
+ else {
43
+ entry.failCount++;
44
+ }
45
+ entry.score = this.bayesianScore(entry.successCount, entry.failCount);
46
+ entry.lastUsed = new Date().toISOString();
47
+ }
48
+ /**
49
+ * Query all navigation edges for a given app, sorted by score descending.
50
+ * Optionally filter by source node.
51
+ */
52
+ query(bundleId, fromNode) {
53
+ const results = [];
54
+ for (const entry of this.entries.values()) {
55
+ if (entry.bundleId !== bundleId)
56
+ continue;
57
+ if (fromNode && entry.fromNode !== fromNode)
58
+ continue;
59
+ results.push(entry);
60
+ }
61
+ return results.sort((a, b) => b.score - a.score);
62
+ }
63
+ /**
64
+ * Get the best navigation edge from a given node, or null if insufficient data.
65
+ */
66
+ recommend(bundleId, fromNode, minSamples = 3) {
67
+ const candidates = this.query(bundleId, fromNode);
68
+ for (const entry of candidates) {
69
+ if (entry.successCount + entry.failCount >= minSamples && entry.score > 0.5) {
70
+ return entry;
71
+ }
72
+ }
73
+ return null;
74
+ }
75
+ clear() {
76
+ this.entries.clear();
77
+ }
78
+ getAllEntries() {
79
+ return [...this.entries.values()];
80
+ }
81
+ loadEntries(entries) {
82
+ for (const entry of entries) {
83
+ this.entries.set(entry.key, { ...entry });
84
+ }
85
+ }
86
+ bayesianScore(successes, failures) {
87
+ return ((successes + this.priorStrength) /
88
+ (successes + failures + 2 * this.priorStrength));
89
+ }
90
+ }
@@ -0,0 +1,9 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ export const DEFAULT_LEARNING_CONFIG = {
4
+ dataDir: "",
5
+ minSamplesForConfidence: 5,
6
+ priorStrength: 2,
7
+ maxEntriesPerFile: 5000,
8
+ maxTimingSamples: 100,
9
+ };
@@ -29,7 +29,10 @@ export class TimelineLogger {
29
29
  }
30
30
  finish(telemetry, status) {
31
31
  const finishedAt = new Date().toISOString();
32
- const totalMs = new Date(finishedAt).getTime() - new Date(telemetry.startedAt).getTime();
32
+ const startTime = new Date(telemetry.startedAt).getTime();
33
+ const totalMs = Number.isFinite(startTime)
34
+ ? new Date(finishedAt).getTime() - startTime
35
+ : 0;
33
36
  const finalized = {
34
37
  ...telemetry,
35
38
  finishedAt,
@@ -0,0 +1,200 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ /**
6
+ * Playbook Seeds — converts playbook reference knowledge into memory-compatible formats.
7
+ *
8
+ * Reads all playbooks from disk and extracts:
9
+ * - errors[] → ErrorPattern[] (for quickErrorCheck auto-warnings)
10
+ * - flows{} → Learning[] (for pattern recall)
11
+ * - selectors{} → Learning[] (for verified selector patterns)
12
+ * - policyNotes{} → Learning[] (for rate limits and safety)
13
+ *
14
+ * Called once during MemoryStore.init() to seed the memory system with
15
+ * months of team-curated platform knowledge.
16
+ */
17
+ import fs from "node:fs";
18
+ import path from "node:path";
19
+ // ── Tool name inference from error/solution text ──
20
+ const TOOL_KEYWORDS = {
21
+ browser_click: ["click", "el.click", ".click()", "button"],
22
+ browser_human_click: ["human_click", "dispatchMouseEvent", "CDP Input"],
23
+ browser_fill_form: ["fill_form", "browser_fill_form", "form"],
24
+ browser_type: ["browser_type", "type into", "typing"],
25
+ browser_js: ["browser_js", "evaluate", "script", "execCommand"],
26
+ browser_navigate: ["navigate", "navigation", "url"],
27
+ browser_dom: ["querySelector", "selector", "DOM"],
28
+ browser_wait: ["wait", "timeout", "load"],
29
+ click: ["native click", "coordinates", "screen click"],
30
+ type_text: ["type_text", "native typing"],
31
+ scroll: ["scroll"],
32
+ };
33
+ function inferTool(text) {
34
+ const lower = text.toLowerCase();
35
+ for (const [tool, keywords] of Object.entries(TOOL_KEYWORDS)) {
36
+ if (keywords.some(kw => lower.includes(kw.toLowerCase())))
37
+ return tool;
38
+ }
39
+ return "browser_click"; // default — most errors are click-related
40
+ }
41
+ // ── Main seed functions ──
42
+ /**
43
+ * Read all playbooks from a directory and extract error patterns.
44
+ * These get loaded into memory's errorsCache so quickErrorCheck() catches them.
45
+ */
46
+ export function seedErrorsFromPlaybooks(playbooksDir) {
47
+ const playbooks = loadPlaybooks(playbooksDir);
48
+ const errors = [];
49
+ const seen = new Set(); // deduplicate by error text
50
+ for (const pb of playbooks) {
51
+ const platform = pb.platform ?? pb.id ?? "unknown";
52
+ // Extract from errors[]
53
+ if (pb.errors) {
54
+ for (const err of pb.errors) {
55
+ const key = `${platform}::${err.error}`;
56
+ if (seen.has(key))
57
+ continue;
58
+ seen.add(key);
59
+ errors.push({
60
+ id: `pb_err_${platform}_${errors.length}`,
61
+ tool: inferTool(`${err.error} ${err.context} ${err.solution}`),
62
+ params: { _source: "playbook", _platform: platform },
63
+ error: `[${platform}] ${err.error}`,
64
+ resolution: err.solution,
65
+ occurrences: err.severity === "high" ? 10 : err.severity === "medium" ? 5 : 2,
66
+ lastSeen: new Date().toISOString(),
67
+ });
68
+ }
69
+ }
70
+ // Extract from flows — tips and why fields often contain error knowledge
71
+ if (pb.flows) {
72
+ for (const [flowName, flow] of Object.entries(pb.flows)) {
73
+ if (flow.why && flow.why.includes("doesn't") || flow.why?.includes("don't") || flow.why?.includes("NOT")) {
74
+ const key = `${platform}::${flowName}::why`;
75
+ if (seen.has(key))
76
+ continue;
77
+ seen.add(key);
78
+ errors.push({
79
+ id: `pb_err_${platform}_flow_${errors.length}`,
80
+ tool: inferTool(flow.why ?? ""),
81
+ params: { _source: "playbook", _platform: platform, _flow: flowName },
82
+ error: `[${platform}/${flowName}] ${flow.why.slice(0, 200)}`,
83
+ resolution: flow.steps?.join(" → ") ?? null,
84
+ occurrences: 5,
85
+ lastSeen: new Date().toISOString(),
86
+ });
87
+ }
88
+ }
89
+ }
90
+ }
91
+ return errors;
92
+ }
93
+ /**
94
+ * Read all playbooks and extract learnings (selectors, patterns, policy notes).
95
+ * These get loaded into memory's learningsCache so queryPatterns() finds them.
96
+ */
97
+ export function seedLearningsFromPlaybooks(playbooksDir) {
98
+ const playbooks = loadPlaybooks(playbooksDir);
99
+ const learnings = [];
100
+ for (const pb of playbooks) {
101
+ const platform = pb.platform ?? pb.id ?? "unknown";
102
+ const reliability = (pb.successCount ?? 0) + (pb.failCount ?? 0) > 0
103
+ ? (pb.successCount ?? 0) / ((pb.successCount ?? 0) + (pb.failCount ?? 0))
104
+ : 0.7;
105
+ // Selectors → learnings (verified working CSS selectors)
106
+ if (pb.selectors) {
107
+ for (const [group, sels] of Object.entries(pb.selectors)) {
108
+ for (const [name, selector] of Object.entries(sels)) {
109
+ // Skip notes/annotations (keys starting with _)
110
+ if (name.startsWith("_"))
111
+ continue;
112
+ learnings.push({
113
+ scope: `chrome/${platform}`,
114
+ pattern: `${group}.${name}: ${selector}`,
115
+ method: "cdp",
116
+ confidence: reliability,
117
+ successCount: Math.max(1, Math.round(reliability * 10)),
118
+ failCount: Math.round((1 - reliability) * 10),
119
+ lastSeen: new Date().toISOString(),
120
+ fix: null,
121
+ });
122
+ }
123
+ }
124
+ }
125
+ // Flow selectors → learnings
126
+ if (pb.flows) {
127
+ for (const [flowName, flow] of Object.entries(pb.flows)) {
128
+ if (flow.selectors) {
129
+ for (const [name, selector] of Object.entries(flow.selectors)) {
130
+ learnings.push({
131
+ scope: `chrome/${platform}/${flowName}`,
132
+ pattern: `${name}: ${selector}`,
133
+ method: "cdp",
134
+ confidence: reliability,
135
+ successCount: Math.max(1, Math.round(reliability * 10)),
136
+ failCount: Math.round((1 - reliability) * 10),
137
+ lastSeen: new Date().toISOString(),
138
+ fix: null,
139
+ });
140
+ }
141
+ }
142
+ }
143
+ }
144
+ // Policy notes → learnings (rate limits, safety rules)
145
+ if (pb.policyNotes) {
146
+ for (const [category, notes] of Object.entries(pb.policyNotes)) {
147
+ for (const note of notes) {
148
+ learnings.push({
149
+ scope: `policy/${platform}`,
150
+ pattern: `[${category}] ${note}`,
151
+ method: "cdp",
152
+ confidence: 1.0,
153
+ successCount: 10,
154
+ failCount: 0,
155
+ lastSeen: new Date().toISOString(),
156
+ fix: null,
157
+ });
158
+ }
159
+ }
160
+ }
161
+ // Detection expressions → learnings
162
+ if (pb.detection) {
163
+ for (const [name, expr] of Object.entries(pb.detection)) {
164
+ learnings.push({
165
+ scope: `chrome/${platform}/detection`,
166
+ pattern: `${name}: ${expr}`,
167
+ method: "cdp",
168
+ confidence: reliability,
169
+ successCount: Math.max(1, Math.round(reliability * 10)),
170
+ failCount: 0,
171
+ lastSeen: new Date().toISOString(),
172
+ fix: null,
173
+ });
174
+ }
175
+ }
176
+ }
177
+ return learnings;
178
+ }
179
+ // ── Helpers ──
180
+ function loadPlaybooks(dir) {
181
+ if (!fs.existsSync(dir))
182
+ return [];
183
+ const files = fs.readdirSync(dir).filter(f => f.endsWith(".json"));
184
+ const playbooks = [];
185
+ for (const file of files) {
186
+ try {
187
+ const raw = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8"));
188
+ // Ensure it has an id
189
+ if (!raw.id)
190
+ raw.id = file.replace(".json", "");
191
+ if (!raw.platform)
192
+ raw.platform = raw.id;
193
+ playbooks.push(raw);
194
+ }
195
+ catch {
196
+ // Skip unparseable files
197
+ }
198
+ }
199
+ return playbooks;
200
+ }
@@ -24,7 +24,7 @@ export class RecallEngine {
24
24
  * Find strategies matching a task description (~0ms, in-memory).
25
25
  * Strategies with high fail rates are penalized.
26
26
  */
27
- recallStrategies(query, limit = 5) {
27
+ recallStrategies(query, limit = 5, currentBundleId) {
28
28
  const strategies = this.store.readStrategies();
29
29
  if (strategies.length === 0)
30
30
  return [];
@@ -32,13 +32,13 @@ export class RecallEngine {
32
32
  if (queryTokens.length === 0)
33
33
  return [];
34
34
  const scored = strategies.map((s) => {
35
+ // Only match against task description, tags, and tool names.
36
+ // Step params (JS code, URLs, selectors) contain too many generic words
37
+ // and cause false positives against unrelated strategies.
35
38
  const targetTokens = new Set([
36
39
  ...tokenize(s.task),
37
40
  ...s.tags,
38
41
  ...s.steps.map((step) => step.tool),
39
- ...s.steps.flatMap((step) => Object.values(step.params)
40
- .filter((v) => typeof v === "string")
41
- .flatMap(tokenize)),
42
42
  ]);
43
43
  let matches = 0;
44
44
  for (const qt of queryTokens) {
@@ -51,7 +51,7 @@ export class RecallEngine {
51
51
  }
52
52
  const relevance = matches / queryTokens.length;
53
53
  const ageMs = Date.now() - new Date(s.lastUsed).getTime();
54
- const ageDays = ageMs / (1000 * 60 * 60 * 24);
54
+ const ageDays = Number.isFinite(ageMs) ? ageMs / (1000 * 60 * 60 * 24) : 0;
55
55
  const recency = Math.max(0.5, 1.0 - ageDays / 365);
56
56
  const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
57
57
  // Penalty for strategies that have failed — reduces score proportionally
@@ -60,7 +60,32 @@ export class RecallEngine {
60
60
  const reliabilityPenalty = totalAttempts > 0
61
61
  ? s.successCount / totalAttempts
62
62
  : 1;
63
- const score = relevance * recency * successBoost * reliabilityPenalty;
63
+ // App-context filtering: penalize strategies whose steps target a different app
64
+ let appContextFactor = 1.0;
65
+ if (currentBundleId) {
66
+ const stepsStr = s.steps.map((step) => JSON.stringify(step.params)).join(" ").toLowerCase();
67
+ const taskStr = s.task.toLowerCase();
68
+ const bundleLower = currentBundleId.toLowerCase();
69
+ // Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
70
+ const appName = bundleLower.split(".").pop() ?? "";
71
+ const mentionsCurrentApp = taskStr.includes(bundleLower) || taskStr.includes(appName)
72
+ || stepsStr.includes(bundleLower);
73
+ // Check if strategy targets a DIFFERENT app via focus/launch steps
74
+ const hasFocusStep = s.steps.some((step) => (step.tool === "focus" || step.tool === "launch") &&
75
+ step.params && "bundleId" in step.params &&
76
+ typeof step.params.bundleId === "string" &&
77
+ step.params.bundleId.toLowerCase() !== bundleLower);
78
+ if (hasFocusStep && !mentionsCurrentApp) {
79
+ appContextFactor = 0.1; // Heavy penalty for wrong-app strategies
80
+ }
81
+ else if (mentionsCurrentApp) {
82
+ appContextFactor = 1.5; // Boost for matching strategies
83
+ }
84
+ }
85
+ // Require at least 50% token overlap to be considered relevant
86
+ if (relevance < 0.5)
87
+ return { ...s, score: 0 };
88
+ const score = relevance * recency * successBoost * reliabilityPenalty * appContextFactor;
64
89
  return { ...s, score };
65
90
  });
66
91
  return scored
@@ -103,9 +128,13 @@ export class RecallEngine {
103
128
  * Tries fingerprint prefix match first (O(1)), then falls back to scan.
104
129
  * Skips unreliable strategies (failCount > successCount).
105
130
  */
106
- quickStrategyHint(recentTools) {
131
+ quickStrategyHint(recentTools, currentBundleId) {
107
132
  if (recentTools.length === 0)
108
133
  return null;
134
+ // Require at least 2 tools in the sequence to reduce false positives
135
+ // from single-tool matches (e.g. just "focus" matching every strategy)
136
+ if (recentTools.length < 2)
137
+ return null;
109
138
  const strategies = this.store.readStrategies();
110
139
  for (const s of strategies) {
111
140
  if (s.steps.length <= recentTools.length)
@@ -114,6 +143,18 @@ export class RecallEngine {
114
143
  const failCount = s.failCount ?? 0;
115
144
  if (failCount > s.successCount)
116
145
  continue;
146
+ // If we know the current app, prefer strategies that mention it
147
+ // and skip strategies clearly for a different app
148
+ if (currentBundleId) {
149
+ const taskLower = s.task.toLowerCase();
150
+ const bundleLower = currentBundleId.toLowerCase();
151
+ // Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
152
+ const appName = bundleLower.split(".").pop() ?? "";
153
+ const mentionsCurrentApp = taskLower.includes(appName) || taskLower.includes(bundleLower);
154
+ const mentionsOtherApp = !mentionsCurrentApp && /com\.\w+\.\w+/.test(s.task);
155
+ if (mentionsOtherApp)
156
+ continue; // strategy is for a different app
157
+ }
117
158
  const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
118
159
  const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
119
160
  if (matches) {
@@ -144,11 +185,22 @@ export class RecallEngine {
144
185
  }
145
186
  }
146
187
  /** Tokenize a string into lowercase keywords (3+ chars) */
188
+ /** Common automation verbs/nouns that match almost any strategy — filter them out */
189
+ const RECALL_STOPWORDS = new Set([
190
+ "open", "close", "click", "set", "get", "the", "and", "for", "from",
191
+ "into", "with", "then", "this", "that", "use", "run", "start", "stop",
192
+ "new", "add", "app", "settings", "window", "button", "text", "page",
193
+ "file", "menu", "tab", "navigate", "type", "select", "find", "wait",
194
+ "send", "save", "copy", "paste", "delete", "create", "edit", "view",
195
+ "show", "hide", "move", "drag", "drop", "enter", "press", "about",
196
+ "input", "form", "link", "image", "video", "upload", "download",
197
+ "first", "last", "next", "take", "result", "search",
198
+ ]);
147
199
  function tokenize(text) {
148
200
  return text
149
201
  .toLowerCase()
150
202
  .split(/[\W_]+/)
151
- .filter((w) => w.length >= 3);
203
+ .filter((w) => w.length >= 2 && !RECALL_STOPWORDS.has(w));
152
204
  }
153
205
  /** Simple string similarity: shared character bigrams / total bigrams */
154
206
  function stringSimilarity(a, b) {
@@ -29,6 +29,7 @@ import { writeFileAtomicSync } from "../util/atomic-write.js";
29
29
  import { MemoryStore } from "./store.js";
30
30
  import { SessionTracker } from "./session.js";
31
31
  import { RecallEngine } from "./recall.js";
32
+ import { seedLearningsFromPlaybooks } from "./playbook-seeds.js";
32
33
  // ── Defaults ─────────────────────────────────────
33
34
  const DEFAULT_POLICY = {
34
35
  maxConsecutiveErrors: 5,
@@ -90,6 +91,26 @@ export class MemoryService {
90
91
  this.ensureMemDir();
91
92
  // Load learnings
92
93
  this.learningsCache = this.readJsonlSafe("learnings.jsonl");
94
+ // Seed learnings from playbooks — selectors, detection, policy notes.
95
+ // These have scope prefixes like "chrome/figma" so they merge cleanly
96
+ // with runtime-discovered learnings without duplicating.
97
+ const playbooksDir = path.join(this.baseDir, "references");
98
+ const pbLearnings = seedLearningsFromPlaybooks(playbooksDir);
99
+ if (pbLearnings.length > 0) {
100
+ const existingKeys = new Set(this.learningsCache.map(l => `${l.scope}::${l.pattern}::${l.method}`));
101
+ let added = 0;
102
+ for (const pl of pbLearnings) {
103
+ const key = `${pl.scope}::${pl.pattern}::${pl.method}`;
104
+ if (!existingKeys.has(key)) {
105
+ this.learningsCache.push({
106
+ id: `pb_lrn_${added}`,
107
+ ...pl,
108
+ });
109
+ existingKeys.add(key);
110
+ added++;
111
+ }
112
+ }
113
+ }
93
114
  // Load existing snapshot if present (to restore mission/policy across restarts)
94
115
  const snapPath = this.filePath("state.json");
95
116
  if (fs.existsSync(snapPath)) {
@@ -285,17 +306,17 @@ export class MemoryService {
285
306
  queryErrors(tool) {
286
307
  return this.recall.recallErrors(tool);
287
308
  }
288
- /** Fuzzy-match strategies by query string. */
289
- recallStrategies(query, limit) {
290
- return this.recall.recallStrategies(query, limit);
309
+ /** Fuzzy-match strategies by query string. Optionally filter by current app. */
310
+ recallStrategies(query, limit, currentBundleId) {
311
+ return this.recall.recallStrategies(query, limit, currentBundleId);
291
312
  }
292
313
  /** Quick error check for interceptor (~0ms). */
293
314
  quickErrorCheck(tool) {
294
315
  return this.recall.quickErrorCheck(tool);
295
316
  }
296
317
  /** Quick strategy hint for interceptor (~0ms). */
297
- quickStrategyHint(recentTools) {
298
- return this.recall.quickStrategyHint(recentTools);
318
+ quickStrategyHint(recentTools, currentBundleId) {
319
+ return this.recall.quickStrategyHint(recentTools, currentBundleId);
299
320
  }
300
321
  /** Record strategy outcome for feedback loop. */
301
322
  recordStrategyOutcome(fingerprint, success) {
@@ -397,6 +418,10 @@ export class MemoryService {
397
418
  return [];
398
419
  let text;
399
420
  try {
421
+ // Guard against oversized files (same 10MB limit as LearningEngine)
422
+ const stat = fs.statSync(fp);
423
+ if (stat.size > 10 * 1024 * 1024)
424
+ return [];
400
425
  text = fs.readFileSync(fp, "utf-8").trim();
401
426
  }
402
427
  catch {
@@ -26,7 +26,9 @@
26
26
  import fs from "node:fs";
27
27
  import path from "node:path";
28
28
  import { writeFileAtomicSync } from "../util/atomic-write.js";
29
+ import { redactPII } from "../util/sanitize.js";
29
30
  import { SEED_STRATEGIES } from "./seeds.js";
31
+ import { seedErrorsFromPlaybooks } from "./playbook-seeds.js";
30
32
  const MAX_ACTION_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
31
33
  const MAX_STRATEGIES = 500;
32
34
  const MAX_ERRORS = 200;
@@ -52,7 +54,9 @@ export class MemoryStore {
52
54
  // Global flag — only register exit handlers once across all instances
53
55
  static exitHandlerRegistered = false;
54
56
  static activeInstance = null;
57
+ baseDir;
55
58
  constructor(baseDir) {
59
+ this.baseDir = baseDir;
56
60
  this.dir = path.join(baseDir, ".screenhand", "memory");
57
61
  this.lockPath = path.join(this.dir, ".lock");
58
62
  }
@@ -70,11 +74,23 @@ export class MemoryStore {
70
74
  if (this.strategiesCache.length === 0 && isFirstBoot) {
71
75
  for (const s of SEED_STRATEGIES)
72
76
  this.strategiesCache.push(s);
73
- this.writeLinesSync("strategies.jsonl", this.strategiesCache);
77
+ this.writeStrategiesRedacted();
74
78
  }
75
79
  this.enforceStrategyLimit();
76
80
  this.rebuildFingerprintIndex();
77
81
  this.errorsCache = this.readLinesSafe("errors.jsonl");
82
+ // Seed errors from playbooks — merge curated platform knowledge into memory.
83
+ // Uses pb_err_ prefix IDs so we can identify and refresh them on each boot.
84
+ const playbooksDir = path.join(this.baseDir, "references");
85
+ const pbErrors = seedErrorsFromPlaybooks(playbooksDir);
86
+ if (pbErrors.length > 0) {
87
+ // Remove stale playbook-seeded errors (they'll be re-added with fresh data)
88
+ this.errorsCache = this.errorsCache.filter(e => !e.id.startsWith("pb_err_"));
89
+ // Merge fresh playbook errors
90
+ for (const e of pbErrors) {
91
+ this.errorsCache.push(e);
92
+ }
93
+ }
78
94
  this.enforceErrorLimit();
79
95
  // Build action stats without caching all entries
80
96
  const actions = this.readLinesSafe("actions.jsonl");
@@ -246,8 +262,9 @@ export class MemoryStore {
246
262
  if (!this.hasLock)
247
263
  return;
248
264
  this.rotateActionsIfNeeded();
249
- // Buffer the write
250
- this.pendingActionWrites.push(JSON.stringify(entry) + "\n");
265
+ // S75 Option C: Redact PII before persisting to disk (not in live responses)
266
+ const redacted = { ...entry, result: entry.result ? redactPII(entry.result) : entry.result };
267
+ this.pendingActionWrites.push(JSON.stringify(redacted) + "\n");
251
268
  // Schedule batch flush (debounced 100ms)
252
269
  if (!this.flushTimer) {
253
270
  this.flushTimer = setTimeout(() => {
@@ -321,7 +338,19 @@ export class MemoryStore {
321
338
  this.rebuildFingerprintIndex();
322
339
  }
323
340
  }
324
- this.writeLinesSync("strategies.jsonl", this.strategiesCache);
341
+ this.writeStrategiesRedacted();
342
+ }
343
+ /** S75 Option C: Redact PII from strategies before any disk write */
344
+ writeStrategiesRedacted() {
345
+ const redacted = this.strategiesCache.map(s => ({
346
+ ...s,
347
+ task: redactPII(s.task),
348
+ steps: s.steps.map(st => ({
349
+ ...st,
350
+ params: Object.fromEntries(Object.entries(st.params).map(([k, v]) => [k, typeof v === "string" ? redactPII(v) : v])),
351
+ })),
352
+ }));
353
+ this.writeLinesSync("strategies.jsonl", redacted);
325
354
  }
326
355
  /** O(1) exact lookup by tool sequence fingerprint */
327
356
  lookupByFingerprint(fingerprint) {
@@ -339,7 +368,7 @@ export class MemoryStore {
339
368
  else {
340
369
  strategy.failCount = (strategy.failCount ?? 0) + 1;
341
370
  }
342
- this.writeLinesSync("strategies.jsonl", this.strategiesCache);
371
+ this.writeStrategiesRedacted();
343
372
  }
344
373
  /** Read from cache — ~0ms */
345
374
  readStrategies() {