screenhand 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +458 -93
  2. package/dist/.audit-log.jsonl +55 -0
  3. package/dist/.screenhand/memory/.lock +1 -0
  4. package/dist/.screenhand/memory/actions.jsonl +85 -0
  5. package/dist/.screenhand/memory/errors.jsonl +5 -0
  6. package/dist/.screenhand/memory/errors.jsonl.bak +4 -0
  7. package/dist/.screenhand/memory/state.json +35 -0
  8. package/dist/.screenhand/memory/state.json.bak +35 -0
  9. package/dist/.screenhand/memory/strategies.jsonl +12 -0
  10. package/dist/agent/cli.js +73 -0
  11. package/dist/agent/loop.js +258 -0
  12. package/dist/config.js +9 -0
  13. package/dist/index.js +56 -0
  14. package/dist/logging/timeline-logger.js +29 -0
  15. package/dist/mcp/mcp-stdio-server.js +448 -0
  16. package/dist/mcp/server.js +347 -0
  17. package/dist/mcp-desktop.js +2731 -0
  18. package/dist/mcp-entry.js +59 -0
  19. package/dist/memory/recall.js +160 -0
  20. package/dist/memory/research.js +98 -0
  21. package/dist/memory/seeds.js +89 -0
  22. package/dist/memory/session.js +161 -0
  23. package/dist/memory/store.js +391 -0
  24. package/dist/memory/types.js +4 -0
  25. package/dist/monitor/codex-monitor.js +377 -0
  26. package/dist/monitor/task-queue.js +84 -0
  27. package/dist/monitor/types.js +49 -0
  28. package/dist/native/bridge-client.js +174 -0
  29. package/dist/native/macos-bridge-client.js +5 -0
  30. package/dist/npm-publish-helper.js +117 -0
  31. package/dist/npm-token-cdp.js +113 -0
  32. package/dist/npm-token-create.js +135 -0
  33. package/dist/npm-token-finish.js +126 -0
  34. package/dist/playbook/engine.js +193 -0
  35. package/dist/playbook/index.js +4 -0
  36. package/dist/playbook/recorder.js +519 -0
  37. package/dist/playbook/runner.js +392 -0
  38. package/dist/playbook/store.js +166 -0
  39. package/dist/playbook/types.js +4 -0
  40. package/dist/runtime/accessibility-adapter.js +377 -0
  41. package/dist/runtime/app-adapter.js +48 -0
  42. package/dist/runtime/applescript-adapter.js +283 -0
  43. package/dist/runtime/ax-role-map.js +80 -0
  44. package/dist/runtime/browser-adapter.js +36 -0
  45. package/dist/runtime/cdp-chrome-adapter.js +505 -0
  46. package/dist/runtime/composite-adapter.js +205 -0
  47. package/dist/runtime/executor.js +250 -0
  48. package/dist/runtime/locator-cache.js +12 -0
  49. package/dist/runtime/planning-loop.js +47 -0
  50. package/dist/runtime/service.js +372 -0
  51. package/dist/runtime/session-manager.js +28 -0
  52. package/dist/runtime/state-observer.js +105 -0
  53. package/dist/runtime/vision-adapter.js +208 -0
  54. package/dist/scripts/codex-monitor-daemon.js +335 -0
  55. package/dist/scripts/supervisor-daemon.js +272 -0
  56. package/dist/scripts/worker-daemon.js +228 -0
  57. package/dist/src/agent/cli.js +82 -0
  58. package/dist/src/agent/loop.js +274 -0
  59. package/{src/config.ts → dist/src/config.js} +5 -10
  60. package/{src/index.ts → dist/src/index.js} +32 -52
  61. package/dist/src/jobs/manager.js +237 -0
  62. package/dist/src/jobs/runner.js +683 -0
  63. package/dist/src/jobs/store.js +102 -0
  64. package/dist/src/jobs/types.js +30 -0
  65. package/dist/src/jobs/worker.js +97 -0
  66. package/dist/src/logging/timeline-logger.js +45 -0
  67. package/dist/src/mcp/mcp-stdio-server.js +464 -0
  68. package/dist/src/mcp/server.js +363 -0
  69. package/dist/src/mcp-entry.js +60 -0
  70. package/dist/src/memory/recall.js +170 -0
  71. package/dist/src/memory/research.js +104 -0
  72. package/dist/src/memory/seeds.js +101 -0
  73. package/dist/src/memory/service.js +421 -0
  74. package/dist/src/memory/session.js +169 -0
  75. package/dist/src/memory/store.js +422 -0
  76. package/dist/src/memory/types.js +17 -0
  77. package/dist/src/monitor/codex-monitor.js +382 -0
  78. package/dist/src/monitor/task-queue.js +97 -0
  79. package/dist/src/monitor/types.js +62 -0
  80. package/dist/src/native/bridge-client.js +190 -0
  81. package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
  82. package/dist/src/playbook/engine.js +201 -0
  83. package/dist/src/playbook/index.js +20 -0
  84. package/dist/src/playbook/recorder.js +535 -0
  85. package/dist/src/playbook/runner.js +408 -0
  86. package/dist/src/playbook/store.js +183 -0
  87. package/dist/src/playbook/types.js +17 -0
  88. package/dist/src/runtime/accessibility-adapter.js +393 -0
  89. package/dist/src/runtime/app-adapter.js +64 -0
  90. package/dist/src/runtime/applescript-adapter.js +299 -0
  91. package/dist/src/runtime/ax-role-map.js +96 -0
  92. package/dist/src/runtime/browser-adapter.js +52 -0
  93. package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
  94. package/dist/src/runtime/composite-adapter.js +221 -0
  95. package/dist/src/runtime/execution-contract.js +159 -0
  96. package/dist/src/runtime/executor.js +266 -0
  97. package/{src/runtime/locator-cache.ts → dist/src/runtime/locator-cache.js} +10 -15
  98. package/dist/src/runtime/planning-loop.js +63 -0
  99. package/dist/src/runtime/service.js +388 -0
  100. package/dist/src/runtime/session-manager.js +60 -0
  101. package/dist/src/runtime/state-observer.js +121 -0
  102. package/dist/src/runtime/vision-adapter.js +224 -0
  103. package/dist/src/supervisor/locks.js +186 -0
  104. package/dist/src/supervisor/supervisor.js +403 -0
  105. package/dist/src/supervisor/types.js +30 -0
  106. package/dist/src/test-mcp-protocol.js +154 -0
  107. package/dist/src/types.js +17 -0
  108. package/dist/src/util/atomic-write.js +118 -0
  109. package/dist/test-mcp-protocol.js +138 -0
  110. package/dist/types.js +1 -0
  111. package/package.json +18 -4
  112. package/.claude/commands/automate.md +0 -28
  113. package/.claude/commands/debug-ui.md +0 -19
  114. package/.claude/commands/screenshot.md +0 -15
  115. package/.github/FUNDING.yml +0 -1
  116. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
  117. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  118. package/.mcp.json +0 -8
  119. package/DESKTOP_MCP_GUIDE.md +0 -92
  120. package/SECURITY.md +0 -44
  121. package/docs/architecture.md +0 -47
  122. package/install-skills.sh +0 -19
  123. package/mcp-bridge.ts +0 -271
  124. package/mcp-desktop.ts +0 -1221
  125. package/native/macos-bridge/Package.swift +0 -21
  126. package/native/macos-bridge/Sources/AccessibilityBridge.swift +0 -261
  127. package/native/macos-bridge/Sources/AppManagement.swift +0 -129
  128. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +0 -242
  129. package/native/macos-bridge/Sources/ObserverBridge.swift +0 -120
  130. package/native/macos-bridge/Sources/VisionBridge.swift +0 -80
  131. package/native/macos-bridge/Sources/main.swift +0 -345
  132. package/native/windows-bridge/AppManagement.cs +0 -234
  133. package/native/windows-bridge/InputBridge.cs +0 -436
  134. package/native/windows-bridge/Program.cs +0 -265
  135. package/native/windows-bridge/ScreenCapture.cs +0 -329
  136. package/native/windows-bridge/UIAutomationBridge.cs +0 -571
  137. package/native/windows-bridge/WindowsBridge.csproj +0 -17
  138. package/playbooks/devpost.json +0 -186
  139. package/playbooks/instagram.json +0 -41
  140. package/playbooks/instagram_v2.json +0 -201
  141. package/playbooks/x_v1.json +0 -211
  142. package/scripts/devpost-live-loop.mjs +0 -421
  143. package/src/logging/timeline-logger.ts +0 -55
  144. package/src/mcp/server.ts +0 -449
  145. package/src/memory/recall.ts +0 -191
  146. package/src/memory/research.ts +0 -146
  147. package/src/memory/seeds.ts +0 -123
  148. package/src/memory/session.ts +0 -201
  149. package/src/memory/store.ts +0 -434
  150. package/src/memory/types.ts +0 -69
  151. package/src/native/bridge-client.ts +0 -239
  152. package/src/runtime/accessibility-adapter.ts +0 -487
  153. package/src/runtime/app-adapter.ts +0 -169
  154. package/src/runtime/applescript-adapter.ts +0 -376
  155. package/src/runtime/ax-role-map.ts +0 -102
  156. package/src/runtime/browser-adapter.ts +0 -129
  157. package/src/runtime/cdp-chrome-adapter.ts +0 -676
  158. package/src/runtime/composite-adapter.ts +0 -274
  159. package/src/runtime/executor.ts +0 -396
  160. package/src/runtime/planning-loop.ts +0 -81
  161. package/src/runtime/service.ts +0 -448
  162. package/src/runtime/session-manager.ts +0 -50
  163. package/src/runtime/state-observer.ts +0 -136
  164. package/src/runtime/vision-adapter.ts +0 -297
  165. package/src/types.ts +0 -297
  166. package/tests/bridge-client.test.ts +0 -176
  167. package/tests/browser-stealth.test.ts +0 -210
  168. package/tests/composite-adapter.test.ts +0 -64
  169. package/tests/mcp-server.test.ts +0 -151
  170. package/tests/memory-recall.test.ts +0 -339
  171. package/tests/memory-research.test.ts +0 -159
  172. package/tests/memory-seeds.test.ts +0 -120
  173. package/tests/memory-store.test.ts +0 -392
  174. package/tests/types.test.ts +0 -92
  175. package/tsconfig.check.json +0 -17
  176. package/tsconfig.json +0 -19
  177. package/vitest.config.ts +0 -8
@@ -0,0 +1,101 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ let seedCounter = 0;
18
+ function makeFingerprint(tools) {
19
+ return tools.join("→");
20
+ }
21
+ function seed(task, steps, tags) {
22
+ seedCounter++;
23
+ return {
24
+ id: `seed_${String(seedCounter).padStart(3, "0")}`,
25
+ task,
26
+ steps,
27
+ totalDurationMs: 0,
28
+ successCount: 10,
29
+ failCount: 0,
30
+ lastUsed: new Date().toISOString(),
31
+ tags,
32
+ fingerprint: makeFingerprint(steps.map((s) => s.tool)),
33
+ };
34
+ }
35
+ export const SEED_STRATEGIES = [
36
+ // 1. Take a photo with Photo Booth
37
+ seed("Take a photo with Photo Booth", [
38
+ { tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
39
+ { tool: "ui_press", params: { title: "Take Photo" } },
40
+ ], ["photo", "booth", "camera"]),
41
+ // 2. Open a URL in Chrome
42
+ seed("Open a URL in Chrome", [
43
+ { tool: "launch", params: { bundleId: "com.google.Chrome" } },
44
+ { tool: "browser_navigate", params: { url: "" } },
45
+ ], ["chrome", "browse", "url"]),
46
+ // 3. Save current document
47
+ seed("Save current document", [
48
+ { tool: "focus", params: { bundleId: "" } },
49
+ { tool: "key", params: { combo: "cmd+s" } },
50
+ ], ["save", "document"]),
51
+ // 4. Copy from one app and paste into another
52
+ seed("Copy from one app and paste into another", [
53
+ { tool: "focus", params: { bundleId: "" } },
54
+ { tool: "key", params: { combo: "cmd+c" } },
55
+ { tool: "focus", params: { bundleId: "" } },
56
+ { tool: "key", params: { combo: "cmd+v" } },
57
+ ], ["copy", "paste"]),
58
+ // 5. Navigate to a folder in Finder
59
+ seed("Navigate to a folder in Finder", [
60
+ { tool: "focus", params: { bundleId: "com.apple.finder" } },
61
+ { tool: "key", params: { combo: "cmd+shift+g" } },
62
+ { tool: "type_text", params: { text: "" } },
63
+ ], ["finder", "folder", "navigate"]),
64
+ // 6. Create a new folder in Finder
65
+ seed("Create a new folder in Finder", [
66
+ { tool: "focus", params: { bundleId: "com.apple.finder" } },
67
+ { tool: "key", params: { combo: "cmd+shift+n" } },
68
+ { tool: "type_text", params: { text: "" } },
69
+ ], ["finder", "folder", "create"]),
70
+ // 7. Close the current window
71
+ seed("Close the current window", [
72
+ { tool: "focus", params: { bundleId: "" } },
73
+ { tool: "key", params: { combo: "cmd+w" } },
74
+ ], ["close", "window"]),
75
+ // 8. Select all and copy
76
+ seed("Select all content and copy", [
77
+ { tool: "focus", params: { bundleId: "" } },
78
+ { tool: "key", params: { combo: "cmd+a" } },
79
+ { tool: "key", params: { combo: "cmd+c" } },
80
+ ], ["select", "all", "copy"]),
81
+ // 9. List running apps
82
+ seed("List all running applications", [
83
+ { tool: "apps", params: {} },
84
+ ], ["apps", "list", "running"]),
85
+ // 10. Inspect app UI tree
86
+ seed("Inspect an app's UI element tree", [
87
+ { tool: "focus", params: { bundleId: "" } },
88
+ { tool: "ui_tree", params: { pid: 0 } },
89
+ ], ["inspect", "tree", "accessibility"]),
90
+ // 11. Open a new tab in Chrome and navigate
91
+ seed("Open a new Chrome tab and navigate to URL", [
92
+ { tool: "focus", params: { bundleId: "com.google.Chrome" } },
93
+ { tool: "key", params: { combo: "cmd+t" } },
94
+ { tool: "browser_navigate", params: { url: "" } },
95
+ ], ["chrome", "tab", "new"]),
96
+ // 12. Export as PDF via menu
97
+ seed("Export document as PDF", [
98
+ { tool: "focus", params: { bundleId: "" } },
99
+ { tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
100
+ ], ["export", "pdf"]),
101
+ ];
@@ -0,0 +1,421 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * MemoryService — unified facade over MemoryStore, SessionTracker, and RecallEngine
19
+ *
20
+ * Single entry-point for all memory operations. Adds:
21
+ * - state.json snapshot (debounced, written on every action)
22
+ * - learnings.jsonl (verified patterns, separate from strategies)
23
+ * - MemoryPolicy (error/stall thresholds)
24
+ * - Mission tracking
25
+ */
26
+ import fs from "node:fs";
27
+ import path from "node:path";
28
+ import { writeFileAtomicSync } from "../util/atomic-write.js";
29
+ import { MemoryStore } from "./store.js";
30
+ import { SessionTracker } from "./session.js";
31
+ import { RecallEngine } from "./recall.js";
32
+ // ── Defaults ─────────────────────────────────────
33
+ const DEFAULT_POLICY = {
34
+ maxConsecutiveErrors: 5,
35
+ stallThresholdMs: 300_000,
36
+ escalateAfterRetries: 3,
37
+ pauseBetweenActionsMs: 500,
38
+ };
39
+ const SNAPSHOT_DEBOUNCE_MS = 200;
40
+ const MAX_LEARNINGS = 1000;
41
+ // ── Service ──────────────────────────────────────
42
+ export class MemoryService {
43
+ store;
44
+ session;
45
+ recall;
46
+ baseDir;
47
+ memDir;
48
+ learningsCache = [];
49
+ snapshot;
50
+ snapshotTimer = null;
51
+ consecutiveErrors = 0;
52
+ lastError = null;
53
+ actionsFailed = 0;
54
+ actionsTotal = 0;
55
+ sessionStartedAt;
56
+ initialized = false;
57
+ constructor(baseDir) {
58
+ this.baseDir = baseDir;
59
+ this.memDir = path.join(baseDir, ".screenhand", "memory");
60
+ this.store = new MemoryStore(baseDir);
61
+ this.session = new SessionTracker(this.store);
62
+ this.recall = new RecallEngine(this.store);
63
+ this.sessionStartedAt = new Date().toISOString();
64
+ this.snapshot = {
65
+ session: {
66
+ id: this.session.getSessionId(),
67
+ client: "unknown",
68
+ startedAt: this.sessionStartedAt,
69
+ lastActionAt: this.sessionStartedAt,
70
+ },
71
+ mission: { current: null, phase: "idle" },
72
+ health: {
73
+ actionsTotal: 0,
74
+ actionsFailed: 0,
75
+ successRate: 1,
76
+ lastError: null,
77
+ consecutiveErrors: 0,
78
+ },
79
+ patterns: { topWorking: [], topFailing: [], knownBlockers: [] },
80
+ policy: { ...DEFAULT_POLICY },
81
+ };
82
+ }
83
+ // ── Initialization ───────────────────────────────
84
+ /** Load all caches from disk. Call once at startup. */
85
+ init() {
86
+ if (this.initialized)
87
+ return;
88
+ this.initialized = true;
89
+ this.store.init();
90
+ this.ensureMemDir();
91
+ // Load learnings
92
+ this.learningsCache = this.readJsonlSafe("learnings.jsonl");
93
+ // Load existing snapshot if present (to restore mission/policy across restarts)
94
+ const snapPath = this.filePath("state.json");
95
+ if (fs.existsSync(snapPath)) {
96
+ try {
97
+ const raw = fs.readFileSync(snapPath, "utf-8");
98
+ const loaded = JSON.parse(raw);
99
+ // Restore mission and policy from previous run
100
+ if (loaded.mission)
101
+ this.snapshot.mission = loaded.mission;
102
+ if (loaded.policy)
103
+ this.snapshot.policy = { ...DEFAULT_POLICY, ...loaded.policy };
104
+ }
105
+ catch {
106
+ // Corrupted snapshot — start fresh
107
+ }
108
+ }
109
+ // Sync stats from store
110
+ const stats = this.store.getStats();
111
+ this.actionsTotal = stats.totalActions;
112
+ this.actionsFailed = stats.totalActions - Math.round(stats.successRate * stats.totalActions);
113
+ this.rebuildPatterns();
114
+ this.updateHealthInSnapshot();
115
+ this.writeSnapshotSync();
116
+ }
117
+ // ── Snapshot ─────────────────────────────────────
118
+ /** Get the current in-memory snapshot (zero-cost). */
119
+ getSnapshot() {
120
+ return this.snapshot;
121
+ }
122
+ scheduleSnapshotWrite() {
123
+ if (this.snapshotTimer)
124
+ return;
125
+ this.snapshotTimer = setTimeout(() => {
126
+ this.snapshotTimer = null;
127
+ this.writeSnapshotSync();
128
+ }, SNAPSHOT_DEBOUNCE_MS);
129
+ }
130
+ writeSnapshotSync() {
131
+ this.ensureMemDir();
132
+ try {
133
+ writeFileAtomicSync(this.filePath("state.json"), JSON.stringify(this.snapshot, null, 2));
134
+ }
135
+ catch {
136
+ // Non-critical
137
+ }
138
+ }
139
+ updateHealthInSnapshot() {
140
+ this.snapshot.health = {
141
+ actionsTotal: this.actionsTotal,
142
+ actionsFailed: this.actionsFailed,
143
+ successRate: this.actionsTotal > 0 ? (this.actionsTotal - this.actionsFailed) / this.actionsTotal : 1,
144
+ lastError: this.lastError,
145
+ consecutiveErrors: this.consecutiveErrors,
146
+ };
147
+ }
148
+ rebuildPatterns() {
149
+ // Top working: learnings with high confidence, sorted desc
150
+ const working = this.learningsCache
151
+ .filter((l) => l.confidence >= 0.6 && l.successCount > l.failCount)
152
+ .sort((a, b) => b.confidence - a.confidence)
153
+ .slice(0, 10)
154
+ .map((l) => `${l.scope}: ${l.pattern} (${l.method})`);
155
+ // Top failing: learnings with low confidence or high fail count
156
+ const failing = this.learningsCache
157
+ .filter((l) => l.failCount > l.successCount)
158
+ .sort((a, b) => b.failCount - a.failCount)
159
+ .slice(0, 10)
160
+ .map((l) => `${l.scope}: ${l.pattern} (${l.method})`);
161
+ // Known blockers: errors with no resolution
162
+ const errors = this.store.readErrors();
163
+ const blockers = errors
164
+ .filter((e) => !e.resolution && e.occurrences >= 2)
165
+ .sort((a, b) => b.occurrences - a.occurrences)
166
+ .slice(0, 10)
167
+ .map((e) => `${e.tool}: ${e.error}`);
168
+ this.snapshot.patterns = {
169
+ topWorking: working,
170
+ topFailing: failing,
171
+ knownBlockers: blockers,
172
+ };
173
+ }
174
+ // ── Recording ────────────────────────────────────
175
+ /** Record an action event. Delegates to store + session tracker, updates snapshot. */
176
+ recordEvent(entry) {
177
+ this.store.appendAction(entry);
178
+ this.session.recordAction(entry);
179
+ this.actionsTotal++;
180
+ if (!entry.success) {
181
+ this.actionsFailed++;
182
+ this.consecutiveErrors++;
183
+ this.lastError = entry.error;
184
+ }
185
+ else {
186
+ this.consecutiveErrors = 0;
187
+ }
188
+ this.snapshot.session.id = this.session.getSessionId();
189
+ this.snapshot.session.lastActionAt = entry.timestamp;
190
+ this.updateHealthInSnapshot();
191
+ this.scheduleSnapshotWrite();
192
+ }
193
+ /** Record an error pattern. Delegates to store, optionally creates a learning. */
194
+ recordError(tool, error, fix, scope) {
195
+ const pattern = {
196
+ id: "err_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
197
+ tool,
198
+ params: {},
199
+ error,
200
+ resolution: fix,
201
+ occurrences: 1,
202
+ lastSeen: new Date().toISOString(),
203
+ };
204
+ this.store.appendError(pattern);
205
+ // If a fix is provided, record it as a learning
206
+ if (fix && scope) {
207
+ this.recordLearning({
208
+ scope,
209
+ pattern: error,
210
+ method: "ax",
211
+ confidence: 0.5,
212
+ successCount: 0,
213
+ failCount: 1,
214
+ lastSeen: new Date().toISOString(),
215
+ fix,
216
+ });
217
+ }
218
+ this.rebuildPatterns();
219
+ this.scheduleSnapshotWrite();
220
+ }
221
+ // ── Learnings ────────────────────────────────────
222
+ /** Append a verified learning to learnings.jsonl. */
223
+ recordLearning(learning) {
224
+ const full = {
225
+ id: "lrn_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
226
+ ...learning,
227
+ };
228
+ // Check for existing learning with same scope + pattern + method
229
+ const idx = this.learningsCache.findIndex((l) => l.scope === full.scope && l.pattern === full.pattern && l.method === full.method);
230
+ if (idx >= 0) {
231
+ const existing = this.learningsCache[idx];
232
+ this.learningsCache[idx] = {
233
+ ...existing,
234
+ successCount: existing.successCount + full.successCount,
235
+ failCount: existing.failCount + full.failCount,
236
+ confidence: this.computeConfidence(existing.successCount + full.successCount, existing.failCount + full.failCount),
237
+ lastSeen: full.lastSeen,
238
+ fix: full.fix ?? existing.fix,
239
+ };
240
+ }
241
+ else {
242
+ this.learningsCache.push(full);
243
+ this.enforceLearningsLimit();
244
+ }
245
+ this.writeLearningsAsync();
246
+ this.rebuildPatterns();
247
+ this.scheduleSnapshotWrite();
248
+ }
249
+ /** Search learnings by scope and/or method. */
250
+ queryPatterns(scope, method) {
251
+ let results = this.learningsCache;
252
+ if (scope) {
253
+ results = results.filter((l) => l.scope === scope || l.scope.startsWith(scope + "/"));
254
+ }
255
+ if (method) {
256
+ results = results.filter((l) => l.method === method);
257
+ }
258
+ return results;
259
+ }
260
+ computeConfidence(success, fail) {
261
+ const total = success + fail;
262
+ if (total === 0)
263
+ return 0;
264
+ return success / total;
265
+ }
266
+ enforceLearningsLimit() {
267
+ if (this.learningsCache.length <= MAX_LEARNINGS)
268
+ return;
269
+ // Evict lowest-confidence, oldest learnings
270
+ this.learningsCache.sort((a, b) => {
271
+ const confDiff = a.confidence - b.confidence;
272
+ if (Math.abs(confDiff) > 0.1)
273
+ return confDiff;
274
+ return new Date(a.lastSeen).getTime() - new Date(b.lastSeen).getTime();
275
+ });
276
+ this.learningsCache = this.learningsCache.slice(-MAX_LEARNINGS);
277
+ }
278
+ writeLearningsAsync() {
279
+ this.ensureMemDir();
280
+ const data = this.learningsCache.map((l) => JSON.stringify(l)).join("\n") + (this.learningsCache.length ? "\n" : "");
281
+ fs.writeFile(this.filePath("learnings.jsonl"), data, () => { });
282
+ }
283
+ // ── Recall (delegates to RecallEngine) ───────────
284
+ /** Search error patterns, optionally filtered by tool. */
285
+ queryErrors(tool) {
286
+ return this.recall.recallErrors(tool);
287
+ }
288
+ /** Fuzzy-match strategies by query string. */
289
+ recallStrategies(query, limit) {
290
+ return this.recall.recallStrategies(query, limit);
291
+ }
292
+ /** Quick error check for interceptor (~0ms). */
293
+ quickErrorCheck(tool) {
294
+ return this.recall.quickErrorCheck(tool);
295
+ }
296
+ /** Quick strategy hint for interceptor (~0ms). */
297
+ quickStrategyHint(recentTools) {
298
+ return this.recall.quickStrategyHint(recentTools);
299
+ }
300
+ /** Record strategy outcome for feedback loop. */
301
+ recordStrategyOutcome(fingerprint, success) {
302
+ this.store.recordStrategyOutcome(fingerprint, success);
303
+ }
304
+ // ── Session / Strategy ───────────────────────────
305
+ /** Get the current session ID. */
306
+ getSessionId() {
307
+ return this.session.getSessionId();
308
+ }
309
+ /** Get recent tool names from session buffer. */
310
+ getRecentToolNames(limit) {
311
+ return this.session.getRecentToolNames(limit);
312
+ }
313
+ /** End current session and save a strategy if successful. */
314
+ saveStrategy(task, tags) {
315
+ const strategy = this.session.endSession(true, task);
316
+ if (strategy && tags && tags.length > 0) {
317
+ // Merge additional tags
318
+ const merged = new Set([...strategy.tags, ...tags]);
319
+ strategy.tags = [...merged];
320
+ }
321
+ return strategy;
322
+ }
323
+ /** Read raw actions from store (for exports/playbooks). */
324
+ readActions() {
325
+ return this.store.readActions();
326
+ }
327
+ /** Read raw errors from store. */
328
+ readErrors() {
329
+ return this.store.readErrors();
330
+ }
331
+ /** Read raw strategies from store. */
332
+ readStrategies() {
333
+ return this.store.readStrategies();
334
+ }
335
+ /** Append an error pattern directly (for interceptor compatibility). */
336
+ appendError(pattern) {
337
+ this.store.appendError(pattern);
338
+ }
339
+ /** Append a strategy directly. */
340
+ appendStrategy(strategy) {
341
+ this.store.appendStrategy(strategy);
342
+ }
343
+ // ── Stats ────────────────────────────────────────
344
+ /** Get aggregate memory stats. */
345
+ getStats() {
346
+ return this.store.getStats();
347
+ }
348
+ // ── Mission ──────────────────────────────────────
349
+ /** Set the current mission and optionally a phase. */
350
+ setMission(mission, phase) {
351
+ this.snapshot.mission.current = mission;
352
+ if (phase)
353
+ this.snapshot.mission.phase = phase;
354
+ this.scheduleSnapshotWrite();
355
+ }
356
+ /** Set the client identifier (e.g., "claude-code", "mcp-desktop"). */
357
+ setClient(client) {
358
+ this.snapshot.session.client = client;
359
+ this.scheduleSnapshotWrite();
360
+ }
361
+ // ── Clear ────────────────────────────────────────
362
+ /** Clear specific memory categories or everything. */
363
+ clear(what) {
364
+ if (what === "learnings" || what === "all") {
365
+ this.learningsCache = [];
366
+ const fp = this.filePath("learnings.jsonl");
367
+ if (fs.existsSync(fp))
368
+ fs.writeFileSync(fp, "");
369
+ }
370
+ if (what !== "learnings") {
371
+ // Delegate non-learnings clears to the store
372
+ const storeWhat = what === "all" ? "all" : what;
373
+ this.store.clear(storeWhat);
374
+ }
375
+ if (what === "all" || what === "actions") {
376
+ this.actionsTotal = 0;
377
+ this.actionsFailed = 0;
378
+ this.consecutiveErrors = 0;
379
+ this.lastError = null;
380
+ }
381
+ this.updateHealthInSnapshot();
382
+ this.rebuildPatterns();
383
+ this.writeSnapshotSync();
384
+ }
385
+ // ── Helpers ──────────────────────────────────────
386
+ ensureMemDir() {
387
+ if (!fs.existsSync(this.memDir)) {
388
+ fs.mkdirSync(this.memDir, { recursive: true });
389
+ }
390
+ }
391
+ filePath(name) {
392
+ return path.join(this.memDir, name);
393
+ }
394
+ readJsonlSafe(file) {
395
+ const fp = this.filePath(file);
396
+ if (!fs.existsSync(fp))
397
+ return [];
398
+ let text;
399
+ try {
400
+ text = fs.readFileSync(fp, "utf-8").trim();
401
+ }
402
+ catch {
403
+ return [];
404
+ }
405
+ if (!text)
406
+ return [];
407
+ const results = [];
408
+ for (const line of text.split("\n")) {
409
+ const trimmed = line.trim();
410
+ if (!trimmed)
411
+ continue;
412
+ try {
413
+ results.push(JSON.parse(trimmed));
414
+ }
415
+ catch {
416
+ // Skip corrupted line
417
+ }
418
+ }
419
+ return results;
420
+ }
421
+ }
@@ -0,0 +1,169 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ import { MemoryStore } from "./store.js";
18
+ const SESSION_GAP_MS = 60_000; // 60s gap = new session
19
+ const MAX_BUFFER_SIZE = 100;
20
+ const MIN_AUTO_SAVE_STEPS = 3; // Need at least 3 successful steps to auto-save
21
+ export class SessionTracker {
22
+ store;
23
+ sessionId;
24
+ taskDescription = null;
25
+ buffer = [];
26
+ lastActionTime = 0;
27
+ constructor(store) {
28
+ this.store = store;
29
+ this.sessionId = SessionTracker.generateId();
30
+ }
31
+ static generateId() {
32
+ return "s_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
33
+ }
34
+ /** Start (or restart) a named task session */
35
+ startSession(taskDescription) {
36
+ // Auto-save previous session if it had successful actions
37
+ this.tryAutoSave();
38
+ this.sessionId = SessionTracker.generateId();
39
+ this.taskDescription = taskDescription ?? null;
40
+ this.buffer = [];
41
+ this.lastActionTime = Date.now();
42
+ return this.sessionId;
43
+ }
44
+ /** Get the current session ID, auto-rotating if stale */
45
+ getSessionId() {
46
+ const now = Date.now();
47
+ if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
48
+ // Session gap detected — auto-save previous sequence then start fresh
49
+ this.tryAutoSave();
50
+ this.sessionId = SessionTracker.generateId();
51
+ this.buffer = [];
52
+ this.taskDescription = null;
53
+ }
54
+ return this.sessionId;
55
+ }
56
+ /** Record an action into the current session buffer */
57
+ recordAction(entry) {
58
+ const now = Date.now();
59
+ if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
60
+ // Gap detected — auto-save then start new session
61
+ this.tryAutoSave();
62
+ this.sessionId = SessionTracker.generateId();
63
+ this.buffer = [];
64
+ this.taskDescription = null;
65
+ }
66
+ this.lastActionTime = now;
67
+ this.buffer.push(entry);
68
+ if (this.buffer.length > MAX_BUFFER_SIZE) {
69
+ this.buffer.shift();
70
+ }
71
+ }
72
+ /** End the session and save a strategy if successful */
73
+ endSession(success, taskDescription) {
74
+ const task = taskDescription ?? this.taskDescription;
75
+ if (!success || !task || this.buffer.length === 0) {
76
+ this.buffer = [];
77
+ return null;
78
+ }
79
+ const strategy = this.buildStrategy(task, this.buffer);
80
+ this.store.appendStrategy(strategy);
81
+ this.buffer = [];
82
+ return strategy;
83
+ }
84
+ /** Get the current session's action buffer */
85
+ getBuffer() {
86
+ return [...this.buffer];
87
+ }
88
+ /** Get recent tool names (for strategy hint matching) */
89
+ getRecentToolNames(limit = 10) {
90
+ return this.buffer.slice(-limit).map((a) => a.tool);
91
+ }
92
+ /** Get current task description */
93
+ getTaskDescription() {
94
+ return this.taskDescription;
95
+ }
96
+ // ── auto-save logic ────────────────────────────
97
+ /**
98
+ * Try to auto-save the current buffer as a strategy.
99
+ * Only saves if there are MIN_AUTO_SAVE_STEPS+ consecutive successes.
100
+ * Uses tool sequence as task description if no explicit one was given.
101
+ */
102
+ tryAutoSave() {
103
+ if (this.buffer.length < MIN_AUTO_SAVE_STEPS)
104
+ return;
105
+ // Find the longest trailing streak of successes
106
+ let successStreak = [];
107
+ for (let i = this.buffer.length - 1; i >= 0; i--) {
108
+ if (this.buffer[i].success) {
109
+ successStreak.unshift(this.buffer[i]);
110
+ }
111
+ else {
112
+ break;
113
+ }
114
+ }
115
+ if (successStreak.length < MIN_AUTO_SAVE_STEPS)
116
+ return;
117
+ // Build a task description from the tool sequence if none provided
118
+ const task = this.taskDescription ?? this.inferTaskDescription(successStreak);
119
+ const strategy = this.buildStrategy(task, successStreak);
120
+ this.store.appendStrategy(strategy);
121
+ }
122
+ /** Infer a task description from a sequence of actions */
123
+ inferTaskDescription(actions) {
124
+ const tools = [...new Set(actions.map((a) => a.tool))];
125
+ // Extract key param values (bundle IDs, titles, URLs, etc.)
126
+ const keyParams = [];
127
+ for (const a of actions) {
128
+ for (const [key, val] of Object.entries(a.params)) {
129
+ if (typeof val === "string" && val.length > 2 && val.length < 60) {
130
+ if (["bundleId", "title", "url", "text", "script", "selector", "menuPath"].includes(key)) {
131
+ keyParams.push(val);
132
+ }
133
+ }
134
+ }
135
+ }
136
+ const paramHint = keyParams.length > 0 ? ` (${keyParams.slice(0, 3).join(", ")})` : "";
137
+ return `${tools.join(" → ")}${paramHint}`;
138
+ }
139
+ buildStrategy(task, actions) {
140
+ const steps = actions.map((a) => ({
141
+ tool: a.tool,
142
+ params: a.params,
143
+ }));
144
+ const totalDurationMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
145
+ const tags = extractTags(task, steps);
146
+ const fingerprint = MemoryStore.makeFingerprint(steps.map((s) => s.tool));
147
+ return {
148
+ id: "str_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
149
+ task,
150
+ steps,
151
+ totalDurationMs,
152
+ successCount: 1,
153
+ failCount: 0,
154
+ lastUsed: new Date().toISOString(),
155
+ tags,
156
+ fingerprint,
157
+ };
158
+ }
159
+ }
160
+ /** Extract tags from task description and tool names */
161
+ function extractTags(task, steps) {
162
+ const tags = new Set();
163
+ const words = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
164
+ for (const w of words)
165
+ tags.add(w);
166
+ for (const s of steps)
167
+ tags.add(s.tool);
168
+ return [...tags];
169
+ }