screenhand 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +458 -93
- package/dist/.audit-log.jsonl +55 -0
- package/dist/.screenhand/memory/.lock +1 -0
- package/dist/.screenhand/memory/actions.jsonl +85 -0
- package/dist/.screenhand/memory/errors.jsonl +5 -0
- package/dist/.screenhand/memory/errors.jsonl.bak +4 -0
- package/dist/.screenhand/memory/state.json +35 -0
- package/dist/.screenhand/memory/state.json.bak +35 -0
- package/dist/.screenhand/memory/strategies.jsonl +12 -0
- package/dist/agent/cli.js +73 -0
- package/dist/agent/loop.js +258 -0
- package/dist/config.js +9 -0
- package/dist/index.js +56 -0
- package/dist/logging/timeline-logger.js +29 -0
- package/dist/mcp/mcp-stdio-server.js +448 -0
- package/dist/mcp/server.js +347 -0
- package/dist/mcp-desktop.js +2731 -0
- package/dist/mcp-entry.js +59 -0
- package/dist/memory/recall.js +160 -0
- package/dist/memory/research.js +98 -0
- package/dist/memory/seeds.js +89 -0
- package/dist/memory/session.js +161 -0
- package/dist/memory/store.js +391 -0
- package/dist/memory/types.js +4 -0
- package/dist/monitor/codex-monitor.js +377 -0
- package/dist/monitor/task-queue.js +84 -0
- package/dist/monitor/types.js +49 -0
- package/dist/native/bridge-client.js +174 -0
- package/dist/native/macos-bridge-client.js +5 -0
- package/dist/npm-publish-helper.js +117 -0
- package/dist/npm-token-cdp.js +113 -0
- package/dist/npm-token-create.js +135 -0
- package/dist/npm-token-finish.js +126 -0
- package/dist/playbook/engine.js +193 -0
- package/dist/playbook/index.js +4 -0
- package/dist/playbook/recorder.js +519 -0
- package/dist/playbook/runner.js +392 -0
- package/dist/playbook/store.js +166 -0
- package/dist/playbook/types.js +4 -0
- package/dist/runtime/accessibility-adapter.js +377 -0
- package/dist/runtime/app-adapter.js +48 -0
- package/dist/runtime/applescript-adapter.js +283 -0
- package/dist/runtime/ax-role-map.js +80 -0
- package/dist/runtime/browser-adapter.js +36 -0
- package/dist/runtime/cdp-chrome-adapter.js +505 -0
- package/dist/runtime/composite-adapter.js +205 -0
- package/dist/runtime/executor.js +250 -0
- package/dist/runtime/locator-cache.js +12 -0
- package/dist/runtime/planning-loop.js +47 -0
- package/dist/runtime/service.js +372 -0
- package/dist/runtime/session-manager.js +28 -0
- package/dist/runtime/state-observer.js +105 -0
- package/dist/runtime/vision-adapter.js +208 -0
- package/dist/scripts/codex-monitor-daemon.js +335 -0
- package/dist/scripts/supervisor-daemon.js +272 -0
- package/dist/scripts/worker-daemon.js +228 -0
- package/dist/src/agent/cli.js +82 -0
- package/dist/src/agent/loop.js +274 -0
- package/{src/config.ts → dist/src/config.js} +5 -10
- package/{src/index.ts → dist/src/index.js} +32 -52
- package/dist/src/jobs/manager.js +237 -0
- package/dist/src/jobs/runner.js +683 -0
- package/dist/src/jobs/store.js +102 -0
- package/dist/src/jobs/types.js +30 -0
- package/dist/src/jobs/worker.js +97 -0
- package/dist/src/logging/timeline-logger.js +45 -0
- package/dist/src/mcp/mcp-stdio-server.js +464 -0
- package/dist/src/mcp/server.js +363 -0
- package/dist/src/mcp-entry.js +60 -0
- package/dist/src/memory/recall.js +170 -0
- package/dist/src/memory/research.js +104 -0
- package/dist/src/memory/seeds.js +101 -0
- package/dist/src/memory/service.js +421 -0
- package/dist/src/memory/session.js +169 -0
- package/dist/src/memory/store.js +422 -0
- package/dist/src/memory/types.js +17 -0
- package/dist/src/monitor/codex-monitor.js +382 -0
- package/dist/src/monitor/task-queue.js +97 -0
- package/dist/src/monitor/types.js +62 -0
- package/dist/src/native/bridge-client.js +190 -0
- package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
- package/dist/src/playbook/engine.js +201 -0
- package/dist/src/playbook/index.js +20 -0
- package/dist/src/playbook/recorder.js +535 -0
- package/dist/src/playbook/runner.js +408 -0
- package/dist/src/playbook/store.js +183 -0
- package/dist/src/playbook/types.js +17 -0
- package/dist/src/runtime/accessibility-adapter.js +393 -0
- package/dist/src/runtime/app-adapter.js +64 -0
- package/dist/src/runtime/applescript-adapter.js +299 -0
- package/dist/src/runtime/ax-role-map.js +96 -0
- package/dist/src/runtime/browser-adapter.js +52 -0
- package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
- package/dist/src/runtime/composite-adapter.js +221 -0
- package/dist/src/runtime/execution-contract.js +159 -0
- package/dist/src/runtime/executor.js +266 -0
- package/{src/runtime/locator-cache.ts → dist/src/runtime/locator-cache.js} +10 -15
- package/dist/src/runtime/planning-loop.js +63 -0
- package/dist/src/runtime/service.js +388 -0
- package/dist/src/runtime/session-manager.js +60 -0
- package/dist/src/runtime/state-observer.js +121 -0
- package/dist/src/runtime/vision-adapter.js +224 -0
- package/dist/src/supervisor/locks.js +186 -0
- package/dist/src/supervisor/supervisor.js +403 -0
- package/dist/src/supervisor/types.js +30 -0
- package/dist/src/test-mcp-protocol.js +154 -0
- package/dist/src/types.js +17 -0
- package/dist/src/util/atomic-write.js +118 -0
- package/dist/test-mcp-protocol.js +138 -0
- package/dist/types.js +1 -0
- package/package.json +18 -4
- package/.claude/commands/automate.md +0 -28
- package/.claude/commands/debug-ui.md +0 -19
- package/.claude/commands/screenshot.md +0 -15
- package/.github/FUNDING.yml +0 -1
- package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
- package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
- package/.mcp.json +0 -8
- package/DESKTOP_MCP_GUIDE.md +0 -92
- package/SECURITY.md +0 -44
- package/docs/architecture.md +0 -47
- package/install-skills.sh +0 -19
- package/mcp-bridge.ts +0 -271
- package/mcp-desktop.ts +0 -1221
- package/native/macos-bridge/Package.swift +0 -21
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +0 -261
- package/native/macos-bridge/Sources/AppManagement.swift +0 -129
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +0 -242
- package/native/macos-bridge/Sources/ObserverBridge.swift +0 -120
- package/native/macos-bridge/Sources/VisionBridge.swift +0 -80
- package/native/macos-bridge/Sources/main.swift +0 -345
- package/native/windows-bridge/AppManagement.cs +0 -234
- package/native/windows-bridge/InputBridge.cs +0 -436
- package/native/windows-bridge/Program.cs +0 -265
- package/native/windows-bridge/ScreenCapture.cs +0 -329
- package/native/windows-bridge/UIAutomationBridge.cs +0 -571
- package/native/windows-bridge/WindowsBridge.csproj +0 -17
- package/playbooks/devpost.json +0 -186
- package/playbooks/instagram.json +0 -41
- package/playbooks/instagram_v2.json +0 -201
- package/playbooks/x_v1.json +0 -211
- package/scripts/devpost-live-loop.mjs +0 -421
- package/src/logging/timeline-logger.ts +0 -55
- package/src/mcp/server.ts +0 -449
- package/src/memory/recall.ts +0 -191
- package/src/memory/research.ts +0 -146
- package/src/memory/seeds.ts +0 -123
- package/src/memory/session.ts +0 -201
- package/src/memory/store.ts +0 -434
- package/src/memory/types.ts +0 -69
- package/src/native/bridge-client.ts +0 -239
- package/src/runtime/accessibility-adapter.ts +0 -487
- package/src/runtime/app-adapter.ts +0 -169
- package/src/runtime/applescript-adapter.ts +0 -376
- package/src/runtime/ax-role-map.ts +0 -102
- package/src/runtime/browser-adapter.ts +0 -129
- package/src/runtime/cdp-chrome-adapter.ts +0 -676
- package/src/runtime/composite-adapter.ts +0 -274
- package/src/runtime/executor.ts +0 -396
- package/src/runtime/planning-loop.ts +0 -81
- package/src/runtime/service.ts +0 -448
- package/src/runtime/session-manager.ts +0 -50
- package/src/runtime/state-observer.ts +0 -136
- package/src/runtime/vision-adapter.ts +0 -297
- package/src/types.ts +0 -297
- package/tests/bridge-client.test.ts +0 -176
- package/tests/browser-stealth.test.ts +0 -210
- package/tests/composite-adapter.test.ts +0 -64
- package/tests/mcp-server.test.ts +0 -151
- package/tests/memory-recall.test.ts +0 -339
- package/tests/memory-research.test.ts +0 -159
- package/tests/memory-seeds.test.ts +0 -120
- package/tests/memory-store.test.ts +0 -392
- package/tests/types.test.ts +0 -92
- package/tsconfig.check.json +0 -17
- package/tsconfig.json +0 -19
- package/vitest.config.ts +0 -8
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
let seedCounter = 0;
|
|
18
|
+
function makeFingerprint(tools) {
|
|
19
|
+
return tools.join("→");
|
|
20
|
+
}
|
|
21
|
+
function seed(task, steps, tags) {
|
|
22
|
+
seedCounter++;
|
|
23
|
+
return {
|
|
24
|
+
id: `seed_${String(seedCounter).padStart(3, "0")}`,
|
|
25
|
+
task,
|
|
26
|
+
steps,
|
|
27
|
+
totalDurationMs: 0,
|
|
28
|
+
successCount: 10,
|
|
29
|
+
failCount: 0,
|
|
30
|
+
lastUsed: new Date().toISOString(),
|
|
31
|
+
tags,
|
|
32
|
+
fingerprint: makeFingerprint(steps.map((s) => s.tool)),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export const SEED_STRATEGIES = [
|
|
36
|
+
// 1. Take a photo with Photo Booth
|
|
37
|
+
seed("Take a photo with Photo Booth", [
|
|
38
|
+
{ tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
|
|
39
|
+
{ tool: "ui_press", params: { title: "Take Photo" } },
|
|
40
|
+
], ["photo", "booth", "camera"]),
|
|
41
|
+
// 2. Open a URL in Chrome
|
|
42
|
+
seed("Open a URL in Chrome", [
|
|
43
|
+
{ tool: "launch", params: { bundleId: "com.google.Chrome" } },
|
|
44
|
+
{ tool: "browser_navigate", params: { url: "" } },
|
|
45
|
+
], ["chrome", "browse", "url"]),
|
|
46
|
+
// 3. Save current document
|
|
47
|
+
seed("Save current document", [
|
|
48
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
49
|
+
{ tool: "key", params: { combo: "cmd+s" } },
|
|
50
|
+
], ["save", "document"]),
|
|
51
|
+
// 4. Copy from one app and paste into another
|
|
52
|
+
seed("Copy from one app and paste into another", [
|
|
53
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
54
|
+
{ tool: "key", params: { combo: "cmd+c" } },
|
|
55
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
56
|
+
{ tool: "key", params: { combo: "cmd+v" } },
|
|
57
|
+
], ["copy", "paste"]),
|
|
58
|
+
// 5. Navigate to a folder in Finder
|
|
59
|
+
seed("Navigate to a folder in Finder", [
|
|
60
|
+
{ tool: "focus", params: { bundleId: "com.apple.finder" } },
|
|
61
|
+
{ tool: "key", params: { combo: "cmd+shift+g" } },
|
|
62
|
+
{ tool: "type_text", params: { text: "" } },
|
|
63
|
+
], ["finder", "folder", "navigate"]),
|
|
64
|
+
// 6. Create a new folder in Finder
|
|
65
|
+
seed("Create a new folder in Finder", [
|
|
66
|
+
{ tool: "focus", params: { bundleId: "com.apple.finder" } },
|
|
67
|
+
{ tool: "key", params: { combo: "cmd+shift+n" } },
|
|
68
|
+
{ tool: "type_text", params: { text: "" } },
|
|
69
|
+
], ["finder", "folder", "create"]),
|
|
70
|
+
// 7. Close the current window
|
|
71
|
+
seed("Close the current window", [
|
|
72
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
73
|
+
{ tool: "key", params: { combo: "cmd+w" } },
|
|
74
|
+
], ["close", "window"]),
|
|
75
|
+
// 8. Select all and copy
|
|
76
|
+
seed("Select all content and copy", [
|
|
77
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
78
|
+
{ tool: "key", params: { combo: "cmd+a" } },
|
|
79
|
+
{ tool: "key", params: { combo: "cmd+c" } },
|
|
80
|
+
], ["select", "all", "copy"]),
|
|
81
|
+
// 9. List running apps
|
|
82
|
+
seed("List all running applications", [
|
|
83
|
+
{ tool: "apps", params: {} },
|
|
84
|
+
], ["apps", "list", "running"]),
|
|
85
|
+
// 10. Inspect app UI tree
|
|
86
|
+
seed("Inspect an app's UI element tree", [
|
|
87
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
88
|
+
{ tool: "ui_tree", params: { pid: 0 } },
|
|
89
|
+
], ["inspect", "tree", "accessibility"]),
|
|
90
|
+
// 11. Open a new tab in Chrome and navigate
|
|
91
|
+
seed("Open a new Chrome tab and navigate to URL", [
|
|
92
|
+
{ tool: "focus", params: { bundleId: "com.google.Chrome" } },
|
|
93
|
+
{ tool: "key", params: { combo: "cmd+t" } },
|
|
94
|
+
{ tool: "browser_navigate", params: { url: "" } },
|
|
95
|
+
], ["chrome", "tab", "new"]),
|
|
96
|
+
// 12. Export as PDF via menu
|
|
97
|
+
seed("Export document as PDF", [
|
|
98
|
+
{ tool: "focus", params: { bundleId: "" } },
|
|
99
|
+
{ tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
|
|
100
|
+
], ["export", "pdf"]),
|
|
101
|
+
];
|
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
/**
|
|
18
|
+
* MemoryService — unified facade over MemoryStore, SessionTracker, and RecallEngine
|
|
19
|
+
*
|
|
20
|
+
* Single entry-point for all memory operations. Adds:
|
|
21
|
+
* - state.json snapshot (debounced, written on every action)
|
|
22
|
+
* - learnings.jsonl (verified patterns, separate from strategies)
|
|
23
|
+
* - MemoryPolicy (error/stall thresholds)
|
|
24
|
+
* - Mission tracking
|
|
25
|
+
*/
|
|
26
|
+
import fs from "node:fs";
|
|
27
|
+
import path from "node:path";
|
|
28
|
+
import { writeFileAtomicSync } from "../util/atomic-write.js";
|
|
29
|
+
import { MemoryStore } from "./store.js";
|
|
30
|
+
import { SessionTracker } from "./session.js";
|
|
31
|
+
import { RecallEngine } from "./recall.js";
|
|
32
|
+
// ── Defaults ─────────────────────────────────────
|
|
33
|
+
const DEFAULT_POLICY = {
|
|
34
|
+
maxConsecutiveErrors: 5,
|
|
35
|
+
stallThresholdMs: 300_000,
|
|
36
|
+
escalateAfterRetries: 3,
|
|
37
|
+
pauseBetweenActionsMs: 500,
|
|
38
|
+
};
|
|
39
|
+
const SNAPSHOT_DEBOUNCE_MS = 200;
|
|
40
|
+
const MAX_LEARNINGS = 1000;
|
|
41
|
+
// ── Service ──────────────────────────────────────
|
|
42
|
+
export class MemoryService {
|
|
43
|
+
store;
|
|
44
|
+
session;
|
|
45
|
+
recall;
|
|
46
|
+
baseDir;
|
|
47
|
+
memDir;
|
|
48
|
+
learningsCache = [];
|
|
49
|
+
snapshot;
|
|
50
|
+
snapshotTimer = null;
|
|
51
|
+
consecutiveErrors = 0;
|
|
52
|
+
lastError = null;
|
|
53
|
+
actionsFailed = 0;
|
|
54
|
+
actionsTotal = 0;
|
|
55
|
+
sessionStartedAt;
|
|
56
|
+
initialized = false;
|
|
57
|
+
constructor(baseDir) {
|
|
58
|
+
this.baseDir = baseDir;
|
|
59
|
+
this.memDir = path.join(baseDir, ".screenhand", "memory");
|
|
60
|
+
this.store = new MemoryStore(baseDir);
|
|
61
|
+
this.session = new SessionTracker(this.store);
|
|
62
|
+
this.recall = new RecallEngine(this.store);
|
|
63
|
+
this.sessionStartedAt = new Date().toISOString();
|
|
64
|
+
this.snapshot = {
|
|
65
|
+
session: {
|
|
66
|
+
id: this.session.getSessionId(),
|
|
67
|
+
client: "unknown",
|
|
68
|
+
startedAt: this.sessionStartedAt,
|
|
69
|
+
lastActionAt: this.sessionStartedAt,
|
|
70
|
+
},
|
|
71
|
+
mission: { current: null, phase: "idle" },
|
|
72
|
+
health: {
|
|
73
|
+
actionsTotal: 0,
|
|
74
|
+
actionsFailed: 0,
|
|
75
|
+
successRate: 1,
|
|
76
|
+
lastError: null,
|
|
77
|
+
consecutiveErrors: 0,
|
|
78
|
+
},
|
|
79
|
+
patterns: { topWorking: [], topFailing: [], knownBlockers: [] },
|
|
80
|
+
policy: { ...DEFAULT_POLICY },
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
// ── Initialization ───────────────────────────────
|
|
84
|
+
/** Load all caches from disk. Call once at startup. */
|
|
85
|
+
init() {
|
|
86
|
+
if (this.initialized)
|
|
87
|
+
return;
|
|
88
|
+
this.initialized = true;
|
|
89
|
+
this.store.init();
|
|
90
|
+
this.ensureMemDir();
|
|
91
|
+
// Load learnings
|
|
92
|
+
this.learningsCache = this.readJsonlSafe("learnings.jsonl");
|
|
93
|
+
// Load existing snapshot if present (to restore mission/policy across restarts)
|
|
94
|
+
const snapPath = this.filePath("state.json");
|
|
95
|
+
if (fs.existsSync(snapPath)) {
|
|
96
|
+
try {
|
|
97
|
+
const raw = fs.readFileSync(snapPath, "utf-8");
|
|
98
|
+
const loaded = JSON.parse(raw);
|
|
99
|
+
// Restore mission and policy from previous run
|
|
100
|
+
if (loaded.mission)
|
|
101
|
+
this.snapshot.mission = loaded.mission;
|
|
102
|
+
if (loaded.policy)
|
|
103
|
+
this.snapshot.policy = { ...DEFAULT_POLICY, ...loaded.policy };
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// Corrupted snapshot — start fresh
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// Sync stats from store
|
|
110
|
+
const stats = this.store.getStats();
|
|
111
|
+
this.actionsTotal = stats.totalActions;
|
|
112
|
+
this.actionsFailed = stats.totalActions - Math.round(stats.successRate * stats.totalActions);
|
|
113
|
+
this.rebuildPatterns();
|
|
114
|
+
this.updateHealthInSnapshot();
|
|
115
|
+
this.writeSnapshotSync();
|
|
116
|
+
}
|
|
117
|
+
// ── Snapshot ─────────────────────────────────────
|
|
118
|
+
/** Get the current in-memory snapshot (zero-cost). */
|
|
119
|
+
getSnapshot() {
|
|
120
|
+
return this.snapshot;
|
|
121
|
+
}
|
|
122
|
+
scheduleSnapshotWrite() {
|
|
123
|
+
if (this.snapshotTimer)
|
|
124
|
+
return;
|
|
125
|
+
this.snapshotTimer = setTimeout(() => {
|
|
126
|
+
this.snapshotTimer = null;
|
|
127
|
+
this.writeSnapshotSync();
|
|
128
|
+
}, SNAPSHOT_DEBOUNCE_MS);
|
|
129
|
+
}
|
|
130
|
+
writeSnapshotSync() {
|
|
131
|
+
this.ensureMemDir();
|
|
132
|
+
try {
|
|
133
|
+
writeFileAtomicSync(this.filePath("state.json"), JSON.stringify(this.snapshot, null, 2));
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
// Non-critical
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
updateHealthInSnapshot() {
|
|
140
|
+
this.snapshot.health = {
|
|
141
|
+
actionsTotal: this.actionsTotal,
|
|
142
|
+
actionsFailed: this.actionsFailed,
|
|
143
|
+
successRate: this.actionsTotal > 0 ? (this.actionsTotal - this.actionsFailed) / this.actionsTotal : 1,
|
|
144
|
+
lastError: this.lastError,
|
|
145
|
+
consecutiveErrors: this.consecutiveErrors,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
rebuildPatterns() {
|
|
149
|
+
// Top working: learnings with high confidence, sorted desc
|
|
150
|
+
const working = this.learningsCache
|
|
151
|
+
.filter((l) => l.confidence >= 0.6 && l.successCount > l.failCount)
|
|
152
|
+
.sort((a, b) => b.confidence - a.confidence)
|
|
153
|
+
.slice(0, 10)
|
|
154
|
+
.map((l) => `${l.scope}: ${l.pattern} (${l.method})`);
|
|
155
|
+
// Top failing: learnings with low confidence or high fail count
|
|
156
|
+
const failing = this.learningsCache
|
|
157
|
+
.filter((l) => l.failCount > l.successCount)
|
|
158
|
+
.sort((a, b) => b.failCount - a.failCount)
|
|
159
|
+
.slice(0, 10)
|
|
160
|
+
.map((l) => `${l.scope}: ${l.pattern} (${l.method})`);
|
|
161
|
+
// Known blockers: errors with no resolution
|
|
162
|
+
const errors = this.store.readErrors();
|
|
163
|
+
const blockers = errors
|
|
164
|
+
.filter((e) => !e.resolution && e.occurrences >= 2)
|
|
165
|
+
.sort((a, b) => b.occurrences - a.occurrences)
|
|
166
|
+
.slice(0, 10)
|
|
167
|
+
.map((e) => `${e.tool}: ${e.error}`);
|
|
168
|
+
this.snapshot.patterns = {
|
|
169
|
+
topWorking: working,
|
|
170
|
+
topFailing: failing,
|
|
171
|
+
knownBlockers: blockers,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
// ── Recording ────────────────────────────────────
|
|
175
|
+
/** Record an action event. Delegates to store + session tracker, updates snapshot. */
|
|
176
|
+
recordEvent(entry) {
|
|
177
|
+
this.store.appendAction(entry);
|
|
178
|
+
this.session.recordAction(entry);
|
|
179
|
+
this.actionsTotal++;
|
|
180
|
+
if (!entry.success) {
|
|
181
|
+
this.actionsFailed++;
|
|
182
|
+
this.consecutiveErrors++;
|
|
183
|
+
this.lastError = entry.error;
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
this.consecutiveErrors = 0;
|
|
187
|
+
}
|
|
188
|
+
this.snapshot.session.id = this.session.getSessionId();
|
|
189
|
+
this.snapshot.session.lastActionAt = entry.timestamp;
|
|
190
|
+
this.updateHealthInSnapshot();
|
|
191
|
+
this.scheduleSnapshotWrite();
|
|
192
|
+
}
|
|
193
|
+
/** Record an error pattern. Delegates to store, optionally creates a learning. */
|
|
194
|
+
recordError(tool, error, fix, scope) {
|
|
195
|
+
const pattern = {
|
|
196
|
+
id: "err_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
|
|
197
|
+
tool,
|
|
198
|
+
params: {},
|
|
199
|
+
error,
|
|
200
|
+
resolution: fix,
|
|
201
|
+
occurrences: 1,
|
|
202
|
+
lastSeen: new Date().toISOString(),
|
|
203
|
+
};
|
|
204
|
+
this.store.appendError(pattern);
|
|
205
|
+
// If a fix is provided, record it as a learning
|
|
206
|
+
if (fix && scope) {
|
|
207
|
+
this.recordLearning({
|
|
208
|
+
scope,
|
|
209
|
+
pattern: error,
|
|
210
|
+
method: "ax",
|
|
211
|
+
confidence: 0.5,
|
|
212
|
+
successCount: 0,
|
|
213
|
+
failCount: 1,
|
|
214
|
+
lastSeen: new Date().toISOString(),
|
|
215
|
+
fix,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
this.rebuildPatterns();
|
|
219
|
+
this.scheduleSnapshotWrite();
|
|
220
|
+
}
|
|
221
|
+
// ── Learnings ────────────────────────────────────
|
|
222
|
+
/** Append a verified learning to learnings.jsonl. */
|
|
223
|
+
recordLearning(learning) {
|
|
224
|
+
const full = {
|
|
225
|
+
id: "lrn_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
|
|
226
|
+
...learning,
|
|
227
|
+
};
|
|
228
|
+
// Check for existing learning with same scope + pattern + method
|
|
229
|
+
const idx = this.learningsCache.findIndex((l) => l.scope === full.scope && l.pattern === full.pattern && l.method === full.method);
|
|
230
|
+
if (idx >= 0) {
|
|
231
|
+
const existing = this.learningsCache[idx];
|
|
232
|
+
this.learningsCache[idx] = {
|
|
233
|
+
...existing,
|
|
234
|
+
successCount: existing.successCount + full.successCount,
|
|
235
|
+
failCount: existing.failCount + full.failCount,
|
|
236
|
+
confidence: this.computeConfidence(existing.successCount + full.successCount, existing.failCount + full.failCount),
|
|
237
|
+
lastSeen: full.lastSeen,
|
|
238
|
+
fix: full.fix ?? existing.fix,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
this.learningsCache.push(full);
|
|
243
|
+
this.enforceLearningsLimit();
|
|
244
|
+
}
|
|
245
|
+
this.writeLearningsAsync();
|
|
246
|
+
this.rebuildPatterns();
|
|
247
|
+
this.scheduleSnapshotWrite();
|
|
248
|
+
}
|
|
249
|
+
/** Search learnings by scope and/or method. */
|
|
250
|
+
queryPatterns(scope, method) {
|
|
251
|
+
let results = this.learningsCache;
|
|
252
|
+
if (scope) {
|
|
253
|
+
results = results.filter((l) => l.scope === scope || l.scope.startsWith(scope + "/"));
|
|
254
|
+
}
|
|
255
|
+
if (method) {
|
|
256
|
+
results = results.filter((l) => l.method === method);
|
|
257
|
+
}
|
|
258
|
+
return results;
|
|
259
|
+
}
|
|
260
|
+
computeConfidence(success, fail) {
|
|
261
|
+
const total = success + fail;
|
|
262
|
+
if (total === 0)
|
|
263
|
+
return 0;
|
|
264
|
+
return success / total;
|
|
265
|
+
}
|
|
266
|
+
enforceLearningsLimit() {
|
|
267
|
+
if (this.learningsCache.length <= MAX_LEARNINGS)
|
|
268
|
+
return;
|
|
269
|
+
// Evict lowest-confidence, oldest learnings
|
|
270
|
+
this.learningsCache.sort((a, b) => {
|
|
271
|
+
const confDiff = a.confidence - b.confidence;
|
|
272
|
+
if (Math.abs(confDiff) > 0.1)
|
|
273
|
+
return confDiff;
|
|
274
|
+
return new Date(a.lastSeen).getTime() - new Date(b.lastSeen).getTime();
|
|
275
|
+
});
|
|
276
|
+
this.learningsCache = this.learningsCache.slice(-MAX_LEARNINGS);
|
|
277
|
+
}
|
|
278
|
+
writeLearningsAsync() {
|
|
279
|
+
this.ensureMemDir();
|
|
280
|
+
const data = this.learningsCache.map((l) => JSON.stringify(l)).join("\n") + (this.learningsCache.length ? "\n" : "");
|
|
281
|
+
fs.writeFile(this.filePath("learnings.jsonl"), data, () => { });
|
|
282
|
+
}
|
|
283
|
+
// ── Recall (delegates to RecallEngine) ───────────
|
|
284
|
+
/** Search error patterns, optionally filtered by tool. */
|
|
285
|
+
queryErrors(tool) {
|
|
286
|
+
return this.recall.recallErrors(tool);
|
|
287
|
+
}
|
|
288
|
+
/** Fuzzy-match strategies by query string. */
|
|
289
|
+
recallStrategies(query, limit) {
|
|
290
|
+
return this.recall.recallStrategies(query, limit);
|
|
291
|
+
}
|
|
292
|
+
/** Quick error check for interceptor (~0ms). */
|
|
293
|
+
quickErrorCheck(tool) {
|
|
294
|
+
return this.recall.quickErrorCheck(tool);
|
|
295
|
+
}
|
|
296
|
+
/** Quick strategy hint for interceptor (~0ms). */
|
|
297
|
+
quickStrategyHint(recentTools) {
|
|
298
|
+
return this.recall.quickStrategyHint(recentTools);
|
|
299
|
+
}
|
|
300
|
+
/** Record strategy outcome for feedback loop. */
|
|
301
|
+
recordStrategyOutcome(fingerprint, success) {
|
|
302
|
+
this.store.recordStrategyOutcome(fingerprint, success);
|
|
303
|
+
}
|
|
304
|
+
// ── Session / Strategy ───────────────────────────
|
|
305
|
+
/** Get the current session ID. */
|
|
306
|
+
getSessionId() {
|
|
307
|
+
return this.session.getSessionId();
|
|
308
|
+
}
|
|
309
|
+
/** Get recent tool names from session buffer. */
|
|
310
|
+
getRecentToolNames(limit) {
|
|
311
|
+
return this.session.getRecentToolNames(limit);
|
|
312
|
+
}
|
|
313
|
+
/** End current session and save a strategy if successful. */
|
|
314
|
+
saveStrategy(task, tags) {
|
|
315
|
+
const strategy = this.session.endSession(true, task);
|
|
316
|
+
if (strategy && tags && tags.length > 0) {
|
|
317
|
+
// Merge additional tags
|
|
318
|
+
const merged = new Set([...strategy.tags, ...tags]);
|
|
319
|
+
strategy.tags = [...merged];
|
|
320
|
+
}
|
|
321
|
+
return strategy;
|
|
322
|
+
}
|
|
323
|
+
/** Read raw actions from store (for exports/playbooks). */
|
|
324
|
+
readActions() {
|
|
325
|
+
return this.store.readActions();
|
|
326
|
+
}
|
|
327
|
+
/** Read raw errors from store. */
|
|
328
|
+
readErrors() {
|
|
329
|
+
return this.store.readErrors();
|
|
330
|
+
}
|
|
331
|
+
/** Read raw strategies from store. */
|
|
332
|
+
readStrategies() {
|
|
333
|
+
return this.store.readStrategies();
|
|
334
|
+
}
|
|
335
|
+
/** Append an error pattern directly (for interceptor compatibility). */
|
|
336
|
+
appendError(pattern) {
|
|
337
|
+
this.store.appendError(pattern);
|
|
338
|
+
}
|
|
339
|
+
/** Append a strategy directly. */
|
|
340
|
+
appendStrategy(strategy) {
|
|
341
|
+
this.store.appendStrategy(strategy);
|
|
342
|
+
}
|
|
343
|
+
// ── Stats ────────────────────────────────────────
|
|
344
|
+
/** Get aggregate memory stats. */
|
|
345
|
+
getStats() {
|
|
346
|
+
return this.store.getStats();
|
|
347
|
+
}
|
|
348
|
+
// ── Mission ──────────────────────────────────────
|
|
349
|
+
/** Set the current mission and optionally a phase. */
|
|
350
|
+
setMission(mission, phase) {
|
|
351
|
+
this.snapshot.mission.current = mission;
|
|
352
|
+
if (phase)
|
|
353
|
+
this.snapshot.mission.phase = phase;
|
|
354
|
+
this.scheduleSnapshotWrite();
|
|
355
|
+
}
|
|
356
|
+
/** Set the client identifier (e.g., "claude-code", "mcp-desktop"). */
|
|
357
|
+
setClient(client) {
|
|
358
|
+
this.snapshot.session.client = client;
|
|
359
|
+
this.scheduleSnapshotWrite();
|
|
360
|
+
}
|
|
361
|
+
// ── Clear ────────────────────────────────────────
|
|
362
|
+
/** Clear specific memory categories or everything. */
|
|
363
|
+
clear(what) {
|
|
364
|
+
if (what === "learnings" || what === "all") {
|
|
365
|
+
this.learningsCache = [];
|
|
366
|
+
const fp = this.filePath("learnings.jsonl");
|
|
367
|
+
if (fs.existsSync(fp))
|
|
368
|
+
fs.writeFileSync(fp, "");
|
|
369
|
+
}
|
|
370
|
+
if (what !== "learnings") {
|
|
371
|
+
// Delegate non-learnings clears to the store
|
|
372
|
+
const storeWhat = what === "all" ? "all" : what;
|
|
373
|
+
this.store.clear(storeWhat);
|
|
374
|
+
}
|
|
375
|
+
if (what === "all" || what === "actions") {
|
|
376
|
+
this.actionsTotal = 0;
|
|
377
|
+
this.actionsFailed = 0;
|
|
378
|
+
this.consecutiveErrors = 0;
|
|
379
|
+
this.lastError = null;
|
|
380
|
+
}
|
|
381
|
+
this.updateHealthInSnapshot();
|
|
382
|
+
this.rebuildPatterns();
|
|
383
|
+
this.writeSnapshotSync();
|
|
384
|
+
}
|
|
385
|
+
// ── Helpers ──────────────────────────────────────
|
|
386
|
+
ensureMemDir() {
|
|
387
|
+
if (!fs.existsSync(this.memDir)) {
|
|
388
|
+
fs.mkdirSync(this.memDir, { recursive: true });
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
filePath(name) {
|
|
392
|
+
return path.join(this.memDir, name);
|
|
393
|
+
}
|
|
394
|
+
readJsonlSafe(file) {
|
|
395
|
+
const fp = this.filePath(file);
|
|
396
|
+
if (!fs.existsSync(fp))
|
|
397
|
+
return [];
|
|
398
|
+
let text;
|
|
399
|
+
try {
|
|
400
|
+
text = fs.readFileSync(fp, "utf-8").trim();
|
|
401
|
+
}
|
|
402
|
+
catch {
|
|
403
|
+
return [];
|
|
404
|
+
}
|
|
405
|
+
if (!text)
|
|
406
|
+
return [];
|
|
407
|
+
const results = [];
|
|
408
|
+
for (const line of text.split("\n")) {
|
|
409
|
+
const trimmed = line.trim();
|
|
410
|
+
if (!trimmed)
|
|
411
|
+
continue;
|
|
412
|
+
try {
|
|
413
|
+
results.push(JSON.parse(trimmed));
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
// Skip corrupted line
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return results;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// Copyright (C) 2025 Clazro Technology Private Limited
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
3
|
+
//
|
|
4
|
+
// This file is part of ScreenHand.
|
|
5
|
+
//
|
|
6
|
+
// ScreenHand is free software: you can redistribute it and/or modify
|
|
7
|
+
// it under the terms of the GNU Affero General Public License as
|
|
8
|
+
// published by the Free Software Foundation, version 3.
|
|
9
|
+
//
|
|
10
|
+
// ScreenHand is distributed in the hope that it will be useful,
|
|
11
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
// GNU Affero General Public License for more details.
|
|
14
|
+
//
|
|
15
|
+
// You should have received a copy of the GNU Affero General Public License
|
|
16
|
+
// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
+
import { MemoryStore } from "./store.js";
|
|
18
|
+
const SESSION_GAP_MS = 60_000; // 60s gap = new session
|
|
19
|
+
const MAX_BUFFER_SIZE = 100;
|
|
20
|
+
const MIN_AUTO_SAVE_STEPS = 3; // Need at least 3 successful steps to auto-save
|
|
21
|
+
export class SessionTracker {
|
|
22
|
+
store;
|
|
23
|
+
sessionId;
|
|
24
|
+
taskDescription = null;
|
|
25
|
+
buffer = [];
|
|
26
|
+
lastActionTime = 0;
|
|
27
|
+
constructor(store) {
|
|
28
|
+
this.store = store;
|
|
29
|
+
this.sessionId = SessionTracker.generateId();
|
|
30
|
+
}
|
|
31
|
+
static generateId() {
|
|
32
|
+
return "s_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
|
|
33
|
+
}
|
|
34
|
+
/** Start (or restart) a named task session */
|
|
35
|
+
startSession(taskDescription) {
|
|
36
|
+
// Auto-save previous session if it had successful actions
|
|
37
|
+
this.tryAutoSave();
|
|
38
|
+
this.sessionId = SessionTracker.generateId();
|
|
39
|
+
this.taskDescription = taskDescription ?? null;
|
|
40
|
+
this.buffer = [];
|
|
41
|
+
this.lastActionTime = Date.now();
|
|
42
|
+
return this.sessionId;
|
|
43
|
+
}
|
|
44
|
+
/** Get the current session ID, auto-rotating if stale */
|
|
45
|
+
getSessionId() {
|
|
46
|
+
const now = Date.now();
|
|
47
|
+
if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
|
|
48
|
+
// Session gap detected — auto-save previous sequence then start fresh
|
|
49
|
+
this.tryAutoSave();
|
|
50
|
+
this.sessionId = SessionTracker.generateId();
|
|
51
|
+
this.buffer = [];
|
|
52
|
+
this.taskDescription = null;
|
|
53
|
+
}
|
|
54
|
+
return this.sessionId;
|
|
55
|
+
}
|
|
56
|
+
/** Record an action into the current session buffer */
|
|
57
|
+
recordAction(entry) {
|
|
58
|
+
const now = Date.now();
|
|
59
|
+
if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
|
|
60
|
+
// Gap detected — auto-save then start new session
|
|
61
|
+
this.tryAutoSave();
|
|
62
|
+
this.sessionId = SessionTracker.generateId();
|
|
63
|
+
this.buffer = [];
|
|
64
|
+
this.taskDescription = null;
|
|
65
|
+
}
|
|
66
|
+
this.lastActionTime = now;
|
|
67
|
+
this.buffer.push(entry);
|
|
68
|
+
if (this.buffer.length > MAX_BUFFER_SIZE) {
|
|
69
|
+
this.buffer.shift();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** End the session and save a strategy if successful */
|
|
73
|
+
endSession(success, taskDescription) {
|
|
74
|
+
const task = taskDescription ?? this.taskDescription;
|
|
75
|
+
if (!success || !task || this.buffer.length === 0) {
|
|
76
|
+
this.buffer = [];
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
const strategy = this.buildStrategy(task, this.buffer);
|
|
80
|
+
this.store.appendStrategy(strategy);
|
|
81
|
+
this.buffer = [];
|
|
82
|
+
return strategy;
|
|
83
|
+
}
|
|
84
|
+
/** Get the current session's action buffer */
|
|
85
|
+
getBuffer() {
|
|
86
|
+
return [...this.buffer];
|
|
87
|
+
}
|
|
88
|
+
/** Get recent tool names (for strategy hint matching) */
|
|
89
|
+
getRecentToolNames(limit = 10) {
|
|
90
|
+
return this.buffer.slice(-limit).map((a) => a.tool);
|
|
91
|
+
}
|
|
92
|
+
/** Get current task description */
|
|
93
|
+
getTaskDescription() {
|
|
94
|
+
return this.taskDescription;
|
|
95
|
+
}
|
|
96
|
+
// ── auto-save logic ────────────────────────────
|
|
97
|
+
/**
|
|
98
|
+
* Try to auto-save the current buffer as a strategy.
|
|
99
|
+
* Only saves if there are MIN_AUTO_SAVE_STEPS+ consecutive successes.
|
|
100
|
+
* Uses tool sequence as task description if no explicit one was given.
|
|
101
|
+
*/
|
|
102
|
+
tryAutoSave() {
|
|
103
|
+
if (this.buffer.length < MIN_AUTO_SAVE_STEPS)
|
|
104
|
+
return;
|
|
105
|
+
// Find the longest trailing streak of successes
|
|
106
|
+
let successStreak = [];
|
|
107
|
+
for (let i = this.buffer.length - 1; i >= 0; i--) {
|
|
108
|
+
if (this.buffer[i].success) {
|
|
109
|
+
successStreak.unshift(this.buffer[i]);
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (successStreak.length < MIN_AUTO_SAVE_STEPS)
|
|
116
|
+
return;
|
|
117
|
+
// Build a task description from the tool sequence if none provided
|
|
118
|
+
const task = this.taskDescription ?? this.inferTaskDescription(successStreak);
|
|
119
|
+
const strategy = this.buildStrategy(task, successStreak);
|
|
120
|
+
this.store.appendStrategy(strategy);
|
|
121
|
+
}
|
|
122
|
+
/** Infer a task description from a sequence of actions */
|
|
123
|
+
inferTaskDescription(actions) {
|
|
124
|
+
const tools = [...new Set(actions.map((a) => a.tool))];
|
|
125
|
+
// Extract key param values (bundle IDs, titles, URLs, etc.)
|
|
126
|
+
const keyParams = [];
|
|
127
|
+
for (const a of actions) {
|
|
128
|
+
for (const [key, val] of Object.entries(a.params)) {
|
|
129
|
+
if (typeof val === "string" && val.length > 2 && val.length < 60) {
|
|
130
|
+
if (["bundleId", "title", "url", "text", "script", "selector", "menuPath"].includes(key)) {
|
|
131
|
+
keyParams.push(val);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const paramHint = keyParams.length > 0 ? ` (${keyParams.slice(0, 3).join(", ")})` : "";
|
|
137
|
+
return `${tools.join(" → ")}${paramHint}`;
|
|
138
|
+
}
|
|
139
|
+
buildStrategy(task, actions) {
|
|
140
|
+
const steps = actions.map((a) => ({
|
|
141
|
+
tool: a.tool,
|
|
142
|
+
params: a.params,
|
|
143
|
+
}));
|
|
144
|
+
const totalDurationMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
|
|
145
|
+
const tags = extractTags(task, steps);
|
|
146
|
+
const fingerprint = MemoryStore.makeFingerprint(steps.map((s) => s.tool));
|
|
147
|
+
return {
|
|
148
|
+
id: "str_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
|
|
149
|
+
task,
|
|
150
|
+
steps,
|
|
151
|
+
totalDurationMs,
|
|
152
|
+
successCount: 1,
|
|
153
|
+
failCount: 0,
|
|
154
|
+
lastUsed: new Date().toISOString(),
|
|
155
|
+
tags,
|
|
156
|
+
fingerprint,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/** Extract tags from task description and tool names */
|
|
161
|
+
function extractTags(task, steps) {
|
|
162
|
+
const tags = new Set();
|
|
163
|
+
const words = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
|
|
164
|
+
for (const w of words)
|
|
165
|
+
tags.add(w);
|
|
166
|
+
for (const s of steps)
|
|
167
|
+
tags.add(s.tool);
|
|
168
|
+
return [...tags];
|
|
169
|
+
}
|