screenhand 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/mcp-desktop.js +502 -98
- package/dist/src/community/fetcher.js +32 -2
- package/dist/src/community/validator.js +15 -1
- package/dist/src/context-tracker.js +115 -43
- package/dist/src/ingestion/reference-merger.js +3 -1
- package/dist/src/learning/engine.js +225 -7
- package/dist/src/learning/locator-policy.js +16 -0
- package/dist/src/learning/pattern-policy.js +9 -0
- package/dist/src/learning/recovery-policy.js +16 -0
- package/dist/src/learning/sensor-policy.js +9 -0
- package/dist/src/learning/timing-model.js +62 -0
- package/dist/src/memory/research.js +7 -1
- package/dist/src/memory/store.js +18 -7
- package/dist/src/perception/coordinator.js +304 -4
- package/dist/src/perception/manager.js +13 -0
- package/dist/src/perception/vision-source.js +14 -4
- package/dist/src/planner/executor.js +125 -2
- package/dist/src/planner/planner.js +509 -10
- package/dist/src/playbook/engine.js +10 -0
- package/dist/src/recovery/engine.js +50 -3
- package/dist/src/runtime/execution-contract.js +67 -5
- package/dist/src/runtime/executor.js +41 -1
- package/dist/src/runtime/service.js +7 -0
- package/dist/src/state/app-map.js +307 -17
- package/dist/src/util/atomic-write.js +25 -4
- package/dist-references/reddit.json +2 -2
- package/package.json +1 -1
|
@@ -4,6 +4,17 @@ import * as fs from "node:fs";
|
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
import * as os from "node:os";
|
|
6
6
|
import { RemoteCommunityAPI } from "./remote-api.js";
|
|
7
|
+
/** Tools that are never allowed in community playbooks (defense in depth). */
|
|
8
|
+
const BLOCKED_TOOLS = new Set([
|
|
9
|
+
"applescript", "browser_js", "browser_stealth", // code execution / anti-detection
|
|
10
|
+
"key", "launch", "focus", // RCE via terminal launch + keystroke typing
|
|
11
|
+
"memory_save", "memory_clear", "memory_snapshot", // data exfil / tampering
|
|
12
|
+
"supervisor_start", "supervisor_stop", "supervisor_install", "supervisor_uninstall",
|
|
13
|
+
"job_create", "worker_start", // persistence
|
|
14
|
+
]);
|
|
15
|
+
function hasBlockedTool(playbook) {
|
|
16
|
+
return playbook.steps.some((s) => BLOCKED_TOOLS.has(s.tool));
|
|
17
|
+
}
|
|
7
18
|
/**
|
|
8
19
|
* PlaybookFetcher — fetches community playbooks from local disk
|
|
9
20
|
* and optionally from a remote API (when SCREENHAND_COMMUNITY_URL is set).
|
|
@@ -36,9 +47,17 @@ export class PlaybookFetcher {
|
|
|
36
47
|
}
|
|
37
48
|
try {
|
|
38
49
|
const remote = await this.remote.fetch(query);
|
|
50
|
+
// Filter remote results for blocked tools (defense in depth — local loadAll already filters)
|
|
51
|
+
const safeRemote = remote.filter((pb) => {
|
|
52
|
+
if (hasBlockedTool(pb)) {
|
|
53
|
+
console.error(`[community] Blocked: remote playbook ${pb.id} uses restricted tool`);
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
return true;
|
|
57
|
+
});
|
|
39
58
|
// Deduplicate: local wins on ID collision
|
|
40
59
|
const localIds = new Set(local.map((pb) => pb.id));
|
|
41
|
-
const merged = [...local, ...
|
|
60
|
+
const merged = [...local, ...safeRemote.filter((pb) => !localIds.has(pb.id))];
|
|
42
61
|
return this.filterAndRank(merged, query);
|
|
43
62
|
}
|
|
44
63
|
catch {
|
|
@@ -97,7 +116,18 @@ export class PlaybookFetcher {
|
|
|
97
116
|
continue;
|
|
98
117
|
try {
|
|
99
118
|
const raw = fs.readFileSync(path.join(this.repoDir, file), "utf-8");
|
|
100
|
-
|
|
119
|
+
const playbook = JSON.parse(raw);
|
|
120
|
+
// Validate required fields
|
|
121
|
+
if (!playbook.id || !playbook.name || !playbook.platform || !Array.isArray(playbook.steps)) {
|
|
122
|
+
console.error(`[community] Skipping invalid playbook: ${file}`);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
// Block playbooks using restricted tools (defense in depth — validator also checks)
|
|
126
|
+
if (hasBlockedTool(playbook)) {
|
|
127
|
+
console.error(`[community] Blocked: community playbook ${file} uses restricted tool`);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
playbooks.push(playbook);
|
|
101
131
|
}
|
|
102
132
|
catch { /* skip malformed */ }
|
|
103
133
|
}
|
|
@@ -6,9 +6,22 @@
|
|
|
6
6
|
* browser_stealth (anti-detection), memory_* (data exfil),
|
|
7
7
|
* supervisor_* (system control), job_* (persistence).
|
|
8
8
|
*/
|
|
9
|
+
/**
|
|
10
|
+
* TERMINAL_BUNDLE_IDS — apps that can execute arbitrary commands.
|
|
11
|
+
* Community playbooks must not launch or focus these.
|
|
12
|
+
*/
|
|
13
|
+
export const TERMINAL_BUNDLE_IDS = new Set([
|
|
14
|
+
"com.apple.Terminal",
|
|
15
|
+
"com.googlecode.iterm2",
|
|
16
|
+
"net.kovidgoyal.kitty",
|
|
17
|
+
"co.zeit.hyper",
|
|
18
|
+
"com.apple.ScriptEditor2",
|
|
19
|
+
"com.microsoft.VSCode", // integrated terminal
|
|
20
|
+
"com.todesktop.230313mzl4w4u92", // Cursor (Electron, has terminal)
|
|
21
|
+
]);
|
|
9
22
|
const SAFE_TOOLS = new Set([
|
|
10
23
|
"click", "click_text", "click_with_fallback", "type_text", "type_with_fallback",
|
|
11
|
-
"
|
|
24
|
+
"drag", "scroll", "scroll_with_fallback",
|
|
12
25
|
"screenshot", "screenshot_file", "ocr", "ui_tree", "ui_find", "ui_press",
|
|
13
26
|
"ui_set_value", "menu_click", "wait_for_state", "read_with_fallback",
|
|
14
27
|
"locate_with_fallback", "select_with_fallback",
|
|
@@ -16,6 +29,7 @@ const SAFE_TOOLS = new Set([
|
|
|
16
29
|
"browser_dom", "browser_wait", "browser_page_info", "browser_tabs",
|
|
17
30
|
"browser_fill_form", "browser_human_click",
|
|
18
31
|
"windows", "apps",
|
|
32
|
+
// REMOVED: "key" (char-by-char RCE via terminal), "launch" (can launch terminals), "focus" (can focus terminals)
|
|
19
33
|
]);
|
|
20
34
|
/**
|
|
21
35
|
* PlaybookValidator — tests a community playbook against the live app
|
|
@@ -96,6 +96,10 @@ export class ContextTracker {
|
|
|
96
96
|
this._previousPageContext = oldPage;
|
|
97
97
|
this._pendingTransition = { from: oldPage, to: newPage };
|
|
98
98
|
}
|
|
99
|
+
else if (!oldPage && newPage) {
|
|
100
|
+
// First page visit after app launch — record as entry point
|
|
101
|
+
this._pendingTransition = { from: "__initial__", to: newPage };
|
|
102
|
+
}
|
|
99
103
|
}
|
|
100
104
|
/**
|
|
101
105
|
* Consume a pending page transition (returns null if no transition occurred).
|
|
@@ -132,6 +136,8 @@ export class ContextTracker {
|
|
|
132
136
|
return; // javascript:, data:, blob: URLs have empty hostname
|
|
133
137
|
if (this.context?.domain === domain)
|
|
134
138
|
return;
|
|
139
|
+
// Flush pending learnings to the CURRENT (old) playbook before switching context
|
|
140
|
+
this.flush();
|
|
135
141
|
const playbook = this.store.matchByDomain(domain);
|
|
136
142
|
this.context = buildCachedContext(domain, playbook);
|
|
137
143
|
return;
|
|
@@ -145,12 +151,18 @@ export class ContextTracker {
|
|
|
145
151
|
const contextKey = `native:${bundleId}`;
|
|
146
152
|
if (this.context?.domain === contextKey)
|
|
147
153
|
return;
|
|
154
|
+
// Flush pending learnings to the CURRENT (old) playbook before switching context
|
|
155
|
+
this.flush();
|
|
148
156
|
const playbook = this.store.matchByBundleId(bundleId);
|
|
149
157
|
this.context = buildCachedContext(contextKey, playbook);
|
|
150
158
|
// Load app mastery map on bundleId change
|
|
151
159
|
if (this.appMap) {
|
|
152
160
|
this.context.appMapData = this.appMap.load(bundleId) ?? null;
|
|
153
161
|
this.appMap.incrementSession(bundleId);
|
|
162
|
+
// Wire #12: flag unknown apps so intelligence wrapper can suggest scan_menu_bar
|
|
163
|
+
if (!this.context.appMapData || Object.keys(this.context.appMapData.zones).length === 0) {
|
|
164
|
+
this.context.needsMenuScan = true;
|
|
165
|
+
}
|
|
154
166
|
}
|
|
155
167
|
}
|
|
156
168
|
}
|
|
@@ -198,14 +210,38 @@ export class ContextTracker {
|
|
|
198
210
|
// ═══════════════════════════════════════════════
|
|
199
211
|
// 2. GET HINTS — 0-2 lines per tool call
|
|
200
212
|
// ═══════════════════════════════════════════════
|
|
213
|
+
/** Wire #12: Clear the needsMenuScan hint after bootstrap succeeds */
|
|
214
|
+
clearMenuScanHint() {
|
|
215
|
+
if (this.context) {
|
|
216
|
+
this.context.needsMenuScan = false;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
201
219
|
/**
|
|
202
220
|
* Returns relevant hints for this tool call. Max 2 hints.
|
|
203
221
|
* Cost: map lookups only, ~0ms.
|
|
204
222
|
*/
|
|
205
223
|
getHints(toolName, params) {
|
|
206
|
-
if (!this.context
|
|
224
|
+
if (!this.context)
|
|
207
225
|
return [];
|
|
208
226
|
const hints = [];
|
|
227
|
+
// Wire #12: suggest scan_menu_bar for unknown apps (no playbook required)
|
|
228
|
+
if (hints.length < 2 && this.context.needsMenuScan) {
|
|
229
|
+
hints.push("💡 New app — run scan_menu_bar(pid, bundleId, appName) to bootstrap menu knowledge");
|
|
230
|
+
}
|
|
231
|
+
// App mastery map hint (no playbook required)
|
|
232
|
+
if (hints.length < 2 && this.context.appMapData) {
|
|
233
|
+
const map = this.context.appMapData;
|
|
234
|
+
const zones = Object.keys(map.zones).length;
|
|
235
|
+
const verifiedPaths = map.navigationGraph.edges.filter((e) => e.verified).length;
|
|
236
|
+
const totalPaths = map.navigationGraph.edges.length;
|
|
237
|
+
const ratingDisplay = map.rating
|
|
238
|
+
? (map.rating.grade === "0" ? "0" : `${map.rating.grade}${map.rating.subTier}`)
|
|
239
|
+
: map.masteryLevel.toUpperCase();
|
|
240
|
+
hints.push(`🗺 AppMap [${ratingDisplay}]: ${zones} zones, ${verifiedPaths}/${totalPaths} verified paths`);
|
|
241
|
+
}
|
|
242
|
+
// Playbook-dependent hints below
|
|
243
|
+
if (!this.context.playbook)
|
|
244
|
+
return hints;
|
|
209
245
|
// Check for known errors relevant to this tool
|
|
210
246
|
const errors = this.context.errorsByTool.get(toolName);
|
|
211
247
|
if (errors && errors.length > 0) {
|
|
@@ -238,50 +274,43 @@ export class ContextTracker {
|
|
|
238
274
|
}
|
|
239
275
|
catch { /* skip — don't break hints for a file read error */ }
|
|
240
276
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
277
|
+
return hints;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* L2→L1: Get a known CSS selector for a target element, if available.
|
|
281
|
+
* Returns the raw selector string for direct injection into querySelector(),
|
|
282
|
+
* not a display hint. Returns null if no matching or valid selector is found.
|
|
283
|
+
*
|
|
284
|
+
* Safety: Rejects non-CSS strings (Playwright pseudo-selectors, prose notes,
|
|
285
|
+
* AX paths) to prevent SyntaxError when passed to document.querySelector().
|
|
286
|
+
*/
|
|
287
|
+
getSelector(target) {
|
|
288
|
+
if (!this.context?.allSelectors || this.context.allSelectors.size === 0)
|
|
289
|
+
return null;
|
|
290
|
+
if (!target || target.length < 2)
|
|
291
|
+
return null;
|
|
292
|
+
const targetLower = target.toLowerCase();
|
|
293
|
+
let bestMatch = null;
|
|
294
|
+
for (const [name, sel] of this.context.allSelectors) {
|
|
295
|
+
// Skip annotation keys (convention: keys starting with "_" are notes)
|
|
296
|
+
if (name.includes("._") || name.startsWith("_"))
|
|
297
|
+
continue;
|
|
298
|
+
// Validate: must look like a CSS selector, not prose or Playwright syntax
|
|
299
|
+
if (!isLikelyCSSSelector(sel))
|
|
300
|
+
continue;
|
|
301
|
+
const nameLower = name.toLowerCase();
|
|
302
|
+
const suffix = nameLower.split(".").pop() ?? "";
|
|
303
|
+
// Prefer exact suffix match over contains match
|
|
304
|
+
if (suffix === targetLower) {
|
|
305
|
+
return sel; // Exact match — return immediately
|
|
261
306
|
}
|
|
262
|
-
//
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
navInfo = `, nav: ${navNodes} pages ${totalPaths} transitions`;
|
|
267
|
-
// Show outgoing edges from current page
|
|
268
|
-
if (this._currentPageContext) {
|
|
269
|
-
const outgoing = map.navigationGraph.edges.filter((e) => e.from === this._currentPageContext);
|
|
270
|
-
if (outgoing.length > 0) {
|
|
271
|
-
const destinations = outgoing
|
|
272
|
-
.slice(0, 3)
|
|
273
|
-
.map((e) => `${e.to} (${e.action})`)
|
|
274
|
-
.join(", ");
|
|
275
|
-
const more = outgoing.length > 3 ? ` +${outgoing.length - 3} more` : "";
|
|
276
|
-
navInfo += ` [from here: ${destinations}${more}]`;
|
|
277
|
-
}
|
|
278
|
-
}
|
|
307
|
+
// Only fuzzy-match against the final key segment (after last "."), not the full path
|
|
308
|
+
// This prevents "ok" from matching "booking.checkout" (contains "ok" in prefix)
|
|
309
|
+
if (!bestMatch && suffix.includes(targetLower)) {
|
|
310
|
+
bestMatch = sel;
|
|
279
311
|
}
|
|
280
|
-
hints.push(`🗺 Map: ${map.appName} — Rating ${ratingDisplay} ` +
|
|
281
|
-
`(${(map.confidence * 100).toFixed(0)}%, ${zones} zones, ` +
|
|
282
|
-
`${verifiedPaths}/${totalPaths} paths${pageInfo}${navInfo})`);
|
|
283
312
|
}
|
|
284
|
-
return
|
|
313
|
+
return bestMatch;
|
|
285
314
|
}
|
|
286
315
|
// ═══════════════════════════════════════════════
|
|
287
316
|
// 3. COLLECT — record outcome in memory buffer
|
|
@@ -396,6 +425,26 @@ export class ContextTracker {
|
|
|
396
425
|
getCurrentDomain() {
|
|
397
426
|
return this.context?.domain ?? null;
|
|
398
427
|
}
|
|
428
|
+
/**
|
|
429
|
+
* Wire F10: Get perception config hints from the active reference/playbook.
|
|
430
|
+
* Returns per-app perception interval overrides, or null if none configured.
|
|
431
|
+
*/
|
|
432
|
+
getPerceptionConfig() {
|
|
433
|
+
const playbook = this.context?.playbook;
|
|
434
|
+
if (!playbook?.perceptionConfig)
|
|
435
|
+
return null;
|
|
436
|
+
const pc = playbook.perceptionConfig;
|
|
437
|
+
const result = {};
|
|
438
|
+
if (typeof pc.fastIntervalMs === "number")
|
|
439
|
+
result.fastIntervalMs = pc.fastIntervalMs;
|
|
440
|
+
if (typeof pc.mediumIntervalMs === "number")
|
|
441
|
+
result.mediumIntervalMs = pc.mediumIntervalMs;
|
|
442
|
+
if (typeof pc.slowIntervalMs === "number")
|
|
443
|
+
result.slowIntervalMs = pc.slowIntervalMs;
|
|
444
|
+
if (typeof pc.enableVision === "boolean")
|
|
445
|
+
result.enableVision = pc.enableVision;
|
|
446
|
+
return Object.keys(result).length > 0 ? result : null;
|
|
447
|
+
}
|
|
399
448
|
}
|
|
400
449
|
// ── Helpers ──
|
|
401
450
|
function buildCachedContext(domain, playbook) {
|
|
@@ -432,6 +481,28 @@ function buildCachedContext(domain, playbook) {
|
|
|
432
481
|
}
|
|
433
482
|
return { domain, playbook, errorsByTool, allSelectors, appMapData: null };
|
|
434
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* Quick heuristic: does this string look like a valid CSS selector?
|
|
486
|
+
* Rejects Playwright pseudo-selectors, prose notes, AX paths, and empty strings.
|
|
487
|
+
* This is NOT a full CSS parser — it catches the common non-CSS patterns in reference files.
|
|
488
|
+
*/
|
|
489
|
+
function isLikelyCSSSelector(sel) {
|
|
490
|
+
if (!sel || sel.length < 2)
|
|
491
|
+
return false;
|
|
492
|
+
// Reject prose: contains spaces AND no CSS-like characters
|
|
493
|
+
if (sel.includes(" ") && !sel.includes("[") && !sel.includes(".") && !sel.includes("#") && !sel.includes(">") && !sel.includes(":"))
|
|
494
|
+
return false;
|
|
495
|
+
// Reject Playwright-specific syntax
|
|
496
|
+
if (sel.includes(">>") || sel.includes(":has-text(") || sel.includes(":text("))
|
|
497
|
+
return false;
|
|
498
|
+
// Reject strings that look like human-readable notes (start with uppercase word followed by space)
|
|
499
|
+
if (/^[A-Z][a-z]+ /.test(sel) && !sel.startsWith("["))
|
|
500
|
+
return false;
|
|
501
|
+
// Must start with a CSS-valid character: letter, #, ., [, *, :
|
|
502
|
+
if (!/^[a-zA-Z#.\[*:]/.test(sel))
|
|
503
|
+
return false;
|
|
504
|
+
return true;
|
|
505
|
+
}
|
|
435
506
|
function extractTarget(params) {
|
|
436
507
|
for (const name of TARGET_PARAM_NAMES) {
|
|
437
508
|
const val = params[name];
|
|
@@ -448,7 +519,8 @@ function findRelevantSelector(target, selectors) {
|
|
|
448
519
|
for (const [name, sel] of selectors) {
|
|
449
520
|
const nameLower = name.toLowerCase();
|
|
450
521
|
// If target text matches a selector name (e.g., target="Search" matches "toolbar.search")
|
|
451
|
-
|
|
522
|
+
const suffix = nameLower.split(".").pop() ?? "";
|
|
523
|
+
if (nameLower.includes(targetLower) || (suffix && targetLower.includes(suffix))) {
|
|
452
524
|
return `${name}: ${sel}`;
|
|
453
525
|
}
|
|
454
526
|
}
|
|
@@ -153,7 +153,9 @@ export class ReferenceMerger {
|
|
|
153
153
|
}
|
|
154
154
|
save(ref) {
|
|
155
155
|
fs.mkdirSync(this.referencesDir, { recursive: true });
|
|
156
|
-
|
|
156
|
+
// Sanitize ref.id to prevent path traversal (e.g. "../../.ssh/evil")
|
|
157
|
+
const safeId = ref.id.replace(/\.\./g, "_").replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
158
|
+
const filePath = path.join(this.referencesDir, `${safeId}.json`);
|
|
157
159
|
writeFileAtomicSync(filePath, JSON.stringify(ref, null, 2) + "\n");
|
|
158
160
|
return filePath;
|
|
159
161
|
}
|
|
@@ -40,11 +40,13 @@ export class LearningEngine {
|
|
|
40
40
|
dirty = false;
|
|
41
41
|
saveTimer = null;
|
|
42
42
|
constructor(config) {
|
|
43
|
+
const defaultDir = path.join(os.homedir(), ".screenhand", "learning");
|
|
44
|
+
const resolvedDir = config?.dataDir || defaultDir;
|
|
43
45
|
this.config = {
|
|
44
46
|
...DEFAULT_LEARNING_CONFIG,
|
|
45
|
-
dataDir: config?.dataDir ??
|
|
46
|
-
path.join(os.homedir(), ".screenhand", "learning"),
|
|
47
47
|
...config,
|
|
48
|
+
// Ensure dataDir is never empty — applied last to override any empty string from spread
|
|
49
|
+
dataDir: resolvedDir,
|
|
48
50
|
};
|
|
49
51
|
this.locators = new LocatorPolicy(this.config.priorStrength);
|
|
50
52
|
this.recovery = new RecoveryPolicy(this.config.priorStrength);
|
|
@@ -148,6 +150,169 @@ export class LearningEngine {
|
|
|
148
150
|
recommendNextNavigation(bundleId, fromNode) {
|
|
149
151
|
return this.topology.recommend(bundleId, fromNode, this.config.minSamplesForConfidence);
|
|
150
152
|
}
|
|
153
|
+
/**
|
|
154
|
+
* Wire #14: Seed TimingModel from AppMap's stored TimingProfiles.
|
|
155
|
+
* Scans known apps and feeds their timing profiles into the timing model
|
|
156
|
+
* so that adaptive budgets work from the first tool call in a new session.
|
|
157
|
+
*/
|
|
158
|
+
seedTimingFromAppMap(appMap) {
|
|
159
|
+
const bundleIds = appMap.listKnownApps();
|
|
160
|
+
for (const bundleId of bundleIds) {
|
|
161
|
+
const profiles = appMap.getTimingProfile(bundleId);
|
|
162
|
+
if (profiles.length > 0) {
|
|
163
|
+
this.timing.seedFromTimingProfiles(profiles, bundleId);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Wire F7a: Seed LocatorPolicy from AppMap's verified elements.
|
|
169
|
+
* Elements with 3+ successes get seeded as known-good locators.
|
|
170
|
+
*/
|
|
171
|
+
seedLocatorsFromAppMap(appMap) {
|
|
172
|
+
const bundleIds = appMap.listKnownApps();
|
|
173
|
+
for (const bundleId of bundleIds) {
|
|
174
|
+
const data = appMap.load(bundleId);
|
|
175
|
+
if (!data)
|
|
176
|
+
continue;
|
|
177
|
+
for (const zone of Object.values(data.zones)) {
|
|
178
|
+
for (const el of zone.elements) {
|
|
179
|
+
if (el.successCount < 3)
|
|
180
|
+
continue;
|
|
181
|
+
const key = LocatorPolicy.makeKey(bundleId, "click");
|
|
182
|
+
const score = (el.successCount + 2) / (el.successCount + el.failCount + 4);
|
|
183
|
+
this.locators.seedEntry({
|
|
184
|
+
key,
|
|
185
|
+
locator: el.label,
|
|
186
|
+
method: "ax",
|
|
187
|
+
successCount: el.successCount,
|
|
188
|
+
failCount: el.failCount,
|
|
189
|
+
score,
|
|
190
|
+
lastUsed: el.lastInteracted,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
// Wire F7b: Seed SensorPolicy from UIArchitecture
|
|
195
|
+
if (data.uiArchitecture) {
|
|
196
|
+
const arch = data.uiArchitecture;
|
|
197
|
+
const now = data.lastValidated || new Date().toISOString();
|
|
198
|
+
if (arch.axSupport === "full") {
|
|
199
|
+
this.sensors.seedEntry({
|
|
200
|
+
key: `${bundleId}::ax`, bundleId, sourceType: "ax",
|
|
201
|
+
successCount: 5, failCount: 0, score: 0.85, avgLatencyMs: 50, lastUsed: now,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
if (arch.rendering === "web-view" || arch.rendering === "hybrid") {
|
|
205
|
+
this.sensors.seedEntry({
|
|
206
|
+
key: `${bundleId}::cdp`, bundleId, sourceType: "cdp",
|
|
207
|
+
successCount: 4, failCount: 0, score: 0.8, avgLatencyMs: 10, lastUsed: now,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
if (arch.rendering === "custom-paint") {
|
|
211
|
+
this.sensors.seedEntry({
|
|
212
|
+
key: `${bundleId}::ocr`, bundleId, sourceType: "ocr",
|
|
213
|
+
successCount: 3, failCount: 0, score: 0.7, avgLatencyMs: 600, lastUsed: now,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Wire F6: Seed SensorPolicy from AppMap ReadySignals.
|
|
221
|
+
* Maps signal types to sensor sources and uses typicalMs as latency.
|
|
222
|
+
*/
|
|
223
|
+
seedSensorsFromReadySignals(appMap) {
|
|
224
|
+
const bundleIds = appMap.listKnownApps();
|
|
225
|
+
for (const bundleId of bundleIds) {
|
|
226
|
+
const signals = appMap.getReadySignals(bundleId);
|
|
227
|
+
for (const signal of signals) {
|
|
228
|
+
if (signal.sampleCount < 3)
|
|
229
|
+
continue;
|
|
230
|
+
let sourceType = null;
|
|
231
|
+
const sig = signal.signal.toLowerCase();
|
|
232
|
+
if (sig.includes("ax") || sig.includes("tree") || sig.includes("accessibility"))
|
|
233
|
+
sourceType = "ax";
|
|
234
|
+
else if (sig.includes("cdp") || sig.includes("dom") || sig.includes("chrome"))
|
|
235
|
+
sourceType = "cdp";
|
|
236
|
+
else if (sig.includes("ocr") || sig.includes("text") || sig.includes("vision"))
|
|
237
|
+
sourceType = "ocr";
|
|
238
|
+
if (!sourceType)
|
|
239
|
+
continue;
|
|
240
|
+
this.sensors.seedEntry({
|
|
241
|
+
key: `${bundleId}::${sourceType}`,
|
|
242
|
+
bundleId, sourceType,
|
|
243
|
+
successCount: signal.sampleCount,
|
|
244
|
+
failCount: 0,
|
|
245
|
+
score: 0.8,
|
|
246
|
+
avgLatencyMs: signal.typicalMs,
|
|
247
|
+
lastUsed: signal.lastSeen,
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Wire F7c: Seed PatternPolicy from AppMap contracts.
|
|
254
|
+
* Contracts with 2+ validations are seeded as known-good patterns.
|
|
255
|
+
*/
|
|
256
|
+
seedPatternsFromAppMap(appMap) {
|
|
257
|
+
const bundleIds = appMap.listKnownApps();
|
|
258
|
+
for (const bundleId of bundleIds) {
|
|
259
|
+
const data = appMap.load(bundleId);
|
|
260
|
+
if (!data)
|
|
261
|
+
continue;
|
|
262
|
+
for (const zone of Object.values(data.zones)) {
|
|
263
|
+
if (!zone.contracts)
|
|
264
|
+
continue;
|
|
265
|
+
for (const contract of zone.contracts) {
|
|
266
|
+
if (contract.validationCount < 2)
|
|
267
|
+
continue;
|
|
268
|
+
const key = `${bundleId}::${contract.action}::${contract.elementLabel}`;
|
|
269
|
+
const score = (contract.validationCount + 2) / (contract.validationCount + 4);
|
|
270
|
+
this.patterns.seedEntry({
|
|
271
|
+
key,
|
|
272
|
+
bundleId,
|
|
273
|
+
tool: contract.action,
|
|
274
|
+
locator: contract.elementLabel,
|
|
275
|
+
method: "ax",
|
|
276
|
+
successCount: contract.validationCount,
|
|
277
|
+
failCount: 0,
|
|
278
|
+
score,
|
|
279
|
+
lastSeen: contract.lastValidated,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Wire F5: Seed RecoveryPolicy from AppMap contracts with undo paths.
|
|
287
|
+
* Validated undo paths become recovery strategies for dialog blockers.
|
|
288
|
+
*/
|
|
289
|
+
seedRecoveryFromContracts(appMap) {
|
|
290
|
+
const bundleIds = appMap.listKnownApps();
|
|
291
|
+
for (const bundleId of bundleIds) {
|
|
292
|
+
const data = appMap.load(bundleId);
|
|
293
|
+
if (!data)
|
|
294
|
+
continue;
|
|
295
|
+
for (const zone of Object.values(data.zones)) {
|
|
296
|
+
if (!zone.contracts)
|
|
297
|
+
continue;
|
|
298
|
+
for (const contract of zone.contracts) {
|
|
299
|
+
if (!contract.undoPath || contract.validationCount < 2)
|
|
300
|
+
continue;
|
|
301
|
+
const key = `dialog::${bundleId}`;
|
|
302
|
+
const strategyId = `undo_${contract.elementLabel}`;
|
|
303
|
+
this.recovery.seedEntry({
|
|
304
|
+
key,
|
|
305
|
+
strategyId,
|
|
306
|
+
successCount: Math.min(contract.validationCount, 5),
|
|
307
|
+
failCount: 0,
|
|
308
|
+
score: 0.8,
|
|
309
|
+
avgDurationMs: 500,
|
|
310
|
+
lastUsed: contract.lastValidated,
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}
|
|
151
316
|
/**
|
|
152
317
|
* Get a summary of learning stats for a given app.
|
|
153
318
|
*/
|
|
@@ -195,7 +360,30 @@ export class LearningEngine {
|
|
|
195
360
|
this.sensors.clear();
|
|
196
361
|
this.patterns.clear();
|
|
197
362
|
this.topology.clear();
|
|
198
|
-
|
|
363
|
+
// Explicitly delete JSONL files — save() skips empty data (falsy guard),
|
|
364
|
+
// so stale files would resurrect on next init().
|
|
365
|
+
const dir = this.config.dataDir;
|
|
366
|
+
const files = [
|
|
367
|
+
"locators.jsonl",
|
|
368
|
+
"recoveries.jsonl",
|
|
369
|
+
"timings.jsonl",
|
|
370
|
+
"sensors.jsonl",
|
|
371
|
+
"patterns.jsonl",
|
|
372
|
+
"topology.jsonl",
|
|
373
|
+
];
|
|
374
|
+
for (const file of files) {
|
|
375
|
+
try {
|
|
376
|
+
fs.unlinkSync(path.join(dir, file));
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
// File may not exist — that's fine
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
this.dirty = false;
|
|
383
|
+
if (this.saveTimer) {
|
|
384
|
+
clearTimeout(this.saveTimer);
|
|
385
|
+
this.saveTimer = null;
|
|
386
|
+
}
|
|
199
387
|
}
|
|
200
388
|
/**
|
|
201
389
|
* Force save all policies to disk.
|
|
@@ -215,7 +403,6 @@ export class LearningEngine {
|
|
|
215
403
|
this.saveTimer = null;
|
|
216
404
|
if (this.dirty) {
|
|
217
405
|
this.save();
|
|
218
|
-
this.dirty = false;
|
|
219
406
|
}
|
|
220
407
|
}, 500);
|
|
221
408
|
}
|
|
@@ -283,6 +470,9 @@ export class LearningEngine {
|
|
|
283
470
|
if (topologyData) {
|
|
284
471
|
writeFileAtomicSync(path.join(dir, "topology.jsonl"), topologyData + "\n");
|
|
285
472
|
}
|
|
473
|
+
// Only clear dirty AFTER all writes succeed — if any write throws,
|
|
474
|
+
// dirty stays true so the next scheduled save will retry
|
|
475
|
+
this.dirty = false;
|
|
286
476
|
}
|
|
287
477
|
catch {
|
|
288
478
|
// Persistence failure is non-fatal — data stays in memory
|
|
@@ -325,11 +515,39 @@ export class LearningEngine {
|
|
|
325
515
|
try {
|
|
326
516
|
if (!fs.existsSync(filePath))
|
|
327
517
|
return [];
|
|
328
|
-
// Guard against oversized files:
|
|
518
|
+
// Guard against oversized files: truncate to recent entries if larger than 10MB
|
|
329
519
|
const stat = fs.statSync(filePath);
|
|
330
520
|
if (stat.size > 10 * 1024 * 1024) {
|
|
331
|
-
console.error(`[Learning]
|
|
332
|
-
|
|
521
|
+
console.error(`[Learning] WARN: Oversized file: ${filePath} (${(stat.size / 1024 / 1024).toFixed(1)}MB) — truncating to recent entries`);
|
|
522
|
+
// Read the last 5MB (most recent entries are at the end)
|
|
523
|
+
const fd = fs.openSync(filePath, "r");
|
|
524
|
+
const tailSize = 5 * 1024 * 1024;
|
|
525
|
+
const buf = Buffer.alloc(tailSize);
|
|
526
|
+
fs.readSync(fd, buf, 0, tailSize, stat.size - tailSize);
|
|
527
|
+
fs.closeSync(fd);
|
|
528
|
+
const tailContent = buf.toString("utf-8");
|
|
529
|
+
// Drop first partial line
|
|
530
|
+
const firstNewline = tailContent.indexOf("\n");
|
|
531
|
+
const cleanContent = firstNewline >= 0 ? tailContent.slice(firstNewline + 1) : tailContent;
|
|
532
|
+
// Overwrite with truncated content so it doesn't grow forever
|
|
533
|
+
try {
|
|
534
|
+
writeFileAtomicSync(filePath, cleanContent);
|
|
535
|
+
}
|
|
536
|
+
catch { /* non-fatal */ }
|
|
537
|
+
const results = [];
|
|
538
|
+
const maxEntries = this.config.maxEntriesPerFile;
|
|
539
|
+
for (const line of cleanContent.split("\n")) {
|
|
540
|
+
if (results.length >= maxEntries)
|
|
541
|
+
break;
|
|
542
|
+
const trimmed = line.trim();
|
|
543
|
+
if (!trimmed)
|
|
544
|
+
continue;
|
|
545
|
+
try {
|
|
546
|
+
results.push(JSON.parse(trimmed));
|
|
547
|
+
}
|
|
548
|
+
catch { /* skip corrupt */ }
|
|
549
|
+
}
|
|
550
|
+
return results;
|
|
333
551
|
}
|
|
334
552
|
const content = fs.readFileSync(filePath, "utf-8");
|
|
335
553
|
const results = [];
|
|
@@ -87,6 +87,22 @@ export class LocatorPolicy {
|
|
|
87
87
|
}
|
|
88
88
|
return result;
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Seed a single entry if no real data exists for that key+locator+method.
|
|
92
|
+
* Used for cold-start bootstrap from AppMap data.
|
|
93
|
+
*/
|
|
94
|
+
seedEntry(entry) {
|
|
95
|
+
const list = this.entries.get(entry.key);
|
|
96
|
+
if (list && list.some((e) => e.locator === entry.locator && e.method === entry.method)) {
|
|
97
|
+
return; // already have real data for this locator+method
|
|
98
|
+
}
|
|
99
|
+
if (!list) {
|
|
100
|
+
this.entries.set(entry.key, [{ ...entry }]);
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
list.push({ ...entry });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
90
106
|
/**
|
|
91
107
|
* Load entries from persisted data.
|
|
92
108
|
*/
|
|
@@ -71,6 +71,15 @@ export class PatternPolicy {
|
|
|
71
71
|
}
|
|
72
72
|
return null;
|
|
73
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Seed a single entry if no real data exists for that key.
|
|
76
|
+
* Used for cold-start bootstrap from AppMap contracts.
|
|
77
|
+
*/
|
|
78
|
+
seedEntry(entry) {
|
|
79
|
+
if (this.entries.has(entry.key))
|
|
80
|
+
return; // already have real data
|
|
81
|
+
this.entries.set(entry.key, { ...entry });
|
|
82
|
+
}
|
|
74
83
|
clear() {
|
|
75
84
|
this.entries.clear();
|
|
76
85
|
}
|
|
@@ -79,6 +79,22 @@ export class RecoveryPolicy {
|
|
|
79
79
|
qualified.sort((a, b) => b.score - a.score);
|
|
80
80
|
return qualified[0].strategyId;
|
|
81
81
|
}
|
|
82
|
+
/**
|
|
83
|
+
* Seed a single entry if no real data exists for that key+strategyId.
|
|
84
|
+
* Used for cold-start bootstrap from AppMap contracts with undo paths.
|
|
85
|
+
*/
|
|
86
|
+
seedEntry(entry) {
|
|
87
|
+
const list = this.entries.get(entry.key);
|
|
88
|
+
if (list && list.some((e) => e.strategyId === entry.strategyId)) {
|
|
89
|
+
return; // already have real data
|
|
90
|
+
}
|
|
91
|
+
if (!list) {
|
|
92
|
+
this.entries.set(entry.key, [{ ...entry }]);
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
list.push({ ...entry });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
82
98
|
clear() {
|
|
83
99
|
this.entries.clear();
|
|
84
100
|
}
|