quicke2e 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +560 -0
- package/bin/quicke2e.mjs +118 -0
- package/codegen/codegen.mjs +240 -0
- package/package.json +58 -0
- package/skill/quicke2e/SKILL.md +87 -0
- package/src/discover-pattern.mjs +8 -0
- package/src/discover.mjs +254 -0
- package/src/loop.mjs +748 -0
- package/src/redact.mjs +89 -0
- package/src/route.mjs +98 -0
- package/src/secret.mjs +94 -0
- package/src/seen-text.js +72 -0
- package/src/snapshot.js +246 -0
package/src/loop.mjs
ADDED
|
@@ -0,0 +1,748 @@
|
|
|
1
|
+
// The run loop. A decision model picks each step; nothing in the loop generates text.
|
|
2
|
+
// decisions -> the engine: jev (hosted, OpenRouter), vercel (Jev via the Vercel AI Gateway) or local
|
|
3
|
+
// text input -> from the test spec
|
|
4
|
+
// "am I done?" -> deterministic assertion
|
|
5
|
+
import { chromium } from "playwright";
|
|
6
|
+
import { buildRedactor, redactSnapshot } from "./redact.mjs";
|
|
7
|
+
import { SECRET_SRC, scrub, scrubText, scrubSpecText, isSecretKey, isStrong, redactPatterns } from "./secret.mjs";
|
|
8
|
+
import { pattern as urlPattern } from "./discover-pattern.mjs";
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
|
|
13
|
+
const __dir = path.dirname(fileURLToPath(import.meta.url));
|
|
14
|
+
const SEEN_TEXT = fs.readFileSync(path.join(__dir, "seen-text.js"), "utf8").replace(/^\/\/.*\n/gm, "").trim();
|
|
15
|
+
const SNAPSHOT = fs.readFileSync(path.join(__dir, "snapshot.js"), "utf8").replaceAll("__SECRET_SRC__", SECRET_SRC.replace(/\\/g, "\\\\"));
|
|
16
|
+
const KEY = process.env.OPENROUTER_API_KEY;
|
|
17
|
+
const LOCAL_URL = process.env.LOCAL_URL || "http://127.0.0.1:8822";
|
|
18
|
+
// FIX (2026-09-21): was hardcoded to the reference app. A client could not point this at their own app
|
|
19
|
+
// without editing the source. Resolution order: flow.base -> runOnce({base}) -> $APP_BASE -> default.
|
|
20
|
+
const DEFAULT_BASE = process.env.APP_BASE || "http://localhost:3000";
|
|
21
|
+
|
|
22
|
+
const RULES = "Choose the one operation that advances the goal from this page. " +
|
|
23
|
+
"Do not repeat a step that is already satisfied. Fill required fields before submitting.";
|
|
24
|
+
|
|
25
|
+
// Retry 429 and 5xx with backoff (roadmap #1; the bake-off lost 1 of 120 runs to an HTTP 520).
|
|
26
|
+
async function post(url, body, auth, tries = 4) {
|
|
27
|
+
for (let k = 0; ; k++) {
|
|
28
|
+
const res = await fetch(url, { method: "POST",
|
|
29
|
+
headers: { "Content-Type": "application/json", ...(auth ? { Authorization: `Bearer ${KEY}` } : {}) },
|
|
30
|
+
body: JSON.stringify(body) }).catch((e) => ({ ok: false, status: 0, text: async () => String(e) }));
|
|
31
|
+
if (res.ok) return res.json();
|
|
32
|
+
if (k + 1 < tries && (res.status === 0 || res.status === 429 || res.status >= 500)) {
|
|
33
|
+
await new Promise((r) => setTimeout(r, 400 * 2 ** k)); continue;
|
|
34
|
+
}
|
|
35
|
+
throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 140)}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ---- compression: the 1,024-token budget is the whole constraint ----
|
|
40
|
+
// FIX (2026-09-21): option/menuitem/link were absent and fell to the ?? 2 default, BELOW every
|
|
41
|
+
// button. Measured consequences: a 62-link list page offered 0 of its links (52 row checkboxes +
|
|
42
|
+
// 58 buttons ate the budget first); an OrangeHRM listbox had 7 options in the snapshot and 0
|
|
43
|
+
// survived, which silently disabled the open-listbox rule entirely; an Ant Pro nav flow scored
|
|
44
|
+
// 0/3 at budget 30 and 3/3 at 60. Options in an OPEN listbox are the only legal move, so they
|
|
45
|
+
// rank first; links are how you navigate, so they rank with buttons rather than below them.
|
|
46
|
+
const PRIORITY = { textbox: 0, password: 0, combobox: 0, button: 1, tab: 1, checkbox: 1, switch: 1, radio: 1 };
|
|
47
|
+
// REVERTED 2026-09-21. Adding option/menuitem at 0 and link at 1 FIXED link starvation
|
|
48
|
+
// (the 62-link list page went from 0 links offered to 13) and BROKE create-campaign 5/5 -> 0/5:
|
|
49
|
+
// with links promoted, the loop spent its budget cycling the five optional "rule" comboboxes and
|
|
50
|
+
// never reached Create campaign. Ranking is a real tension between list pages and forms, and it
|
|
51
|
+
// needs a measured fix, not a guess. Tracked separately; do not re-apply this naively.
|
|
52
|
+
// FIX (2026-09-21): `inputs` is now passed in so TYPE_TEXT can be suppressed on any field
|
|
53
|
+
// the test spec has no value for. Text comes from the spec by design, so a field with no
|
|
54
|
+
// spec value is not part of this test -- offering it produced a hard NO_SPEC_VALUE failure
|
|
55
|
+
// on the Budget field. Remove the option rather than handle the error. (README finding #2.)
|
|
56
|
+
// FIX (2026-09-22, found by codegen's drift test): matching the spec key against the LABEL is
|
|
57
|
+
// exactly what a rename breaks -- which makes it the one place the project cannot afford it.
|
|
58
|
+
// Measured on fixtures/pages/login.mutated.html, 3/3 trials on the PRISTINE repo: "Password" ->
|
|
59
|
+
// "Passphrase" makes hasSpec false, TYPE_TEXT is suppressed, the model CLICKS the password field
|
|
60
|
+
// instead, and the run ends MAX_STEPS on /login-denied.html. The README's flagship claim ("This
|
|
61
|
+
// loop: 5/5 on the rename, scripted Playwright 0/3") does not reproduce on the repo's own rename
|
|
62
|
+
// fixture. A password field is identified by its ROLE, which a rename cannot touch.
|
|
63
|
+
const ROLE_KEY = { password: "password" };
|
|
64
|
+
function keyFor(label, inputs, role) {
|
|
65
|
+
const l = (label || "").toLowerCase();
|
|
66
|
+
// Best match, not first match (audit A3: "Username" took the "name" key meant for "Full name"):
|
|
67
|
+
// exact label > whole-word match > substring; ties go to the longer key.
|
|
68
|
+
let best = null, bestScore = 0;
|
|
69
|
+
for (const k of Object.keys(inputs || {})) {
|
|
70
|
+
const kl = k.toLowerCase();
|
|
71
|
+
if (!l.includes(kl)) continue;
|
|
72
|
+
const word = new RegExp(`(^|[^a-z0-9])${kl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}([^a-z0-9]|$)`).test(l);
|
|
73
|
+
const score = (l === kl ? 3000 : word ? 2000 : 1000) + kl.length;
|
|
74
|
+
if (score > bestScore) { best = k; bestScore = score; }
|
|
75
|
+
}
|
|
76
|
+
if (best) return best;
|
|
77
|
+
const rk = ROLE_KEY[role];
|
|
78
|
+
if (rk && inputs && rk in inputs) return rk;
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
function hasSpec(label, inputs, role) { return keyFor(label, inputs, role) !== null; }
|
|
82
|
+
// SECURITY (v1 audit A7): a secret is not only a type=password field. An API key, token, PIN or card
|
|
83
|
+
// field is a plain textbox. Its value never goes to the model, and neither do secret spec values in
|
|
84
|
+
// the goal string (the audit found "Demo-Pass-42!" inside goals).
|
|
85
|
+
function redactGoal(goal, inputs, keepWeak = []) { return scrubSpecText(String(goal || ""), inputs, { keepWeak }); }
|
|
86
|
+
// The engine maps each option to ONE LETTER and reads only those logits: server.py:18 has 52
|
|
87
|
+
// letters, :127-128 does keys[:52] and reports the loss ONLY as timing.truncated -- which this
|
|
88
|
+
// file counted and never read. Criteria are emitted in document order with WAIT/DONE/BLOCKED
|
|
89
|
+
// appended LAST, so anything past the 52nd is discarded silently AND the escape hatches are the
|
|
90
|
+
// first casualties. This is reachable at the DEFAULT budget: 30 elements can emit 60 criteria
|
|
91
|
+
// (CLICK + TYPE_TEXT each) + 3 meta = 63. Measured by the adversary harness: at budget 250 the
|
|
92
|
+
// target link WAS in the criteria and the model was never shown it.
|
|
93
|
+
const LETTER_CEILING = 52;
|
|
94
|
+
const fillKey = (url, a) => `${new URL(url).pathname}|${a.role}|${a.label}`;
|
|
95
|
+
// BAKEOFF 2026-09-24 (docs/bakeoff-2026-09-24.md): goal-anchored keep. A candidate whose label
|
|
96
|
+
// tokens ALL occur in the goal is ranked -1, so neither the budget cut nor the letter-ceiling trim
|
|
97
|
+
// can drop it. Deep-row T3 went 0/40 -> 40/40 across 8 stacks with no representation change.
|
|
98
|
+
const tok = (t) => String(t || "").toLowerCase().split(/[^a-z0-9\u00c0-\u024f]+/).filter((w) => w.length > 1);
|
|
99
|
+
function compact(snap, budget, filled = new Set(), inputs = null, reserve = 3,
|
|
100
|
+
keyOf = (label, role) => keyFor(label, inputs, role), goal = null, typedKeys = new Map()) {
|
|
101
|
+
let els = [], targets = {}, criteria = {};
|
|
102
|
+
const idx = {};
|
|
103
|
+
const ops = { click: "CLICK", fill: "TYPE_TEXT", select: "SELECT" };
|
|
104
|
+
const raw = [];
|
|
105
|
+
for (const a of snap.actions) {
|
|
106
|
+
const op = ops[a.kind]; if (!op) continue;
|
|
107
|
+
// FIX (v1): native <select>. Each OPTION is its own action, keyed by option -- keying by node
|
|
108
|
+
// kept only the placeholder, and SELECT had no dispatch branch, so fixtures/select.html burned
|
|
109
|
+
// all 10 steps in 64 s. The value comes from the spec when the spec names this control (text
|
|
110
|
+
// comes from the spec, never from the model); otherwise a few real options are offered.
|
|
111
|
+
// Option actions carry selectNode/selectLabel/optionLabel (snapshot.js, A+).
|
|
112
|
+
if (op === "SELECT") {
|
|
113
|
+
if (!a.value || a.value === a.current_value) continue; // placeholder, or already set
|
|
114
|
+
const k = keyOf(a.selectLabel, "combobox");
|
|
115
|
+
if (k !== null) { const want = norm(inputs[k]).toLowerCase();
|
|
116
|
+
if (norm(a.optionLabel).toLowerCase() !== want && norm(a.value).toLowerCase() !== want) continue; }
|
|
117
|
+
else if (raw.filter((r) => r.a.selectNode === a.selectNode).length >= 8) continue;
|
|
118
|
+
raw.push({ a: { ...a, label: `${a.optionLabel} in ${a.selectLabel}` }, ops: [op] });
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
// FIX (v1): keyed by page + role + label, not the per-snapshot ordinal. `n0` on page 2 is a
|
|
122
|
+
// different field from `n0` on page 1 (fixtures/ordinal-a/b.html: page 2's only field vanished).
|
|
123
|
+
if (op === "TYPE_TEXT") {
|
|
124
|
+
const k = keyOf(a.label, a.role);
|
|
125
|
+
if (k === null) continue; // no spec value: not this test's field
|
|
126
|
+
// One spec key fills ONE field per page (audit B1 / fixtures/ambiguous.html): once "name" went
|
|
127
|
+
// into one of First/Middle/Last name, the other two are not this key's fields. A re-fill of
|
|
128
|
+
// the SAME field after a reload (TicketBay D1) is still allowed.
|
|
129
|
+
const used = typedKeys.get(`${new URL(snap.url).pathname}|${k}`);
|
|
130
|
+
if (used && used !== a.label) continue;
|
|
131
|
+
// FIX (v1, TicketBay D1): "filled" is read off the DOM, not remembered. A field that already
|
|
132
|
+
// holds its spec value is done; a field we typed into that is EMPTY again (the form reloaded,
|
|
133
|
+
// e.g. an Apply-code GET submit) is offered again. Remembered-and-non-empty stays suppressed,
|
|
134
|
+
// so a masked field (phone formatting) cannot loop.
|
|
135
|
+
if (norm(a.value) === norm(inputs[k])) continue;
|
|
136
|
+
if (filled.has(fillKey(snap.url, a)) && norm(a.value)) continue;
|
|
137
|
+
}
|
|
138
|
+
// FIX (2026-09-21): never offer CLICK on a plain textbox. Text is entered via TYPE_TEXT from
|
|
139
|
+
// the spec, so clicking one is always a wasted step -- measured: 5 of 22 steps burned clicking
|
|
140
|
+
// Budget/Campaign name, exhausting the budget before Save. Comboboxes keep CLICK (they open).
|
|
141
|
+
// FIX (2026-09-22): "password" was missing. snapshot.js:87 gives a password input role
|
|
142
|
+
// "password", not "textbox", so this guard never covered it -- and on the rename fixture the
|
|
143
|
+
// model duly clicked the password field, three trials out of three.
|
|
144
|
+
if (op === "CLICK" && (a.role === "textbox" || a.role === "password")) continue;
|
|
145
|
+
if (!(a.node in idx)) { idx[a.node] = raw.length; raw.push({ a, ops: [] }); }
|
|
146
|
+
if (!raw[idx[a.node]].ops.includes(op)) raw[idx[a.node]].ops.push(op);
|
|
147
|
+
}
|
|
148
|
+
// FIX (v1): hold back a form's SUBMIT while a field the spec names in that form is still unset.
|
|
149
|
+
// Measured on fixtures/select.html: the model typed Team, then chose "Save subscription" (0.61)
|
|
150
|
+
// over "SELECT Growth" and saved an empty plan -- finding #2 again. Remove the option.
|
|
151
|
+
// A select is pending only while the CONTROL is unset (bake-off re-run: counting every remaining
|
|
152
|
+
// option kept "Create project" hidden forever after Enterprise was chosen -- 0/5 on two stacks).
|
|
153
|
+
const pending = new Set(raw.filter((r) => r.a.form >= 0 && (r.ops.includes("TYPE_TEXT")
|
|
154
|
+
|| (r.ops.includes("SELECT") && (!r.a.current_value || keyOf(r.a.selectLabel, "combobox") !== null))))
|
|
155
|
+
.map((r) => r.a.form));
|
|
156
|
+
for (let j = raw.length - 1; j >= 0; j--)
|
|
157
|
+
if (raw[j].a.submit && pending.has(raw[j].a.form)) raw.splice(j, 1);
|
|
158
|
+
// Rank: form controls and buttons first, then links in document order. Links are the
|
|
159
|
+
// repetitive part of a list page (55 creator rows) and the first to be dropped.
|
|
160
|
+
const goalToks = goal ? new Set(tok(goal)) : null;
|
|
161
|
+
const named = (a) => { const t = tok(a.label); return t.length > 0 && t.every((w) => goalToks.has(w)); };
|
|
162
|
+
const ranked = raw.map((r, i) => ({ ...r, i, rank: goalToks && named(r.a) ? -1 : PRIORITY[r.a.role] ?? 2 }))
|
|
163
|
+
.sort((x, y) => x.rank - y.rank || x.i - y.i);
|
|
164
|
+
let keep = budget ? ranked.slice(0, budget) : ranked;
|
|
165
|
+
// Trim from the LOWEST-RANKED end until every criterion fits under the letter ceiling with room
|
|
166
|
+
// for the meta options. Dropping here is visible (space.dropped); dropping in the engine is not.
|
|
167
|
+
const critCount = (k) => k.reduce((n, r) => n + r.ops.length, 0);
|
|
168
|
+
let lettersLost = 0;
|
|
169
|
+
while (keep.length && critCount(keep) + reserve > LETTER_CEILING) { keep.pop(); lettersLost++; }
|
|
170
|
+
keep.sort((x, y) => x.i - y.i);
|
|
171
|
+
|
|
172
|
+
// BAKEOFF: removing the chosen option also removed the evidence it was chosen (SELECT@0.99 then
|
|
173
|
+
// BLOCKED 4/4). A <select>'s value is shown as a non-choosable element, and a checkbox shows
|
|
174
|
+
// checked/unchecked instead of its value attribute ("on", always).
|
|
175
|
+
// v1: measured on fixtures/select.html -- the state as a non-choosable ELEMENT made Jev answer
|
|
176
|
+
// BLOCKED (0.37) with Save as the only real option; the same facts as a separate `values` map
|
|
177
|
+
// gave CLICK Save at 0.99. So form state goes in `values`, and `elements` holds only choices.
|
|
178
|
+
const values = {};
|
|
179
|
+
for (const a of snap.actions) {
|
|
180
|
+
if (a.kind === "state" && a.value && !/^(choose|select|pick|--)/i.test(a.value)) values[a.label] = String(a.value).slice(0, 40);
|
|
181
|
+
// A text field's CONTENT is sent only when it is exactly a NON-secret spec value (text the spec
|
|
182
|
+
// supplied); anything else -- a pre-filled page value, a secret key's value -- is "(filled)".
|
|
183
|
+
// Measured: with "(filled)" everywhere, Jev typed WELCOME10 and paid without clicking Apply
|
|
184
|
+
// (TicketBay book-with-code 5/5 -> 0/5); seeing "Discount code: WELCOME10" it applies it.
|
|
185
|
+
if (a.kind === "fill" && norm(a.value)) {
|
|
186
|
+
const k = keyOf(a.label, a.role);
|
|
187
|
+
values[a.label] = k !== null && !a.secret && !isSecretKey(k) && norm(a.value) === norm(inputs?.[k])
|
|
188
|
+
? String(a.value).slice(0, 40) : "(filled)";
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
// Rendering is a function of the rows, so the tournament (runOnce) can render ANY subset of the
|
|
192
|
+
// ranked candidates with exactly the same labels, values and keys as the normal path.
|
|
193
|
+
const render = (rows) => {
|
|
194
|
+
const els = [], criteria = {}, targets = {};
|
|
195
|
+
let num = 0;
|
|
196
|
+
for (const { a, ops: o } of rows) {
|
|
197
|
+
const n = String(++num);
|
|
198
|
+
// scrub BEFORE truncating: a cut-off echo of a secret cannot be recognised afterwards (audit R2b)
|
|
199
|
+
const label = scrubText(a.label || "", inputs).slice(0, 44);
|
|
200
|
+
const specShown = (x) => { const k = keyOf(x.label, x.role);
|
|
201
|
+
return k !== null && !x.secret && !isSecretKey(k) && norm(x.value) && norm(x.value) === norm(inputs?.[k]) ? x.value : null; };
|
|
202
|
+
const v = a.role === "checkbox" || a.role === "radio" ? (a.checked ? "checked" : "unchecked")
|
|
203
|
+
: a.kind === "fill" || a.role === "textbox" ? specShown(a) // only a non-secret spec value we typed
|
|
204
|
+
: a.kind === "select" || a.role === "password" ? null
|
|
205
|
+
: a.role === "combobox" && a.tag !== "input" && !a.secret ? a.value : null; // audit S5
|
|
206
|
+
els.push({ i: n, r: a.role, l: label, ...(a.popup ? { p: 1 } : {}), ...(v && !a.secret ? { v: String(v).slice(0, 24) } : {}) });
|
|
207
|
+
// NB a select OPTION's `value` is the option, not the control's state: sending it as `v` told
|
|
208
|
+
// the model the plan was already "Growth", and it answered DONE four times (fixtures/select.html).
|
|
209
|
+
// FIX (2026-09-21): put the ROLE in the choice text. Without it the model had to pick between
|
|
210
|
+
// "CLICK 5 Clients" (a nav link) and "CLICK 12 Client" (the form combobox) on label alone,
|
|
211
|
+
// and chose the nav link at 0.55 confidence -- then wandered off the form entirely. The role
|
|
212
|
+
// is already in state.elements; it was simply missing from the thing being chosen between.
|
|
213
|
+
for (const op of o) { (targets[op] ??= {})[n] = a; criteria[`${op}:${n}`] = `${op} ${n} ${label} [${a.role}]`; }
|
|
214
|
+
}
|
|
215
|
+
return { els, criteria, targets };
|
|
216
|
+
};
|
|
217
|
+
({ els, criteria, targets } = render(keep));
|
|
218
|
+
const dropped = ranked.length - keep.length;
|
|
219
|
+
const lettersTrimmed = lettersLost;
|
|
220
|
+
return { els, criteria, targets, dropped, lettersTrimmed, total: ranked.length, values,
|
|
221
|
+
render, all: [...ranked].sort((x, y) => x.i - y.i) };
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async function decide({ els, criteria, values }, snap, goal, history, engine, inputs, weakKeys, redact) {
|
|
225
|
+
const state = {
|
|
226
|
+
url: urlPattern(snap.url), // SECURITY (audit S4): token-like path segments become :id
|
|
227
|
+
title: snap.title.slice(0, 60),
|
|
228
|
+
elements: els,
|
|
229
|
+
...(values && Object.keys(values).length ? { values } : {}),
|
|
230
|
+
done: history.slice(-5),
|
|
231
|
+
};
|
|
232
|
+
if (process.env.JEV_DEBUG) console.error(JSON.stringify({ state, criteria }));
|
|
233
|
+
return choose({ state, criteria, goal, engine, inputs, weakKeys, redact });
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// One typed choice over the offered keys, on any engine. decide() uses it for the next action;
|
|
237
|
+
// the map router (src/route.mjs) uses it to pick a target page. Same wire shape everywhere.
|
|
238
|
+
export async function choose({ state, criteria, goal, rules = RULES, engine = "jev", inputs, weakKeys = [], redact = [] }) {
|
|
239
|
+
// SECURITY: every secret spec value is scrubbed from the entire payload, wherever it appears
|
|
240
|
+
// (weak ones only where the loop saw them echoed); declared redact patterns apply to page content.
|
|
241
|
+
({ state, criteria, goal, rules } = scrub({ state, criteria, goal, rules }, inputs, { weakKeys }));
|
|
242
|
+
({ state, criteria } = redactPatterns({ state, criteria }, redact));
|
|
243
|
+
const questions = { action: { type: "choice", criteria, instructions: { goal, rules } } };
|
|
244
|
+
const t0 = performance.now();
|
|
245
|
+
if (engine === "vercel") {
|
|
246
|
+
const res = await fetch("https://ai-gateway.vercel.sh/v1/evaluate", {
|
|
247
|
+
method: "POST",
|
|
248
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${process.env.AI_GATEWAY_API_KEY}` },
|
|
249
|
+
body: JSON.stringify({ model: "typesafe-ai/jev", state: JSON.stringify(state), questions }),
|
|
250
|
+
});
|
|
251
|
+
if (!res.ok) throw new Error(`vercel HTTP ${res.status}: ${(await res.text()).slice(0,200)}`);
|
|
252
|
+
const r = await res.json();
|
|
253
|
+
const g = r.providerMetadata?.gateway || {};
|
|
254
|
+
return { choice: r.answers.action.choice, confidence: r.answers.action.confidence ?? 1,
|
|
255
|
+
ms: performance.now() - t0, tokens: r.usage?.inputTokens || 0,
|
|
256
|
+
cost: Number(g.cost ?? 0), marketCost: Number(g.marketCost ?? 0), truncated: false };
|
|
257
|
+
}
|
|
258
|
+
if (engine === "jev") {
|
|
259
|
+
const r = await post("https://openrouter.ai/api/alpha/decisions",
|
|
260
|
+
{ model: "typesafe/jev-1.13", state, questions }, true);
|
|
261
|
+
return { choice: r.answers.action.choice, confidence: r.answers.action.confidence,
|
|
262
|
+
ms: performance.now() - t0, tokens: r.usage?.input_tokens || 0, cost: r.usage?.cost || 0, truncated: false };
|
|
263
|
+
}
|
|
264
|
+
// "local": an open decision model served by local-engine/server.py, constrained to the offered
|
|
265
|
+
// option labels by construction -- one prefill, then read only the labels' logits.
|
|
266
|
+
if (engine !== "local") throw new Error(`unknown engine "${engine}": use jev, vercel or local`);
|
|
267
|
+
const r = await post(LOCAL_URL, { state, questions }, false);
|
|
268
|
+
if (r.error) throw new Error(engine + ": " + r.error);
|
|
269
|
+
return { choice: r.answers.action.choice, confidence: r.answers.action.confidence,
|
|
270
|
+
ms: performance.now() - t0, inferMs: r.timing.infer_ms, queuedMs: r.timing.queued_ms,
|
|
271
|
+
tokens: r.usage.input_tokens, cost: 0, truncated: r.timing.truncated };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Text comes from the test spec. No model invents test data.
|
|
275
|
+
// Same resolution as compact()'s suppression check, or the loop offers a field it cannot fill.
|
|
276
|
+
function specValue(flow, action, keyMap) {
|
|
277
|
+
const k = keyFor(action.label, flow.inputs, action.role) ?? keyMap?.get(action.label) ?? null;
|
|
278
|
+
return k === null ? null : flow.inputs[k];
|
|
279
|
+
}
|
|
280
|
+
// CODEGEN (2026-09-22): which inputs KEY matched, not the value. The generated spec references
|
|
281
|
+
// INPUTS.<key>, so the credential stays in the spec file and never enters the trace on disk.
|
|
282
|
+
function specKey(flow, action, keyMap) { return keyFor(action.label, flow.inputs, action.role) ?? keyMap?.get(action.label) ?? null; }
|
|
283
|
+
|
|
284
|
+
// Jev maps an unmatched text field to one of the spec's UNUSED keys, or to NONE.
|
|
285
|
+
async function resolveKeys(snap, inputs, keyMap, engine, addCost, redact = []) {
|
|
286
|
+
const fields = snap.actions.filter((a) => a.kind === "fill" && !keyMap.has(a.label));
|
|
287
|
+
if (!fields.length) return;
|
|
288
|
+
const used = new Set();
|
|
289
|
+
for (const a of snap.actions) if (a.kind === "fill") {
|
|
290
|
+
const k = keyFor(a.label, inputs, a.role) ?? keyMap.get(a.label);
|
|
291
|
+
if (k) used.add(k);
|
|
292
|
+
}
|
|
293
|
+
for (const a of fields) {
|
|
294
|
+
if (keyFor(a.label, inputs, a.role) !== null) continue;
|
|
295
|
+
// Never map free text into a typed input (measured: the local engine put a name into the
|
|
296
|
+
// "Tickets" number field, the fill failed, and the run retried it 12 times).
|
|
297
|
+
if (/^(number|range|date|time|datetime-local|month|week|color|file)$/.test(a.itype || "")) { keyMap.set(a.label, null); continue; }
|
|
298
|
+
const free = Object.keys(inputs).filter((k) => !used.has(k));
|
|
299
|
+
if (!free.length || !a.label || a.label === "(unlabeled)") { keyMap.set(a.label, null); continue; }
|
|
300
|
+
const criteria = { NONE: "none of these values belongs in this field" };
|
|
301
|
+
free.forEach((k, i) => { criteria[`KEY:${i}`] = `the test value "${k}" goes in this field`; });
|
|
302
|
+
const d = await choose({ state: { field: a.label, role: a.role, page: new URL(snap.url).pathname },
|
|
303
|
+
criteria, goal: `Which test value belongs in the form field labelled "${a.label}"?`,
|
|
304
|
+
rules: "Match by meaning: e-mail is email, passphrase is password, ticket holder is name.", engine, inputs, redact });
|
|
305
|
+
addCost(d.cost || 0);
|
|
306
|
+
const k = String(d.choice).startsWith("KEY:") && d.confidence >= 0.5 ? free[Number(d.choice.split(":")[1])] : null;
|
|
307
|
+
keyMap.set(a.label, k);
|
|
308
|
+
if (k) used.add(k);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// FIX (2026-09-21): ONE normaliser, used by both sides. The model read a \s+-normalised copy
|
|
313
|
+
// while checkGoal compared raw innerText, so a correct run was failed by a single .
|
|
314
|
+
// Playwright\u2019s rule: the element name and the expected name go through the SAME function
|
|
315
|
+
// (roleSelectorEngine.ts:167-169). \\s+ already matches NBSP; zero-width chars are stripped
|
|
316
|
+
// because MUI renders one to hold line height and it reads as a truthy value.
|
|
317
|
+
const norm = (t) => String(t == null ? "" : t).replace(/\s+/g, " ").trim();
|
|
318
|
+
|
|
319
|
+
// v1: counts controls inside open shadow roots too -- a Web Components page has ~0 light-DOM
|
|
320
|
+
// controls, so every step there waited the full 4 s timeout (bake-off: T2 took 18.7 s).
|
|
321
|
+
// v1: "at least one control, once parsed" -- the old "> 2 controls" was false on small pages, so every
|
|
322
|
+
// non-typing step there waited 2.5 s + 4 s (audit: ~7 s per SELECT step). Counts open shadow roots
|
|
323
|
+
// too: a Web Components page has ~0 light-DOM controls (bake-off: T2 18.7 s -> 2.8 s).
|
|
324
|
+
const READY = () => {
|
|
325
|
+
if (document.readyState === "loading") return false;
|
|
326
|
+
// A fully loaded page is ready even with no controls (a confirmation page): waiting for one cost
|
|
327
|
+
// 4 s per landing (fixtures login-done.html). SPA hydration is settle()'s job (DOM quiet).
|
|
328
|
+
if (document.readyState === "complete") return true;
|
|
329
|
+
const Q = "input:not([type=hidden]),button,a[href],select,textarea,[role=button]";
|
|
330
|
+
if (document.querySelector(Q)) return true;
|
|
331
|
+
for (const el of document.querySelectorAll("*")) if (el.shadowRoot?.querySelector(Q)) return true;
|
|
332
|
+
return false;
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
// FIX (2026-09-21): READY only asks "are there a few controls?", which is true almost immediately
|
|
336
|
+
// and does NOT mean React has finished rendering. Measured: identical code and spec passed 5/5
|
|
337
|
+
// headless but failed 3/3 headed with slowMo:120 -- the snapshot was catching half-rendered forms,
|
|
338
|
+
// so the model saw different option sets and diverged. Wait for the DOM to go QUIET instead.
|
|
339
|
+
async function settle(page, quietMs = 150, timeout = 1200) {
|
|
340
|
+
await page.evaluate(
|
|
341
|
+
([quiet, cap]) =>
|
|
342
|
+
new Promise((resolve) => {
|
|
343
|
+
let t = setTimeout(done, quiet);
|
|
344
|
+
const obs = new MutationObserver(() => { clearTimeout(t); t = setTimeout(done, quiet); });
|
|
345
|
+
obs.observe(document.documentElement, { childList: true, subtree: true, attributes: true, characterData: true });
|
|
346
|
+
const hard = setTimeout(done, cap);
|
|
347
|
+
function done() { clearTimeout(t); clearTimeout(hard); obs.disconnect();
|
|
348
|
+
const anims = document.getAnimations ? document.getAnimations() : [];
|
|
349
|
+
if (!anims.length) return resolve();
|
|
350
|
+
Promise.race([Promise.all(anims.map((a) => a.finished.catch(() => {}))),
|
|
351
|
+
new Promise((r) => setTimeout(r, 400))]).then(resolve, resolve); }
|
|
352
|
+
}),
|
|
353
|
+
[quietMs, timeout]
|
|
354
|
+
).catch(() => {});
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// FIX (2026-09-22): `expect` is a DOM substring, and a substring cannot express a SELECTION.
|
|
358
|
+
// Measured on a create form with a Radix client picker: expect:["Acme DevTools"] is true on the very first
|
|
359
|
+
// snapshot, before the loop touches anything -- Radix renders a visually-hidden native <select>
|
|
360
|
+
// carrying every client's name for form submission, so the text is on the page from page load.
|
|
361
|
+
// Opening the portal dropdown has the same effect on any library. `expectState` asserts the
|
|
362
|
+
// {role, name, value} triple of the CONTROL instead, which is only true once a value is set:
|
|
363
|
+
//
|
|
364
|
+
// expectState: [{ role: "combobox", name: "Client", value: "Acme DevTools" }]
|
|
365
|
+
// expectState: [{ role: "checkbox", name: "Select Never Activated", checked: true }]
|
|
366
|
+
//
|
|
367
|
+
// Matched against the SAME in-page snapshot the model is offered, so it costs one extra
|
|
368
|
+
// evaluate() (~1-7 ms) and needs no CDP session. `value` matches by substring because the
|
|
369
|
+
// snapshot truncates a combobox value at 40 chars; the flags match exactly.
|
|
370
|
+
function stateOk(snap, want) {
|
|
371
|
+
return (want || []).every((w) => (snap?.actions || []).some((a) => {
|
|
372
|
+
if (w.role && a.role !== w.role) return false;
|
|
373
|
+
if (w.name != null && norm(a.label) !== norm(w.name)) return false;
|
|
374
|
+
if (w.value != null && !norm(a.value).includes(norm(w.value))) return false;
|
|
375
|
+
if (w.selected != null && a.selected !== w.selected) return false;
|
|
376
|
+
if (w.checked != null && a.checked !== w.checked) return false;
|
|
377
|
+
if (w.expanded != null && a.expanded !== w.expanded) return false;
|
|
378
|
+
return true;
|
|
379
|
+
}));
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Every text node that appears is remembered for the life of the document (capped).
|
|
383
|
+
const SEEN_RECORDER = `(() => {
|
|
384
|
+
window.__jevSeen = [];
|
|
385
|
+
const rec = (n) => { const t = (n.textContent || "").trim(); if (t && t.length < 300 && window.__jevSeen.length < 2000) window.__jevSeen.push(t); };
|
|
386
|
+
new MutationObserver((ms) => { for (const m of ms) { m.addedNodes.forEach(rec); if (m.type === "characterData") rec(m.target); } })
|
|
387
|
+
.observe(document, { childList: true, subtree: true, characterData: true });
|
|
388
|
+
})();`;
|
|
389
|
+
|
|
390
|
+
async function checkGoal(page, flow, snap) {
|
|
391
|
+
try {
|
|
392
|
+
const urlOk = flow.expectUrl ? new RegExp(flow.expectUrl).test(page.url()) : true;
|
|
393
|
+
if (!urlOk) return false;
|
|
394
|
+
// `expect` asserts against what a USER perceives (src/seen-text.js), not raw innerText: form
|
|
395
|
+
// controls contribute nothing (assert them with expectState), hidden/aria-hidden text not at all.
|
|
396
|
+
const need = [...(flow.expect || []), ...(flow.expectSeen || [])];
|
|
397
|
+
const seen = need.length ? norm(await page.evaluate(`(${SEEN_TEXT})()`)) : "";
|
|
398
|
+
if (!(flow.expect || []).every((x) => seen.includes(norm(x)))) return false;
|
|
399
|
+
// v1 (audit A5): `expectSeen` passes if the text appeared at ANY moment since the page loaded --
|
|
400
|
+
// for a toast that lives 300 ms, shorter than a step. Recorded by SEEN_RECORDER (init script).
|
|
401
|
+
if (flow.expectSeen?.length) {
|
|
402
|
+
const past = norm((await page.evaluate(() => window.__jevSeen || [])).join(" "));
|
|
403
|
+
if (!flow.expectSeen.every((x) => seen.includes(norm(x)) || past.includes(norm(x)))) return false;
|
|
404
|
+
}
|
|
405
|
+
if (!flow.expectState?.length) return true;
|
|
406
|
+
// A control's state is only readable once the portal has closed and put the value back on
|
|
407
|
+
// the trigger: while a Radix listbox is open the trigger is not in the candidate set at all.
|
|
408
|
+
// The loop already checks the goal on EVERY snapshot (finding #8), so the first snapshot
|
|
409
|
+
// after the dropdown closes is where this fires. Nothing extra is needed to wait for it.
|
|
410
|
+
return stateOk(snap ?? await page.evaluate(`(${SNAPSHOT})()`), flow.expectState);
|
|
411
|
+
} catch { return false; }
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
// A snapshot-time attribute goes stale the moment a component re-renders its portal
|
|
416
|
+
// (Radix/shadcn selects, menus, dialogs). Resolve semantically at click time instead.
|
|
417
|
+
const ARIA = { textbox: "textbox", password: "textbox", button: "button", link: "link",
|
|
418
|
+
option: "option", combobox: "combobox", tab: "tab", checkbox: "checkbox", switch: "switch",
|
|
419
|
+
radio: "radio", menuitem: "menuitem" };
|
|
420
|
+
// BAKEOFF: the snapshot stamps inside open shadow roots (Playwright CSS pierces them) and
|
|
421
|
+
// same-origin iframes (it does not), so the stamp is looked up in every frame, then the role chain.
|
|
422
|
+
async function byRole(scope, action, fn) {
|
|
423
|
+
const role = ARIA[action.role], name = (action.label || "").trim();
|
|
424
|
+
if (role && name) {
|
|
425
|
+
try { await fn(scope.getByRole(role, { name, exact: true }).first(), 1500); return "role"; } catch {}
|
|
426
|
+
try { await fn(scope.getByRole(role, { name }).first(), 1200); return "role~"; } catch {}
|
|
427
|
+
}
|
|
428
|
+
if (name) {
|
|
429
|
+
try { await fn(scope.getByText(name, { exact: true }).filter({ visible: true }).first(), 1200); return "text"; } catch {}
|
|
430
|
+
}
|
|
431
|
+
throw new Error("unresolvable");
|
|
432
|
+
}
|
|
433
|
+
async function act(page, action, fn) {
|
|
434
|
+
for (const f of page.frames()) {
|
|
435
|
+
const byNode = f.locator(`[data-jev-node="${action.node}"]`);
|
|
436
|
+
if (!(await byNode.count().catch(() => 0))) continue;
|
|
437
|
+
try { await fn(byNode, 1200); return f === page.mainFrame() ? "node" : "node@frame"; } catch {}
|
|
438
|
+
break;
|
|
439
|
+
}
|
|
440
|
+
for (const f of page.frames()) { try { return await byRole(f, action, fn); } catch {} }
|
|
441
|
+
throw new Error("unresolvable");
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
export async function runOnce({ flow, engine = "local", budget = 30, browser: shared, trace,
|
|
445
|
+
base, context: ctxOpts, map, route: routeMode = "click", video } = {}) {
|
|
446
|
+
base = (flow.base || base || DEFAULT_BASE).replace(/\/$/, "");
|
|
447
|
+
const t0 = performance.now();
|
|
448
|
+
const browser = shared || await chromium.launch({ headless: true });
|
|
449
|
+
// VIDEO (--video dir): Playwright records the viewport; the step record then carries each step's
|
|
450
|
+
// start time and the target's box, so a replay can show what was clicked and when. The video is
|
|
451
|
+
// what the browser showed -- typed values included -- so it is written only when asked for.
|
|
452
|
+
const size = { width: 1280, height: 900 };
|
|
453
|
+
const context = await browser.newContext({ viewport: size,
|
|
454
|
+
...(video ? { recordVideo: { dir: video, size } } : {}),
|
|
455
|
+
...(flow.storageState ? { storageState: flow.storageState } : {}), ...(ctxOpts || {}) });
|
|
456
|
+
if (flow.expectSeen?.length) await context.addInitScript(SEEN_RECORDER);
|
|
457
|
+
const page = await context.newPage();
|
|
458
|
+
const videoAt = video ? Math.round(performance.now() - t0) : null;
|
|
459
|
+
const steps = [], history = [];
|
|
460
|
+
let outcome = "UNKNOWN", error = null, doneRejects = 0, blocked = 0;
|
|
461
|
+
const filled = new Set();
|
|
462
|
+
const keyMap = new Map(); // field label -> spec key, resolved by Jev when no substring matches
|
|
463
|
+
const stuck = new Set(); // fields whose fill threw: never offered for typing again
|
|
464
|
+
const typedKeys = new Map(); // "path|key" -> the field label that key was typed into
|
|
465
|
+
let redactUrl = (u) => u; // declared regex redaction for URLs written to the trace
|
|
466
|
+
// ECHO DETECTION for weak secrets ("admin", "letmein1"): such a value cannot be told from a page
|
|
467
|
+
// word by its shape, but it can by its HISTORY. If the word was on a page before we typed it, it
|
|
468
|
+
// is the page's own word ("Admin" in the nav) and stays; if it appears only after, it is an echo
|
|
469
|
+
// of what we typed and is scrubbed from everything the engine sees. (audit W1)
|
|
470
|
+
const weak = Object.entries(flow.inputs || {}).filter(([k, v]) => isSecretKey(k) && v != null && String(v).length >= 3 && !isStrong(v));
|
|
471
|
+
// Static-ness is per STRING, not per word (audit r7: "summer" was in "Summer sale" before typing,
|
|
472
|
+
// so the echo "Unlock with summer" went out). Every label and title seen before a weak key was
|
|
473
|
+
// typed is the page's own; after typing, a weak word is scrubbed only from strings that are new.
|
|
474
|
+
const weakTyped = new Set(), before = new Set();
|
|
475
|
+
const noteWeak = (snap) => {
|
|
476
|
+
if (weakTyped.size === weak.length && weak.length) return;
|
|
477
|
+
for (const t of [snap.title, ...snap.actions.map((a) => a.label)]) before.add(norm(t).toLowerCase());
|
|
478
|
+
};
|
|
479
|
+
const echoed = () => [...weakTyped];
|
|
480
|
+
const isOwn = (text) => before.has(norm(text).toLowerCase());
|
|
481
|
+
let decideMs = 0, inferMs = 0, cost = 0, truncs = 0, routeRec = null, invalidChoice = null;
|
|
482
|
+
|
|
483
|
+
try {
|
|
484
|
+
await page.goto(base + (flow.start || "/"), { waitUntil: "domcontentloaded" });
|
|
485
|
+
// MAP (v1): with a discovered map, Jev first picks the page where the work begins and code
|
|
486
|
+
// walks there. The step loop below then only makes local choices. See src/route.mjs.
|
|
487
|
+
if (map && routeMode) {
|
|
488
|
+
const { pickTarget, findRoute, walk } = await import("./route.mjs");
|
|
489
|
+
const { pattern } = await import("./discover.mjs");
|
|
490
|
+
const t = await pickTarget(map, redactGoal(flow.goal, flow.inputs), engine, pattern(page.url()), flow.inputs, flow.redact);
|
|
491
|
+
cost += t.cost; decideMs += t.ms;
|
|
492
|
+
routeRec = { target: t.target, pattern: t.pattern, decisions: t.decisions, hops: [] };
|
|
493
|
+
// Only move when the start page is IN the map and a real click route exists. A jump to a
|
|
494
|
+
// page the map cannot reach by links would skip state the flow needs (measured: teleporting
|
|
495
|
+
// off an unmapped start page took the fixture suite from 18/33 to 12/33).
|
|
496
|
+
const here = page.url().replace(/#.*$/, "");
|
|
497
|
+
const known = map.pages.some((p) => p.urls.some((u) => u.replace(/#.*$/, "") === here));
|
|
498
|
+
if (t.target && known && t.target.replace(/#.*$/, "") !== here) {
|
|
499
|
+
const r = findRoute(map, here, t.target);
|
|
500
|
+
if (r) routeRec.hops = routeMode === "goto"
|
|
501
|
+
? (await page.goto(t.target, { waitUntil: "domcontentloaded" }), [{ goto: t.target }])
|
|
502
|
+
: await walk(page, r, t.target);
|
|
503
|
+
else routeRec.unreachable = true;
|
|
504
|
+
}
|
|
505
|
+
routeRec.ms = Math.round(performance.now() - t0);
|
|
506
|
+
}
|
|
507
|
+
for (let i = 0; i < (flow.maxSteps || 14); i++) {
|
|
508
|
+
let snap;
|
|
509
|
+
const at = Math.round(performance.now() - t0);
|
|
510
|
+
await settle(page);
|
|
511
|
+
try { snap = await page.evaluate(`(${SNAPSHOT})()`); }
|
|
512
|
+
catch { await page.waitForTimeout(150); snap = await page.evaluate(`(${SNAPSHOT})()`); }
|
|
513
|
+
// FIX (2026-09-21): check the goal on EVERY snapshot, not only when the model claims DONE.
|
|
514
|
+
// Measured on create-campaign: the campaign WAS created at step 17, the model never noticed,
|
|
515
|
+
// and it then clicked away to /team and destroyed its own result -- reported as MAX_STEPS.
|
|
516
|
+
// The assertion is deterministic and costs ~2ms, so there is no reason to wait to be told.
|
|
517
|
+
// Checked here, before deciding, rather than after acting: checking post-action races with
|
|
518
|
+
// the navigation the action just started, and can pass on a URL that is already going away.
|
|
519
|
+
// DECLARED REDACTION, AT THE SOURCE (src/redact.mjs): before anything is built from the snapshot.
|
|
520
|
+
if (flow.redact?.length) {
|
|
521
|
+
const r = await buildRedactor(page, flow.redact);
|
|
522
|
+
redactSnapshot(snap, r);
|
|
523
|
+
redactUrl = r.url;
|
|
524
|
+
}
|
|
525
|
+
noteWeak(snap);
|
|
526
|
+
// A weak value that is exactly a label on THIS page ("Admin") names that element when the goal
|
|
527
|
+
// says it ("open the Admin page"), so the goal keeps the word here (audit r8, weak-later-page).
|
|
528
|
+
const labelsNow = new Set(snap.actions.map((a) => norm(a.label).toLowerCase()));
|
|
529
|
+
const keepWeak = weak.filter(([, v]) => labelsNow.has(String(v).toLowerCase())).map(([k]) => k);
|
|
530
|
+
// Scrub weak-secret ECHOES at the source: only in strings the page did not have before typing.
|
|
531
|
+
if (echoed().length) {
|
|
532
|
+
const w = { weakKeys: echoed() };
|
|
533
|
+
// A string that IS exactly the weak word ("Admin") is the page's word, not an echo of what we
|
|
534
|
+
// typed (audit r8: the dashboard's "Admin" link reached the model as <password>). A weak
|
|
535
|
+
// secret is guessable by definition; navigation must not break over it.
|
|
536
|
+
const exact = new Set(weak.map(([, v]) => String(v).toLowerCase()));
|
|
537
|
+
const fix = (t) => (t == null || isOwn(t) || exact.has(norm(t).toLowerCase()) ? t : scrubText(t, flow.inputs, w));
|
|
538
|
+
snap.title = fix(snap.title);
|
|
539
|
+
for (const a of snap.actions) { a.label = fix(a.label); if (a.optionLabel) a.optionLabel = fix(a.optionLabel); }
|
|
540
|
+
}
|
|
541
|
+
if ((i > 0 || routeRec?.hops?.length) && await checkGoal(page, flow, snap)) { outcome = "DONE_VERIFIED"; break; }
|
|
542
|
+
|
|
543
|
+
// FIX (v1, TicketBay D2): a field whose label contains no spec key (a rename: "Email" ->
|
|
544
|
+
// "E-mail") used to be invisible to the test. Jev now picks WHICH spec key belongs in it --
|
|
545
|
+
// a typed choice over the keys, so the text still comes only from the spec. Once per label.
|
|
546
|
+
if (flow.inputs) await resolveKeys(snap, flow.inputs, keyMap, engine, (c) => { cost += c; }, flow.redact);
|
|
547
|
+
const keyOf = (label, role) => stuckLabels.has(label) ? null : keyFor(label, flow.inputs, role) ?? keyMap.get(label) ?? null;
|
|
548
|
+
const here = new URL(snap.url).pathname;
|
|
549
|
+
const stuckLabels = new Set([...stuck].filter((k) => k.startsWith(here + "|")).map((k) => k.split("|").slice(2).join("|")));
|
|
550
|
+
let space = compact(snap, budget, filled, flow.inputs, 3, keyOf, flow.goal, typedKeys);
|
|
551
|
+
// FIX (2026-09-21): while a modal listbox/menu is open, its options are the ONLY legal
|
|
552
|
+
// moves -- which is why the snapshot can see nothing else. Offering WAIT/DONE/BLOCKED here
|
|
553
|
+
// makes the model take the escape hatch: measured DONE@0.90, and BLOCKED@0.9997 when DONE
|
|
554
|
+
// was removed. Removing the bad options (rather than instructing against them) gives
|
|
555
|
+
// CLICK@1.00. Same principle as finding #2 in the README.
|
|
556
|
+
const listboxOpen = space.all.some((r) => r.a.popup);
|
|
557
|
+
// The history shown to the model must agree with the page: a field offered for typing again
|
|
558
|
+
// (it was emptied by a reload) is NOT "typed". Measured on the D1 repro: with the stale
|
|
559
|
+
// "typed Email" line the model answered BLOCKED on a page with two empty required fields.
|
|
560
|
+
const shownHistory = () => {
|
|
561
|
+
const retype = new Set(Object.values(space.targets.TYPE_TEXT || {}).map((a) => `typed ${a.label}`));
|
|
562
|
+
return history.filter((h) => !retype.has(h));
|
|
563
|
+
};
|
|
564
|
+
// TOURNAMENT (v1). When the page has more candidates than one decision can hold (the budget,
|
|
565
|
+
// then the 52-letter ceiling), the old loop DROPPED the rest -- fixtures/table.html's "Next
|
|
566
|
+
// page" is candidate #63 and was never offered (0/3). Now nothing is dropped: the candidates
|
|
567
|
+
// are split into heats that each fit, every heat is decided in parallel with a "none of
|
|
568
|
+
// these" option, and the final decision is between the heat winners. Pages that fit in one
|
|
569
|
+
// decision never get here, so their behaviour is unchanged.
|
|
570
|
+
let heats = null;
|
|
571
|
+
if (space.dropped > 0) {
|
|
572
|
+
const CAP = LETTER_CEILING - 4, chunks = [];
|
|
573
|
+
let cur = [], n = 0;
|
|
574
|
+
for (const r of space.all) {
|
|
575
|
+
if (cur.length && n + r.ops.length > CAP) { chunks.push(cur); cur = []; n = 0; }
|
|
576
|
+
cur.push(r); n += r.ops.length;
|
|
577
|
+
}
|
|
578
|
+
if (cur.length) chunks.push(cur);
|
|
579
|
+
const goal = redactGoal(flow.goal, flow.inputs, keepWeak);
|
|
580
|
+
const results = await Promise.all(chunks.map(async (rows) => {
|
|
581
|
+
const sub = space.render(rows);
|
|
582
|
+
sub.criteria.NONE = "none of these options advances the goal";
|
|
583
|
+
const d = await decide({ ...sub, values: space.values }, snap, goal, shownHistory(), engine, flow.inputs, [], flow.redact);
|
|
584
|
+
return { rows, sub, d };
|
|
585
|
+
}));
|
|
586
|
+
const winners = [];
|
|
587
|
+
for (const { rows, sub, d } of results) {
|
|
588
|
+
cost += d.cost; decideMs += d.ms;
|
|
589
|
+
if (d.choice === "NONE" || !(d.choice in sub.criteria)) continue;
|
|
590
|
+
const [o, ix] = String(d.choice).split(":");
|
|
591
|
+
const row = rows.find((r) => r.a === sub.targets[o]?.[ix]);
|
|
592
|
+
if (row && !winners.includes(row)) winners.push(row);
|
|
593
|
+
}
|
|
594
|
+
winners.sort((x, y) => x.i - y.i);
|
|
595
|
+
space = { ...space, ...space.render(winners) };
|
|
596
|
+
heats = { heats: chunks.length, winners: winners.length };
|
|
597
|
+
// The final exists to COMPARE winners. With exactly one there is nothing to compare: the heat
|
|
598
|
+
// already decided it over a "none of these" option, so the final round trip is skipped.
|
|
599
|
+
// (bake-off: two sequential rounds cost +335 ms on the 60-row list and changed no outcome.)
|
|
600
|
+
if (winners.length === 1) {
|
|
601
|
+
const only = results.find(({ rows }) => rows.includes(winners[0])).d;
|
|
602
|
+
const key = Object.keys(space.criteria).find((k) => k.startsWith(String(only.choice).split(":")[0] + ":"));
|
|
603
|
+
heats.decided = key ? { choice: key, confidence: only.confidence, ms: 0, cost: 0, tokens: 0 } : null;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
if (!listboxOpen) {
|
|
607
|
+
space.criteria.WAIT = "wait for the page";
|
|
608
|
+
// No DONE option. The goal is checked deterministically on EVERY snapshot before this decision,
|
|
609
|
+
// so when the model is asked, "done" is false by construction; offering it only created a
|
|
610
|
+
// failure mode (Jev 27/30 from false DONEs in the spike; the local engine answered DONE four
|
|
611
|
+
// times in a row on late-enable.html and card-year.html). Take the bad option away.
|
|
612
|
+
// BLOCKED ("nothing can progress") is false by definition while a value from the spec is still
|
|
613
|
+
// waiting to be typed or chosen on this page -- that IS progress -- so it is offered only when
|
|
614
|
+
// no such action exists.
|
|
615
|
+
const specWork = Object.keys(space.criteria).some((k) => k.startsWith("TYPE_TEXT:") || k.startsWith("SELECT:"));
|
|
616
|
+
if (!specWork) space.criteria.BLOCKED = "nothing can progress";
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const d = heats?.decided ? { ...heats.decided, truncated: false }
|
|
620
|
+
: await decide(space, snap, redactGoal(flow.goal, flow.inputs, keepWeak), shownHistory(), engine, flow.inputs, [], flow.redact);
|
|
621
|
+
if (heats) delete heats.decided;
|
|
622
|
+
// The engine must return one of OUR keys. Anything else is treated as BLOCKED, never acted on.
|
|
623
|
+
if (!(d.choice in space.criteria)) { invalidChoice = d.choice; d.choice = "BLOCKED"; }
|
|
624
|
+
decideMs += d.ms; inferMs += d.inferMs || 0; cost += d.cost;
|
|
625
|
+
// Loud, because a silent truncation means the model never saw options we believe we offered.
|
|
626
|
+
if (d.truncated) { truncs++; console.warn(`[truncated] ${flow.name} step ${i + 1}: the engine `
|
|
627
|
+
+ `discarded options past the ${LETTER_CEILING}-letter ceiling -- the model did not see them`); }
|
|
628
|
+
|
|
629
|
+
let op = d.choice, tIdx = null, action = null;
|
|
630
|
+
if (!["WAIT", "DONE", "BLOCKED"].includes(op)) {
|
|
631
|
+
const [o, ix] = String(d.choice).split(":");
|
|
632
|
+
op = o; tIdx = ix; action = space.targets[o]?.[ix];
|
|
633
|
+
if (!action) op = "BLOCKED";
|
|
634
|
+
}
|
|
635
|
+
// CODEGEN (2026-09-22): `role` and `tag` added. act() already resolves by
|
|
636
|
+
// getByRole(ARIA[action.role], {name: action.label}) at line 232 -- but the step record
|
|
637
|
+
// kept only the label, so a trace could not reproduce the locator act() itself used.
|
|
638
|
+
// `tag` distinguishes a native <select> combobox from a <button role=combobox> trigger,
|
|
639
|
+
// which need different assertions (toHaveValue vs toContainText).
|
|
640
|
+
const step = { n: i + 1, at, decidedAt: Math.round(performance.now() - t0), url: snap.url, op, target: tIdx, label: action?.label ?? null,
|
|
641
|
+
role: action?.role ?? null, tag: action?.tag ?? null,
|
|
642
|
+
confidence: d.confidence, ms: Math.round(d.ms), inferMs: d.inferMs, tokens: d.tokens,
|
|
643
|
+
options: Object.keys(space.criteria).length, dropped: space.dropped, truncated: d.truncated,
|
|
644
|
+
...(heats ? { tournament: heats } : {}) };
|
|
645
|
+
|
|
646
|
+
if (op === "DONE") {
|
|
647
|
+
await page.waitForFunction(READY, null, { timeout: 2000, polling: 50 }).catch(() => {});
|
|
648
|
+
if (await checkGoal(page, flow)) { steps.push(step); outcome = "DONE_VERIFIED"; break; }
|
|
649
|
+
if (doneRejects < 3) { doneRejects++; step.doneRejected = true; steps.push(step); await page.waitForTimeout(200); continue; }
|
|
650
|
+
steps.push(step); outcome = "DONE_REJECTED"; break;
|
|
651
|
+
}
|
|
652
|
+
if (op === "BLOCKED") {
|
|
653
|
+
// "Nothing can progress" is only believable on a STABLE page. Wait up to 3 s for the page to
|
|
654
|
+
// change (an async check finishing, a button enabling); a BLOCKED counts toward giving up only
|
|
655
|
+
// when nothing changed. (fixtures/late-enable.html: three BLOCKEDs in ~1.5 s during a 1.8 s
|
|
656
|
+
// availability check, then MODEL_BLOCKED, although the submit enabled a moment later.)
|
|
657
|
+
const changed = await page.evaluate((cap) => new Promise((resolve) => {
|
|
658
|
+
const obs = new MutationObserver(() => { obs.disconnect(); clearTimeout(t); resolve(true); });
|
|
659
|
+
obs.observe(document.documentElement, { subtree: true, childList: true, attributes: true, characterData: true });
|
|
660
|
+
const t = setTimeout(() => { obs.disconnect(); resolve(false); }, cap);
|
|
661
|
+
}), 3000).catch(() => true);
|
|
662
|
+
step.retried = true; steps.push(step);
|
|
663
|
+
if (changed) continue;
|
|
664
|
+
if (++blocked < 3) continue;
|
|
665
|
+
outcome = "MODEL_BLOCKED"; break;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
const before = page.url();
|
|
669
|
+
if (video && action) {
|
|
670
|
+
const b = await page.locator(`[data-jev-node="${op === "SELECT" ? action.selectNode : action.node}"]`).first()
|
|
671
|
+
.boundingBox({ timeout: 300 }).catch(() => null);
|
|
672
|
+
if (b) step.box = { x: Math.round(b.x), y: Math.round(b.y), w: Math.round(b.width), h: Math.round(b.height) };
|
|
673
|
+
}
|
|
674
|
+
if (op === "TYPE_TEXT") {
|
|
675
|
+
const v = specValue(flow, action, keyMap);
|
|
676
|
+
if (v === null) { step.noSpecValue = true; steps.push(step); outcome = "NO_SPEC_VALUE"; break; }
|
|
677
|
+
step.inputKey = specKey(flow, action, keyMap); // CODEGEN: the key, never the value
|
|
678
|
+
try { step.via = await act(page, action, (loc, t) => loc.fill(v, { timeout: t })); }
|
|
679
|
+
catch {
|
|
680
|
+
// A field that cannot take its value is dropped, not retried: a mapped key is unmapped,
|
|
681
|
+
// and the field is not offered for typing again in this run.
|
|
682
|
+
step.stale = true; steps.push(step); history.push(`stale ${action.label}`);
|
|
683
|
+
if (keyMap.get(action.label)) keyMap.set(action.label, null);
|
|
684
|
+
stuck.add(fillKey(snap.url, action)); continue;
|
|
685
|
+
}
|
|
686
|
+
filled.add(fillKey(snap.url, action));
|
|
687
|
+
if (step.inputKey) typedKeys.set(`${new URL(snap.url).pathname}|${step.inputKey}`, action.label);
|
|
688
|
+
if (step.inputKey && weak.some(([k]) => k === step.inputKey)) weakTyped.add(step.inputKey);
|
|
689
|
+
history.push(`typed ${action.label}`);
|
|
690
|
+
} else if (op === "SELECT") {
|
|
691
|
+
try { step.via = await act(page, { ...action, node: action.selectNode, label: action.selectLabel, role: "combobox" },
|
|
692
|
+
(loc, t) => loc.selectOption({ label: action.optionLabel }, { timeout: t })); }
|
|
693
|
+
catch { step.stale = true; steps.push(step); history.push(`stale ${action.label}`); continue; }
|
|
694
|
+
step.option = action.optionLabel; step.control = action.selectLabel;
|
|
695
|
+
history.push(`selected ${action.label}`);
|
|
696
|
+
} else if (op === "CLICK") {
|
|
697
|
+
try { step.via = await act(page, action, (loc, t) => loc.click({ timeout: t })); }
|
|
698
|
+
catch { step.stale = true; steps.push(step); history.push(`stale ${action.label}`); continue; }
|
|
699
|
+
// FIX (2026-09-21): a link click starts a navigation that has not begun yet when we
|
|
700
|
+
// re-snapshot. READY is true on the OLD page and settle() sees a DOM that is quiet only
|
|
701
|
+
// because nothing has happened, so the loop re-reads the old page and clicks the same link
|
|
702
|
+
// again. Measured after links were promoted to rank 1: "New campaign" clicked TEN times,
|
|
703
|
+
// burning 10 of 22 steps and failing create-campaign 0/5. Only links pay this wait.
|
|
704
|
+
if (action.role === "link") {
|
|
705
|
+
await page.waitForURL((u) => u.toString() !== before, { timeout: 1200 }).catch(() => {});
|
|
706
|
+
}
|
|
707
|
+
history.push(`clicked ${action.label}`);
|
|
708
|
+
} else {
|
|
709
|
+
await Promise.race([
|
|
710
|
+
page.waitForURL((u) => u.toString() !== before, { timeout: 2500 }),
|
|
711
|
+
page.waitForFunction(READY, null, { timeout: 2500, polling: 50 }),
|
|
712
|
+
]).catch(() => {});
|
|
713
|
+
history.push("waited");
|
|
714
|
+
}
|
|
715
|
+
if (op !== "TYPE_TEXT") await page.waitForFunction(READY, null, { timeout: 4000, polling: 50 }).catch(() => {});
|
|
716
|
+
step.wallAt = Math.round(performance.now() - t0);
|
|
717
|
+
steps.push(step);
|
|
718
|
+
|
|
719
|
+
}
|
|
720
|
+
if (outcome === "UNKNOWN") outcome = "MAX_STEPS";
|
|
721
|
+
} catch (e) { error = String(e.message || e).slice(0, 180); outcome = "ERROR"; }
|
|
722
|
+
|
|
723
|
+
const tLoopEnd = performance.now() - t0;
|
|
724
|
+
const passed = await checkGoal(page, flow);
|
|
725
|
+
const tCheck = performance.now() - t0;
|
|
726
|
+
const finalUrl = page.url();
|
|
727
|
+
const videoFile = video ? await page.video()?.path() : null;
|
|
728
|
+
await context.close();
|
|
729
|
+
if (!shared) await browser.close();
|
|
730
|
+
const rec = { flow: flow.name, base, engine, budget, outcome, passed, finalUrl, error, steps,
|
|
731
|
+
...(routeRec ? { route: routeRec } : {}), ...(invalidChoice ? { invalidChoice } : {}),
|
|
732
|
+
decideMs: Math.round(decideMs), inferMs: Math.round(inferMs), cost, truncations: truncs,
|
|
733
|
+
doneRejects, wallMs: Math.round(performance.now() - t0),
|
|
734
|
+
...(video ? { video: { file: videoFile, at: videoAt, size } } : {}),
|
|
735
|
+
tLoopEnd: Math.round(tLoopEnd), tCheck: Math.round(tCheck) };
|
|
736
|
+
// SECURITY (audit): the trace on disk is scrubbed too -- a GET form puts a password in finalUrl.
|
|
737
|
+
rec.weakEchoed = echoed();
|
|
738
|
+
// Labels in the record were scrubbed at the source; a weak value can still sit in a URL's QUERY
|
|
739
|
+
// (a GET form), so queries get the weak scrub too -- paths stay intact for codegen's toHaveURL.
|
|
740
|
+
const q = (u) => { if (!u || !rec.weakEchoed.length) return u; const i = u.indexOf("?");
|
|
741
|
+
return i < 0 ? u : u.slice(0, i) + scrubText(u.slice(i), flow.inputs, { weakKeys: rec.weakEchoed }); };
|
|
742
|
+
rec.finalUrl = redactUrl(q(rec.finalUrl));
|
|
743
|
+
for (const st of rec.steps) { st.url = redactUrl(q(st.url)); if (st.label) st.label = String(st.label); }
|
|
744
|
+
if (trace) fs.writeFileSync(trace, JSON.stringify(scrub(rec, flow.inputs), null, 2));
|
|
745
|
+
return rec;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
export { compact, redactGoal, stateOk, chromium, SNAPSHOT, SEEN_TEXT, settle, act, keyFor, norm, checkGoal, ARIA, LETTER_CEILING };
|