scenescout 3.9.0 → 3.11.0

package/dist/engine/oracles.js

@@ -1,4 +1,13 @@
 import { redactSecrets } from "./memory.js";
+/** How an HTTP error violation states its request. One formatter, so a reader of the detail (the check) cannot drift from its writer. */
+export function httpErrorDetail(method, url, status) {
+    return `${method} ${url.slice(0, 200)} → HTTP ${status}`;
+}
+/** The status an http_error detail names, or null when the detail is not one. */
+export function httpStatusOf(detail) {
+    const m = /→ HTTP (\d{3})$/.exec(detail.replace(/ \[\d+ secrets? redacted\]$/, ""));
+    return m ? Number(m[1]) : null;
+}
 /** URLs whose failures are noise, not findings (favicons, source maps). */
 const BENIGN_URL_RE = /favicon|\.map($|\?)/i;
 /**
@@ -35,6 +44,68 @@ export function isPolicyInduced(v, msSincePolicyBlock) {
         return false;
     return /ERR_BLOCKED_BY_CLIENT|Failed to fetch|NetworkError when attempting to fetch|Load failed|the server responded with a status of 403\b/i.test(v.detail);
 }
+/**
+ * The request a console message is the browser's own echo of, or null when it
+ * is not one.
+ *
+ * Chromium and WebKit print "Failed to load resource: …" for a subresource
+ * that answered with an error status or got no answer, and give the
+ * resource's address as the message's location (Firefox prints nothing; see
+ * `echoesFailedLoads` in browsers.ts). The line carries no address in its
+ * text, so without this it could only be charged to the page. Returned in the
+ * form `requestKey` gives a request's URL, so the two can be matched.
+ */
+export function failedLoadEchoOf(text, locationUrl, pageUrl) {
+    if (!/^Failed to load resource: /.test(text) || !locationUrl)
+        return null;
+    try {
+        return requestKey(new URL(locationUrl, pageUrl).href);
+    }
+    catch {
+        return null;
+    }
+}
+/** A request's address as the echo lookup keys it: absolute, and without its fragment, which is never sent. */
+export function requestKey(url) {
+    try {
+        const u = new URL(url);
+        u.hash = "";
+        return u.href;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Which recent requests an embed sent, by address, so the browser's echo of
+ * one that failed is charged where the request was.
+ *
+ * Only addresses an embed requested are kept, and the latest request to an
+ * address decides: when the app then requests the same address, its echo is
+ * the app's again. Bounded, dropping the oldest, so a long run does not hold
+ * every address it saw.
+ */
+export class EmbedRequestLog {
+    cap;
+    byKey = new Map();
+    constructor(cap = 500) {
+        this.cap = cap;
+    }
+    note(url, embed) {
+        const key = requestKey(url);
+        this.byKey.delete(key);
+        if (!embed)
+            return;
+        this.byKey.set(key, embed);
+        if (this.byKey.size > this.cap)
+            this.byKey.delete(this.byKey.keys().next().value);
+    }
+    /** The embed whose request a console message echoes, or null when it echoes none of theirs. */
+    embedOfEcho(text, locationUrl, pageUrl) {
+        const key = failedLoadEchoOf(text, locationUrl, pageUrl);
+        return key === null ? null : (this.byKey.get(key) ?? null);
+    }
+}
 /**
  * Invariant oracles: passive listeners that record violations regardless of
  * what the agent is doing. The engine drains the buffer after every action and
@@ -58,8 +129,12 @@ export class OracleMonitor {
                 severity: "high",
                 detail: text.slice(0, 500),
                 url: page.url(),
+                // The one console line that names its request: the browser's echo of an embed's failed load is the embed's too.
+                embed: this.embedRequests.embedOfEcho(text, msg.location().url, page.url()) ?? undefined,
             });
         });
+        // Noted as each request starts, which is always before the browser can echo its failure.
+        page.on("request", (req) => this.embedRequests.note(req.url(), this.embedOfRequest(req)));
         page.on("pageerror", (err) => {
             this.record({
                 kind: "page_error",
@@ -105,7 +180,7 @@ export class OracleMonitor {
             this.record({
                 kind: "http_error",
                 severity: status >= 500 ? "high" : "medium",
-                detail: `${res.request().method()} ${res.url().slice(0, 200)} → HTTP ${status}`,
+                detail: httpErrorDetail(res.request().method(), res.url(), status),
                 url: page.url(),
                 embed: this.embedOfRequest(res.request()) ?? undefined,
             });
@@ -131,12 +206,14 @@ export class OracleMonitor {
      */
     policyAttributed = 0;
     embedOfRequest = () => null;
+    embedRequests = new EmbedRequestLog();
     /**
      * The engine knows which frame a request came from; a failing request is
      * attributed to an embed through this. Console and page errors are not
      * attributed: a console message says where its script was served from, not
      * which frame ran it, so an SDK the app's page loads from the embed's own
-     * site would be taken for the embed.
+     * site would be taken for the embed. The exception is the browser's echo of
+     * a failed load, which names its request and goes where the request went.
      */
     setEmbedAttribution(ofRequest) {
         this.embedOfRequest = ofRequest;

package/dist/engine/pace.js

@@ -1,10 +1,23 @@
+import { FORMS_READ_FAILED, FORMS_SUBMIT_UNMATCHED } from "./forms.js";
 /**
  * Log entries that are not actions: a stated task, an attach, the note that a
  * resource was created. They carry no browser work and take no time, so
  * counting them inflates the action count and drags the median gap toward
  * zero. One real run logged 325 entries of which 107 were stated tasks.
  */
-const MARKER_ACTIONS = new Set(["task", "attach", "close", "lane-report", "created-resource", "journey:start", "journey:end", "record:full", "record:failed"]);
+const MARKER_ACTIONS = new Set([
+    "task",
+    "attach",
+    "close",
+    "lane-report",
+    "created-resource",
+    "journey:start",
+    "journey:end",
+    "record:full",
+    "record:failed",
+    FORMS_READ_FAILED,
+    FORMS_SUBMIT_UNMATCHED,
+]);
 /** Whether a log entry represents work the browser actually did. */
 export function isActing(action) {
     return !MARKER_ACTIONS.has(action);

package/dist/engine/verify.js

@@ -23,6 +23,7 @@
  * Pure, so the ordering and the wording can be table-tested.
  */
 import { normalizePath } from "./fingerprint.js";
+import { failingSignatures } from "./memory.js";
 /** What a re-test found. */
 export const VERDICTS = ["gone", "present", "changed"];
 export const VERDICT_MEANING = {
@@ -140,3 +141,121 @@ export function sayVerification(f) {
         return ` · re-tested ${on}, behaviour has changed since it was filed`;
     return ` · re-tested ${on}`;
 }
+// ---------------------------------------------------------------------------
+// Re-testing by page load, for `scenescout check`.
+//
+// The campaign above hands findings to an agent, who re-walks them with
+// judgement. A check has none, so it re-tests only the findings whose
+// reproduction is mechanical: the evidence names a GET that failed with a
+// status, and the finding's repro shows nothing done on its page but looking
+// at it. Loading that page again and watching the same request is then the
+// whole reproduction, and its answer is a fact rather than an opinion.
+// ---------------------------------------------------------------------------
+/**
+ * Actions that only look at a page. A repro whose steps on the finding's page
+ * are all of these was reproduced by loading the page; any other step (a
+ * click, a typed value, a replayed request, anything this list does not know)
+ * means loading it is not the reproduction.
+ */
+const LOOK_ONLY = new Set([
+    "attach",
+    "navigate",
+    "plan:navigate",
+    "crawl",
+    "back",
+    "snapshot",
+    "screenshot",
+    "design-audit",
+    "task",
+    "journey:start",
+    "journey:end",
+    "record:full",
+]);
+/** How the finding's page is reached again, or null when a check cannot re-test it deterministically. */
+export function loadRetest(item) {
+    const signatures = [...failingSignatures(item.evidence)];
+    // A POST that failed is not re-sent by loading a page, and a check would refuse to send it anyway.
+    if (signatures.length === 0 || signatures.some((s) => !s.startsWith("GET ")))
+        return null;
+    // The first repro line is how the page was reached; every later one happened on it.
+    if (item.repro.length === 0)
+        return null;
+    for (const line of item.repro.slice(1)) {
+        if (!LOOK_ONLY.has(line.split(/\s/, 1)[0]))
+            return null;
+    }
+    let url;
+    try {
+        url = new URL(item.url);
+    }
+    catch {
+        return null;
+    }
+    // A secret was redacted out of the stored URL: loading what is left is a different page.
+    if (/redacted/i.test(item.url))
+        return null;
+    return { item, path: `${url.pathname}${url.search}`, signatures };
+}
+export const RETEST_VERDICTS = ["reproduces", "possibly-fixed", "not-reached"];
+/**
+ * Each re-testable finding against the page load that re-tests it.
+ *
+ * `reproduces` when the load saw one of the finding's failing requests fail
+ * with the same status; `possibly-fixed` when the page loaded and none did —
+ * possibly, because the page may simply not have asked this time; and
+ * `not-reached` when the page did not load or bounced to sign-in, which says
+ * nothing about the finding either way.
+ */
+export function retestResults(candidates, pages) {
+    return candidates.map(({ item, path, signatures }) => {
+        const base = { id: item.id, severity: item.severity, title: item.title, path, signatures };
+        const page = pages.find((p) => p.path === path);
+        if (!page)
+            return { ...base, verdict: "not-reached", note: "the page was not visited" };
+        if (page.loadError !== undefined)
+            return { ...base, verdict: "not-reached", note: "the page did not load" };
+        if (page.loginRedirect)
+            return { ...base, verdict: "not-reached", note: "the page sent the browser to sign-in" };
+        const seen = new Set();
+        for (const detail of page.httpErrors)
+            for (const s of failingSignatures(detail))
+                seen.add(s);
+        // The document's own failure is left out of the violations (withoutOwnResponse), so it is added back here.
+        if (page.status !== null && page.status >= 400)
+            for (const s of failingSignatures(`GET ${page.path.split("?")[0]} ${page.status}`))
+                seen.add(s);
+        return { ...base, verdict: signatures.some((s) => seen.has(s)) ? "reproduces" : "possibly-fixed" };
+    });
+}
+/**
+ * The findings a check re-tests: the open ones it can reproduce by loading a
+ * page, in the campaign's order and within its cap, plus how many open
+ * findings there are in all, so the report can say how many it left to a run.
+ */
+export function checkRetestPlan(findings) {
+    const open = findings.filter((f) => (f.status ?? "open") !== "resolved");
+    const eligible = open.filter((f) => loadRetest(item(f)) !== null);
+    const candidates = verifyWorklist(eligible)
+        .map(loadRetest)
+        .filter((c) => c !== null);
+    return { candidates, open: open.length };
+}
+/**
+ * The entries of a memory file's findings list a re-test can read. The file is
+ * JSON someone may have edited; an entry missing the fields the rules read is
+ * left out rather than failing the check, which only ever reports re-tests.
+ */
+export function wellFormedFindings(list) {
+    return list.filter((f) => {
+        if (typeof f !== "object" || f === null)
+            return false;
+        const x = f;
+        return (typeof x.id === "string" &&
+            typeof x.url === "string" &&
+            typeof x.title === "string" &&
+            typeof x.severity === "string" &&
+            (x.evidence === undefined || typeof x.evidence === "string") &&
+            Array.isArray(x.repro) &&
+            x.repro.every((r) => typeof r === "string"));
+    });
+}

package/dist/mcp-server.js

@@ -35,12 +35,13 @@ import { z } from "zod";
 import { BrowserEngine } from "./engine/browser.js";
 import { reapOrphanBrowsers } from "./engine/reaper.js";
 import { FINDING_CATEGORIES, MemoryStore, redactSecrets } from "./engine/memory.js";
-import { LANE_NAME_MAX, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
+import { decodedEntitiesNote, LANE_NAME_MAX, LaneLedger, laneCloseGuard, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
 import { MAX_UNFILED_NAMED, unfiledDefects } from "./engine/calibration.js";
 import { SessionQueue, withWatchdog } from "./engine/dispatch.js";
 import { FIXTURE_KINDS } from "./engine/fixtures.js";
 import { feedForSession, LIVE_ENV, writeStatusFile, LIVE_TOKEN_FILE, liveEngines, liveTokenFileName, pidAlive, statusFileName, LiveServer, StatusBoard, } from "./engine/live.js";
 import { formatBriefs, MAX_LANES, planLanes } from "./engine/brief.js";
+import { formatNeverSubmittedEmpty } from "./engine/forms.js";
 import { computeGaps, formatRouteCoverage, formatUnchosenOptions, generateReport, replayDocument, reportEvidence } from "./engine/report.js";
 import { describeVerdict, formatWorklist, unknownIds, VERDICTS, verifyWorklist } from "./engine/verify.js";
 import { RECORD_MAX_FRAMES, resolveFrame } from "./engine/replay.js";
@@ -500,6 +501,9 @@ server.server.setRequestHandler(GetPromptRequestSchema, (request) => {
 // Splitting the app between lanes: the other half of the parallel protocol.
 // scout_lane_report is how a lane hands its answers back; this is what the
 // planner hands it in the first place.
+// Which sessions those two tools have named as lanes, and whose report was
+// folded, so scout_close can refuse to drop a lane's decisions.
+const laneLedger = new LaneLedger();
 server.registerTool("scout_lane_brief", {
     description: "For a run split across parallel agents (lanes). Divides the app's known routes between N lanes and returns each lane's session name, the `objective` to attach it with, and the routes it owns — whole modules per lane, balanced by route count, so no two lanes audit the same area and none is left unopened. Call it after the first crawl, when route knowledge is complete. Touches no browser; pass the briefs to your lane agents, then use scout_lane_report for what they hand back.",
     inputSchema: {
@@ -512,7 +516,9 @@ server.registerTool("scout_lane_brief", {
     try {
         const eng = engineFor(session);
         const all = routes && routes.length > 0 ? routes : eng.allKnownRoutes();
-        return text(formatBriefs(planLanes(all, lanes, { goal, mode: eng.mode, role: eng.role }), { goal, mode: eng.mode, role: eng.role }), session);
+        const briefs = planLanes(all, lanes, { goal, mode: eng.mode, role: eng.role });
+        laneLedger.nameBriefed(briefs.map((b) => b.lane), (s) => engines.has(s));
+        return text(formatBriefs(briefs, { goal, mode: eng.mode, role: eng.role }), session);
     }
     catch (err) {
         return errorText(err);
@@ -527,10 +533,13 @@ server.registerTool("scout_lane_report", {
     },
 }, async ({ lane, reply }) => {
     try {
-        if (reply === undefined)
+        if (reply === undefined) {
+            laneLedger.name(lane);
             return { content: [{ type: "text", text: laneReportInstruction(lane) }] };
+        }
         const parsed = parseLaneReport(reply, lane);
         if (!parsed.ok) {
+            laneLedger.refuse(lane, engines.get(lane)?.attached === true);
             return {
                 content: [
                     { type: "text", text: `Lane report REFUSED: ${parsed.reason}. Ask the lane once for the corrected object; do not re-judge its prose.` },
@@ -575,8 +584,12 @@ server.registerTool("scout_lane_report", {
                 (unfiled.length > MAX_UNFILED_NAMED ? `\n  … +${unfiled.length - MAX_UNFILED_NAMED} more` : "") +
                 `\nFile each with scout_finding (the same evidence), or confirm which finding already covers it, before closing the lane's session. A judged defect that is never filed is not in the report.`
             : "";
+        laneLedger.fold(lane, engines.get(lane)?.attached === true);
         const around = parsed.aroundIgnored ? `\n(The text around the report's JSON block was discarded unread.)` : "";
-        return { content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${followUp}` }] };
+        const decoded = decodedEntitiesNote(parsed.entitiesDecoded);
+        return {
+            content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${decoded}${followUp}` }],
+        };
     }
     catch (err) {
         return errorText(err);
@@ -836,7 +849,7 @@ server.registerTool("scout_crawl", {
     }
 }, 600_000));
 server.registerTool("scout_run_plan", {
-    description: "Execute up to 20 actions in ONE call — use for mechanical sequences (fill a form, walk a wizard) so each step doesn't cost a round-trip. Targets resolve at execution time by semantic locator: 'testid=…', 'text=…', or 'label=…' (never snapshot refs). An `upload` step attaches a file as scout_upload does (target required — the file input or the control that opens its chooser; value = a fixture kind or a project-relative path). The plan ABORTS at the first NEW oracle violation, policy refusal, or failed step, returning a transcript of how far it got; repeats of already-reported violations do not abort (they stay logged for the report).",
+    description: "Execute up to 20 actions in ONE call — use for mechanical sequences (fill a form, walk a wizard) so each step doesn't cost a round-trip. Targets resolve at execution time by semantic locator: 'testid=…', 'text=…', 'label=…' or 'role=button[name=Save]' (never snapshot refs). The same steps, saved to .scenescout/flows/<name>.json with expect-text / expect-url / expect-request steps added, are replayed by `scenescout check` on every pull request. An `upload` step attaches a file as scout_upload does (target required — the file input or the control that opens its chooser; value = a fixture kind or a project-relative path). The plan ABORTS at the first NEW oracle violation, policy refusal, or failed step, returning a transcript of how far it got; repeats of already-reported violations do not abort (they stay logged for the report).",
     inputSchema: {
         steps: z
             .array(z.object({
@@ -844,7 +857,7 @@ server.registerTool("scout_run_plan", {
             target: z
                 .string()
                 .optional()
-                .describe("testid=…, text=…, label=… (or a path for navigate; 'top'/'bottom'/±px for scroll; for upload: the file input or the control that opens its chooser)"),
+                .describe("testid=…, text=…, label=…, role=<role>[name=\"…\"] (or a path for navigate; 'top'/'bottom'/±px for scroll; for upload: the file input or the control that opens its chooser)"),
             value: z
                 .string()
                 .optional()
@@ -1162,7 +1175,7 @@ server.registerTool("scout_finding", {
     }
 }));
 server.registerTool("scout_coverage", {
-    description: "Show exploration coverage: states visited across all runs, which elements remain unexercised, and which options of a dropdown used this run no session has chosen yet. Use to decide where to explore next and when the level's budget is satisfied.",
+    description: "Show exploration coverage: states visited across all runs, which elements remain unexercised, which options of a dropdown used this run no session has chosen yet, and which forms seen this run no session has submitted with every text field blank. Use to decide where to explore next and when the level's budget is satisfied.",
     inputSchema: { session: sessionParam },
 }, serializedPerSession("scout_coverage", async (_args, session) => {
     try {
@@ -1182,6 +1195,7 @@ server.registerTool("scout_coverage", {
             `Unexercised elements by route:`,
             ...cov.unexercised.slice(0, 25).map((u) => `  ${u.state}: ${u.keys.slice(0, 6).join(", ")}${u.keys.length > 6 ? ` … +${u.keys.length - 6}` : ""}`),
             ...formatUnchosenOptions(eng.memory.unchosenOptions()),
+            ...formatNeverSubmittedEmpty(eng.memory.formsNeverSubmittedEmpty()),
         ];
         return text(lines.join("\n"), session);
     }
@@ -1333,15 +1347,23 @@ server.registerTool("scout_verify", {
     }
 }));
 server.registerTool("scout_close", {
-    description: "Close a session's browser (memory persists on disk). Default: the DEFAULT session. Pass session to close a specific one, or all=true to close every live session at the end of a multi-role run.",
+    description: "Close a session's browser (memory persists on disk). Default: the DEFAULT session. Pass session to close a specific one, or all=true to close every live session at the end of a multi-role run. " +
+        "A session that is a lane of a parallel run (named by scout_lane_brief or scout_lane_report) is not closed until its lane report has been accepted by scout_lane_report, since folding needs the session attached; the refusal names every such lane.",
     inputSchema: {
         session: z.string().max(40).optional().describe("Session to close (default: the default session)"),
         all: z.boolean().default(false).describe("Close every live session"),
+        force: z
+            .boolean()
+            .default(false)
+            .describe("Close even a lane whose report has not been folded yet. Its decisions are then lost: nothing is kept for calibration and nothing checks its defects were filed. Prefer folding it with scout_lane_report first."),
     },
-}, serializedControl(async ({ session, all }) => {
+}, serializedControl(async ({ session, all, force }) => {
     try {
         if (all) {
             const names = [...engines.keys()];
+            const guard = laneCloseGuard(names, laneLedger, force);
+            if (!guard.ok)
+                return text(`${guard.message}\nNo session was closed.`, activeName);
             // Closes are independent per-browser — run them in parallel so N wedged
             // sessions cost one 8s teardown cap total, not N of them.
             const dirs = new Set();
@@ -1359,6 +1381,7 @@ server.registerTool("scout_close", {
                 store.endRun();
             sessionQueue.clear();
             board.clear();
+            laneLedger.clear();
             lastWriter = null;
             for (const dir of dirs)
                 flushStatus(dir);
@@ -1368,6 +1391,9 @@ server.registerTool("scout_close", {
         const eng = engines.get(name);
         if (!eng)
             return text(`No live session "${name}".`, name);
+        const guard = laneCloseGuard([name], laneLedger, force);
+        if (!guard.ok)
+            return text(guard.message, name);
         keepReport(eng);
         live?.dropSession(name);
         await eng.close();
@@ -1378,6 +1404,11 @@ server.registerTool("scout_close", {
             eng.memory.endRun();
         sessionQueue.forget(name);
         board.remove(name);
+        laneLedger.forget(name);
+        // The last session closing ends the run, and its lanes with it: a brief's
+        // names that never attached must not make a later run's session a lane.
+        if (engines.size === 0)
+            laneLedger.clear();
         if (lastWriter?.session === name)
             lastWriter = null;
         if (eng.memory?.dir)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scenescout",
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
   "license": "MIT",
   "author": "brunoboto96",
@@ -64,6 +64,7 @@
     "format:check": "prettier --check .",
     "demo": "npm run build && tsx scripts/demo.ts",
     "demo:serve": "node demo-app/server.mjs",
+    "holdout:serve": "node holdout-app/server.mjs",
     "dev": "tsx src/cli.ts",
     "serve": "node dist/mcp-server.js",
     "smoke": "npm run build && npm run smoke:run",
@@ -71,19 +72,21 @@
     "mcp-check": "npm run build && npm run mcp-check:run",
     "mcp-check:run": "tsx scripts/mcp-check.ts",
     "test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
-    "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
+    "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test",
     "scan-test": "tsx scripts/scan-test.ts",
     "oracle-test": "tsx --test scripts/oracle-test.ts",
     "policy-test": "tsx --test scripts/policy-test.ts",
     "fixture-test": "tsx --test scripts/fixture-test.ts",
     "dispatch-test": "tsx --test scripts/dispatch-test.ts",
     "design-test": "tsx --test scripts/design-test.ts",
+    "check-test": "tsx --test scripts/check-test.ts",
     "contract-test": "tsx --test scripts/contract-test.ts",
     "lane-test": "tsx --test scripts/lane-test.ts",
     "memory-test": "tsx --test scripts/memory-test.ts",
     "install-test": "tsx --test scripts/install-test.ts",
     "live-test": "tsx --test scripts/live-test.ts",
     "demo-test": "tsx --test scripts/demo-test.ts",
+    "holdout-test": "tsx --test scripts/holdout-test.ts",
     "hygiene-test": "tsx --test scripts/hygiene-test.ts",
     "pace-test": "tsx --test scripts/pace-test.ts",
     "request-test": "tsx --test scripts/request-test.ts",
@@ -104,7 +107,8 @@
     "@changesets/cli": "3.0.3",
     "@types/node": "^20.19.0",
     "prettier": "3.9.8",
-    "tsx": "^4.23.13",
-    "typescript": "^5.9.3"
+    "tsx": "^4.23.15",
+    "typescript": "^5.9.3",
+    "yaml": "^2.9.1"
   }
 }

package/skills/scenescout/SKILL.md

@@ -33,7 +33,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
 
 1. **`scout_crawl` first, always.** One call visits every known route (pass `paths` to sweep a specific subset instead), records coverage, and returns per-route health. This is the whole breadth pass — do not visit routes one-by-one with navigate+snapshot.
 2. **Investigate what the crawl flagged.** For each problem route (violations, dead-ends, auth-redirects): navigate there, `scout_snapshot`, reproduce, then `scout_finding`.
-3. **Run journeys with `scout_run_plan {steps}`.** Mechanical sequences (fill form → submit → check) go in ONE plan call — `steps` is an ordered list of `{action, target, value}` — with `testid=`/`text=`/`label=` targets — not one LLM turn per click. The plan aborts at the first violation and tells you where; that's your cue to investigate interactively.
+3. **Run journeys with `scout_run_plan {steps}`.** Mechanical sequences (fill form → submit → check) go in ONE plan call — `steps` is an ordered list of `{action, target, value}` — with `testid=`/`text=`/`label=`/`role=button[name="Save"]` targets — not one LLM turn per click. The plan aborts at the first violation and tells you where; that's your cue to investigate interactively. When the user wants a flow kept working, save the plan that walked it as `.scenescout/flows/<name>.json` — `{"name", "steps"}`, the same steps, starting with a `navigate`, plus `{"action":"expect-text","text"}`, `{"action":"expect-url","pattern"}` or `{"action":"expect-request","request":"GET /api/…","status":200}` where the outcome shows — and `scenescout check` replays it on every pull request. Save only flows that read: a check replays flows under observe's rule unless the project passes `--flow-writes allow`, so a step that sends a write is reported as "could not run" and the check exits 2.
 4. **Snapshot economics:** `scout_snapshot` after landing somewhere new; re-snapshots of the same route return *diffs* with stable refs — "No element changes" costs you almost nothing. `scout_screenshot` ONLY for suspected pixel-native issues (a canvas, a rendering glitch); geometry problems (overlap, off-screen, a covered control) are already in the snapshot as GEOMETRY issues, and images that failed to load are listed under BROKEN IMAGES — file those, quoting the line. Controls inside embeds (`<iframe>`) are listed with the page's, each marked `⟨in … frame⟩`, and act by ref like any other. A frame of the app's own origin is the app: test it fully. A frame of another site (`cross-origin`) is someone else's system that the user has not authorised you to test: click and type there as a user would, to see the embed render and respond, but never send it hostile input — the engine refuses markup, values over 200 characters, control characters, repeated-click probes and uploads there, and you must not try SQL, template or other injection shapes either — and file what you find there as the embed's behaviour, naming its origin, not as the app's bug. A failing request such a frame sent outside the app says so (`in an embed of …`) and is kept at medium; console errors cannot be told apart by frame and stay the app's; its controls are counted apart and never enter the app's coverage or gap ledger. Its container text is masked, and the writes it sends outside the app are refused in every mode but destructive — unless the user names that origin as a trusted embed (a provider in test mode, say): pass it in `scout_attach {trustedEmbeds: ["https://…"]}` and, in safe-write only, its writes go out. Never add an origin the user did not name. The FRAMES line names frames that were not read; say in your summary that their contents were not explored.
 5. **Native-user behaviours.** `scout_type {ref, textValue}` (or its alias `value`, matching `scout_select` and a plan step) APPENDS when a field already has content (menu clicks often insert @-mention chips or commands into composers — appending preserves them; the result reports what was already there); pass `replace=true` only to deliberately clear, and `pressEnter=true` to submit from the field the way a user would. Before concluding a badge, icon, or "N errors" indicator *does nothing*, `scout_hover` it — tooltips and hover cards are invisible to snapshots and clicks, and hover output includes what appeared. In HEADED mode (`scout_attach {headed:true}`, which the user asks for when they want to watch) the user's physical mouse competes with the synthetic pointer: if a hover reveals nothing and the finding matters, ask the user to move their mouse off the browser window and retry before filing. **Scroll long pages with `scout_scroll`** — the design audit and snapshot measure at the current scroll position, so judge deep sections by scrolling then re-auditing; it refuses to scroll where a real user couldn't and reports SCROLL LOCKED (the leaked modal scroll-lock that silently amputates everything below the fold — snapshots also flag it passively as an OVERLAY line), and scrolling triggers lazy-loaded content whose failures surface as fresh oracle violations. Elements fully clipped inside an overflow-hidden container are flagged UNREACHABLE in GEOMETRY issues — no amount of scrolling reveals them; that's a high-value layout bug, distinct from merely below-the-fold content. **A page can hold SEVERAL independent scroll regions** and plain `scout_scroll` moves the largest one, so a sidebar nav beside a taller main pane never budges: pass `scout_scroll {target:"testid=…"}` to scroll one region. Never report a nav item, tab or list row as missing/truncated until you have scrolled ITS container — content scrolled out of a secondary pane looks exactly like content that was cut off.
 6. **The rest of the input vocabulary.** `scout_select` sets a `<select>` option by value or visible label — use it rather than clicking a native dropdown open, which does not render as page DOM. `scout_press` sends a real key to the focused element (`Escape` to dismiss a modal, `Tab` to walk focus order, `Enter` to submit from a field); it is also how the keyboard-only pass at `extensive` is performed, and it vets the focused control first so a destructive action cannot be triggered blind in read-only mode. **`scout_upload {ref}` attaches a file the way a user does** — `ref` is a visible `<input type=file>` (snapshots list these with role `file`; `scout_type` on one redirects here) OR the button/label/dropzone that opens the file chooser (the chooser is intercepted and answered — that is how the hidden input behind a styled "Choose file" control is reached); omit `ref` when the page has exactly one file input, hidden or not (snapshots disclose hidden ones on a FILE INPUTS line). Nothing needs to exist on disk: a small VALID fixture is generated in memory, its kind inferred from the input's `accept` attribute or chosen with `fixture` (`pdf`, `png`, `txt`, `csv`, `json`); `filePath` uploads a real file but must live inside the attached project (fenced, like navigation is fenced to the origin); `name` overrides the filename. The result flags a file that violates `accept` (a mismatch the app then ACCEPTS is a validation finding), warns when the app cleared the input after selection, and says whether a state-changing request fired on selection — if none did, either click the form's submit or read the next snapshot for a client-side rejection. Plans take `{action:"upload", target, value:"pdf"}` steps (`target` required). When the input or its trigger was addressed by `ref`, the gap ledger counts an attached-but-unsent file as filled-never-submitted; the ref-less path has no listed element to mark.
@@ -42,7 +42,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
 8. **Design-connoisseur pass without pixels: `scout_design_audit`.** Run it once per representative page (dashboard, a form, a detail view, a data table). Its output has two tiers: **⚠ measurable defects** (WCAG contrast, tiny targets, clipped text, aspect-distorted images, horizontal overflow, keyboard tab stops with no visible focus indicator — sampled with real Tab presses) and **→ craft suggestions** (line measure and line-height rhythm, spacing-grid adherence, typography entropy, gray census and accent-hue count, pure-#000 body text, elevation/control consistency, heading structure, indistinguishable links, AI-slop tells like gradient text/glassmorphism/side-stripe borders/identical card grids), closing with a SYSTEM SUMMARY of design-system coherence. Judge every line with product context (dense tables legitimately have small targets; a chart page legitimately uses many hues). File ⚠ defects as `visual`/`a11y`, and genuine → opportunities as `ux-polish` findings **quoting the concrete numbers** — "~142 characters per line (65–75 ideal)" beats "text feels wide". Every audit ends with a **PAGE SCORE** (0–100 overall + a11y/craft/consistency/task-clarity subscores) persisted per route — the report ranks pages worst-first, so re-runs show whether pages got better or worse. Separately, every `scout_snapshot` runs an **overlay/modal probe** automatically: an empty dialog over a grayed page, a backdrop with no dialog, a far-off-centre dialog leaving a blank band, or a dialog extending unreachably below the viewport appear as OVERLAY lines in GEOMETRY issues — treat these as high-value findings (the user is visually stuck). This is where "how could this page be better" gets answered, not just "is it broken".
 9. **Measure task EASE with `scout_journey`, not just correctness.** Wrap each module's primary task (`{action:"start", goal:"Create an order"}` → do it → `{action:"end", completed:…}`). Navigate by CLICKING like a first-time user — typing a known deep URL shortcuts the very thing being measured (a route you can only reach by editing the address bar is itself a finding). The result gives interaction count, distinct screens, the path taken, and BACKTRACKS — returning to a screen already left is the clearest evidence the next step wasn't discoverable. An abandoned journey (`completed:false`) is a high-severity finding: the task is blocked or undiscoverable, which no passing e2e suite would ever reveal.
 10. **Walk the auth surface too — anonymously.** Attach a second session WITHOUT a storage-state file (a fresh logged-out profile) and exercise signup, login failure states, and forgot/reset-password **as far as they physically go**. The mailbox wall is expected — reaching "check your email" IS the success condition; everything before it is what you're testing: does submit actually fire (a dead signup button is a high finding), are errors specific and actionable, can the user resend or recover from a typo, does the flow dead-end. Use plausible synthetic identities only (invent `qa-<runid>@example.com`-style addresses, never a real person's), submit each form valid AND invalid, and judge the feedback. Two classic findings live here: a forgot-password that answers "no account with that email" is an **account-enumeration leak** (file as security; "if an account exists, we sent a link" is the correct shape), and a signup that accepts the form then lands on a blank or logged-out page with no guidance is a **journey dead-end**. Signup creates a record, so what this pass may do depends on the mode. In `observe`, fill and submit the auth forms for their CLIENT-SIDE behaviour only: the engine blocks signup, password change and reset, and lets only a login itself go out. Disclose the server-side half as a gap. Actually creating an account needs the user's explicit okay and safe-write mode; the engine tracks the created account like any other creation.
-11. **`scout_coverage` decides what's next** — it lists unvisited routes, unexercised elements, and the options of each dropdown you used that no session has chosen this run (a filter counts as exercised after one choice, and the option you skipped can be the one whose request fails). Trust it over your memory. Prefer reaching routes by clicking real navigation; fall back to direct URLs for coverage completeness and re-verification, and say which you used when it affects the finding (see the provenance rule below).
+11. **`scout_coverage` decides what's next** — it lists unvisited routes, unexercised elements, and the options of each dropdown you used that no session has chosen this run (a filter counts as exercised after one choice, and the option you skipped can be the one whose request fails), and each `<form>` seen this run that no session has submitted with every text field blank (submit it once empty: a submit that silently does nothing — no request, no message — is the commonest defect there, and filling a form in first never finds it). Trust it over your memory. Prefer reaching routes by clicking real navigation; fall back to direct URLs for coverage completeness and re-verification, and say which you used when it affects the finding (see the provenance rule below).
 
 ## Levels (completion contracts — the engine ENFORCES them via `scout_report {level}`)
 
@@ -72,7 +72,7 @@ Some flows need a TEAM — a document one role submits and another approves, a r
 - While roles are NOT collaborating, use each one productively where its permissions matter (admin in /admin surfaces, low-privilege probing for permission leaks) — same coverage contract, different vantage points.
 - **Infer the PERSONA behind each role, and write it down.** From what a role can see and do (its nav, its dashboard, the capability matrix in the report), state what this person is FOR: "qa = reviewer — approves orders, assigns reviewers, no admin" / "user = front-line user — reads documents, completes reviews, raises orders". Record it with `scout_note {section:'roles'}`. Then test the persona's WORLD, not just the permissions: does the operator's landing page serve an operator? Is anything they need N clicks deep? The capability matrix's divergent rows are questions, not verdicts — each is either a correct boundary or a gap ("should this role be able to do this?"); say which you believe it is and why.
 - `scout_close {all: true}` at the end of a multi-role run; `scout_close {session}` to drop one role early.
-- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
+- **Several agents in parallel** (subagents or a workflow, each driving its own session): each agent attaches its session when it STARTS, and the PLANNER closes it by name once it has folded that lane's report. A lane must NOT close its own session before reporting: `scout_lane_report` keeps its decisions against that session's project memory, so a lane that closes first hands back decisions with nowhere to write — the tool says so rather than accepting silently, and the run's calibration section ends up with nothing to measure. Fold, then close. `scout_close` enforces it: a session that `scout_lane_brief` or `scout_lane_report` named as a lane is not closed until a report from it has been ACCEPTED (a refused one does not count, so re-send the corrected object before closing), and `scout_close {all: true}` names every such lane and closes nothing. Pass `force: true` only when a lane will never report, knowing its decisions are lost. Keep the planner's own session open until the last lane is folded: closing every session ends the run, and with it the record of which sessions are lanes. Never open sessions ahead for agents that have not started, and never hand an open session from one agent to the next: an agent waiting for its turn should hold no browser. Give each session an `objective` when you attach it (`scout_attach {session, objective:"Approve and reject orders as a manager"}`) and wrap each goal in `scout_journey`: the live view shows that objective and the task it is on beside the session's feed, which is how the person watching knows what every agent is for. Keep that goal TRUE: one journey per goal, one goal per thing you are checking ("Save a settings change as the auditor", not "Check every page"), ended the moment it is decided and the next one started before you move on. A journey that outlives its goal shows the viewer an objective the session left behind minutes ago. Run roughly as many agents at once as the machine has cores, less two, since each drives a real browser; beyond that they only queue. Exploring one area is well within a mid-tier model, so run these agents on one (Sonnet or its equivalent in your client) unless the user names a model; keep the larger model for the agent that plans the split and writes the report. No agent may call `scout_close {all: true}` while others run — only the last step, once every agent has finished.
 - **Let the engine divide the app: `scout_lane_brief {lanes, goal}`.** Call it after the first crawl, when route knowledge is complete. It splits the known routes into whole modules — everything under `/orders` goes to one lane, so that lane carries state between its own steps instead of re-learning the app on every route — deals the modules out so the lanes come out within a route or two of each other, and returns each lane's session name, the `objective` to attach it with, the routes it owns, the route it lands on (its own first route: lanes that all attached on the home page all met its defects first, and several filed the same one), and the rules each lane must follow — file each defect as it is judged, check a list's request status before calling it empty or stuck, submit one markup value on a create form and open where it is listed, probe a withheld control's endpoint, check coverage, and leave the session open for you to fold. Pass `routes` to split a subset instead. Hand each brief to its agent verbatim. Dividing by hand fails in two ways that a finished run cannot tell apart from success: two lanes audit the same register while a third module is never opened, and route coverage reads complete either way; and lanes launch without an objective, so the person watching sees browsers clicking through their app with nothing to say why.
 - **Lanes report in typed decisions, not prose.** Each lane files its findings with `scout_finding` as it goes, so the report already has them; what the lane hands BACK to the planner is one JSON object, the lane report, and nothing else. Get the paragraph to put in a lane's prompt from `scout_lane_report {lane}`: it names the shape (`status`, one decision per observation with `verdict`, `severity`, `category`, a calibrated `confidence` and a machine-signature `evidence`, the `routes` covered, `blocked_by`), every value's closed set (the categories are `scout_finding`'s) and every length limit, because a limit a lane is not told refuses good replies. When the lane hands back, pass its text to `scout_lane_report {lane, reply}`: it returns the one-line fold (defects, highs, unsure, mean confidence, routes, what blocked it) or the reason the reply was refused, and keeps each decision so the confidence the lane stated can be checked against what the run went on to file — the report's calibration section is built from that, and it only appears once enough decisions exist to mean something. **So the confidence is worth stating honestly**: a lane that writes 0.95 on everything makes the section say so. The fold also lists every defect the lane judged that no finding on the project matches yet: have the lane file each with the same `evidence` (a finding filed without evidence cannot be matched), or name the finding that covers it, before you close its session, because a defect that is only in a lane report never reaches the run's report. Prose around ONE fenced JSON block is discarded unread and the report accepted; anything less clear-cut is refused. Relay a refusal to the lane once and ask for the corrected object; never re-judge its prose yourself, and tell every lane that the object IS its final report, since a lane that writes the object and then hands back a summary of it has failed. The planner then folds lanes by counting, not by reading: a lane's reply is a few hundred tokens whatever it found, and a value the schema refuses is caught at the boundary instead of becoming a severity like "Low-Medium" in the report. Where the client lets you set a lane's reasoning effort, use **medium**: in this project's benchmark it judged as well as high in three-quarters of the time and under half of xhigh, low inflated severities, and max took three minutes per lane for no better agreement.