scenescout 3.10.0 → 3.11.0

package/dist/engine/flow.js

@@ -0,0 +1,398 @@
+/**
+ * Saved flows: the steps `scenescout check` replays, as data.
+ *
+ * A flow is a short script a person or an agent wrote down once because it
+ * matters that it keeps working: open a page, click a control, see a result.
+ * The check replays every flow in `.scenescout/flows/*.json` after its crawl,
+ * with no model, so a flow that passed yesterday and breaks today fails the
+ * gate naming the flow and the step (ADR 12).
+ *
+ * The steps are `scout_run_plan`'s steps, so a plan that worked in an
+ * exploratory run can be saved as it is, plus three assertions a plan has no
+ * need for: text is visible, the URL matches, a request answered with a status.
+ *
+ * Everything here is pure: the schema and its error messages, the target
+ * grammar, and the rules that decide whether a URL or a request matched. The
+ * browser half is BrowserEngine.replayFlow, which only acts and reports.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { beaconResourceType, BROWSER_ENGINES } from "../browsers.js";
+import { MEMORY_DIRNAME, SELF_IGNORE_KEEP } from "./memory.js";
+/** Most steps one flow may hold. A longer flow is several flows, and each would fail more usefully on its own. */
+export const MAX_FLOW_STEPS = 50;
+/** Most flows one check replays. */
+export const MAX_FLOWS = 50;
+/** How long a step waits for its target, text, URL or request before it counts as broken. */
+export const FLOW_STEP_TIMEOUT_MS = 5000;
+/**
+ * How long a flow waits after its last step for a write that step set off
+ * late (a save on a timer, a debounced autosave), before it counts as passed.
+ */
+export const FLOW_AFTER_LAST_STEP_MS = 750;
+/** Where a project keeps its flows, relative to the project directory. */
+export const FLOWS_SUBDIR = path.join(MEMORY_DIRNAME, SELF_IGNORE_KEEP);
+export const TARGET_HELP = 'testid=…, text=…, label=… or role=<role>[name="…"]';
+/**
+ * The ARIA roles a `role=` target may name (WAI-ARIA 1.2, as Playwright's
+ * getByRole accepts them). A misspelt role matches nothing, so a flow using
+ * one would only ever fail at replay; it is refused when the file is read.
+ */
+export const ARIA_ROLE_LIST = [
+    "alert",
+    "alertdialog",
+    "application",
+    "article",
+    "banner",
+    "blockquote",
+    "button",
+    "caption",
+    "cell",
+    "checkbox",
+    "code",
+    "columnheader",
+    "combobox",
+    "complementary",
+    "contentinfo",
+    "definition",
+    "deletion",
+    "dialog",
+    "directory",
+    "document",
+    "emphasis",
+    "feed",
+    "figure",
+    "form",
+    "generic",
+    "grid",
+    "gridcell",
+    "group",
+    "heading",
+    "img",
+    "insertion",
+    "link",
+    "list",
+    "listbox",
+    "listitem",
+    "log",
+    "main",
+    "marquee",
+    "math",
+    "meter",
+    "menu",
+    "menubar",
+    "menuitem",
+    "menuitemcheckbox",
+    "menuitemradio",
+    "navigation",
+    "none",
+    "note",
+    "option",
+    "paragraph",
+    "presentation",
+    "progressbar",
+    "radio",
+    "radiogroup",
+    "region",
+    "row",
+    "rowgroup",
+    "rowheader",
+    "scrollbar",
+    "search",
+    "searchbox",
+    "separator",
+    "slider",
+    "spinbutton",
+    "status",
+    "strong",
+    "subscript",
+    "superscript",
+    "switch",
+    "tab",
+    "table",
+    "tablist",
+    "tabpanel",
+    "term",
+    "textbox",
+    "time",
+    "timer",
+    "toolbar",
+    "tooltip",
+    "tree",
+    "treegrid",
+    "treeitem",
+];
+export const ARIA_ROLES = new Set(ARIA_ROLE_LIST);
+export const ARIA_ROLES_MATCH_PLAYWRIGHT = ARIA_ROLE_LIST;
+/** Parse a target string, or null when it is not one. */
+export function parseTarget(target) {
+    for (const by of ["testid", "text", "label"]) {
+        if (target.startsWith(`${by}=`)) {
+            const value = target.slice(by.length + 1);
+            return value ? { by, value } : null;
+        }
+    }
+    const role = /^role=([a-z]+)(?:\[name=(?:"([^"]*)"|'([^']*)'|([^\]"']*))\])?$/i.exec(target);
+    if (!role)
+        return null;
+    const name = role[2] ?? role[3] ?? role[4];
+    return name === undefined ? { by: "role", role: role[1].toLowerCase() } : { by: "role", role: role[1].toLowerCase(), name };
+}
+const REQUEST_RE = /^(GET|POST|PUT|PATCH|DELETE|HEAD) (\/\S*)$/;
+function isRegex(pattern) {
+    try {
+        new RegExp(pattern);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const target = z
+    .string()
+    .refine((t) => parseTarget(t) !== null, { message: `must be ${TARGET_HELP}` })
+    .superRefine((t, ctx) => {
+    const parsed = parseTarget(t);
+    if (parsed?.by === "role" && !ARIA_ROLES.has(parsed.role)) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `names the role "${parsed.role}", which is not an ARIA role (e.g. button, link, textbox, checkbox, tab)`,
+        });
+    }
+});
+const STEP_SCHEMAS = [
+    z.object({ action: z.literal("navigate"), target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
+    z.object({ action: z.literal("click"), target }).strict(),
+    z.object({ action: z.literal("type"), target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
+    z.object({ action: z.literal("select"), target, value: z.string() }).strict(),
+    z.object({ action: z.literal("press"), value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
+    z.object({ action: z.literal("expect-text"), text: z.string().min(1, "is the text that must be visible") }).strict(),
+    z.object({ action: z.literal("expect-url"), pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) }).strict(),
+    z
+        .object({
+        action: z.literal("expect-request"),
+        request: z.string().regex(REQUEST_RE, 'must be a method and a path, e.g. "GET /api/things"'),
+        status: z.union([z.number().int().min(100).max(599), z.string().regex(/^[1-5]xx$/, 'must be a status such as 200, or a class such as "2xx"')]),
+    })
+        .strict(),
+];
+const stepSchema = z.discriminatedUnion("action", [...STEP_SCHEMAS]);
+export const FLOW_ACTIONS = STEP_SCHEMAS.map((s) => s.shape.action.value);
+const flowSchema = z
+    .object({
+    name: z.string().min(1).max(100).optional(),
+    description: z.string().max(500).optional(),
+    steps: z.array(stepSchema).min(1, "needs at least one step").max(MAX_FLOW_STEPS, `holds at most ${MAX_FLOW_STEPS} steps`),
+})
+    .strict()
+    .superRefine((flow, ctx) => {
+    // A flow starts from a page it names, never from wherever the crawl left the browser.
+    if (flow.steps[0] && flow.steps[0].action !== "navigate") {
+        ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["steps", 0, "action"], message: 'must be "navigate": a flow starts from a page it names' });
+    }
+});
+/** `steps[2].target`, the way a person would look for it in the file. */
+function fieldPath(p) {
+    if (p.length === 0)
+        return "(the whole file)";
+    return p.map((part, i) => (typeof part === "number" ? `[${part}]` : i === 0 ? part : `.${part}`)).join("");
+}
+function issueMessage(issue) {
+    if (issue.code === "unrecognized_keys")
+        return `unknown field(s) ${issue.keys.map((k) => `"${k}"`).join(", ")}`;
+    if (issue.code === "invalid_union_discriminator")
+        return `must be one of ${FLOW_ACTIONS.join(", ")}`;
+    if (issue.code === "invalid_type" && issue.received === "undefined")
+        return "is required";
+    return issue.message;
+}
+/** Validate one flow file's text. Every mistake names the file and the field. */
+export function parseFlow(text, file) {
+    let json;
+    try {
+        json = JSON.parse(text);
+    }
+    catch (err) {
+        return { ok: false, error: `${file}: not valid JSON (${err instanceof Error ? err.message : String(err)})` };
+    }
+    const parsed = flowSchema.safeParse(json);
+    if (!parsed.success) {
+        const issues = parsed.error.issues.slice(0, 5).map((i) => `${fieldPath(i.path)} ${issueMessage(i)}`);
+        const more = parsed.error.issues.length > 5 ? ` (and ${parsed.error.issues.length - 5} more)` : "";
+        return { ok: false, error: `${file}: ${issues.join("; ")}${more}` };
+    }
+    return { ok: true, flow: { name: parsed.data.name ?? file.replace(/\.json$/i, ""), file, steps: parsed.data.steps } };
+}
+/**
+ * Read every `*.json` in a flows directory, in name order. Throws on the first
+ * file that is not a valid flow: a flow that cannot be read would otherwise
+ * pass by never running. Everything else in the directory is returned as
+ * skipped with its reason, so nothing there is left out without saying so. A
+ * symbolic link is followed when it resolves to a file inside the directory,
+ * and skipped when it resolves anywhere else.
+ */
+export function loadFlows(dir) {
+    const realDir = fs.realpathSync(dir);
+    const files = [];
+    const skipped = [];
+    for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((x, y) => (x.name < y.name ? -1 : x.name > y.name ? 1 : 0))) {
+        if (e.isDirectory()) {
+            skipped.push({ file: e.name, reason: "a directory: flows are not read from subdirectories" });
+            continue;
+        }
+        if (!e.name.toLowerCase().endsWith(".json")) {
+            skipped.push({ file: e.name, reason: "not a .json file" });
+            continue;
+        }
+        if (e.isSymbolicLink()) {
+            let target;
+            try {
+                target = fs.realpathSync(path.join(dir, e.name));
+            }
+            catch {
+                skipped.push({ file: e.name, reason: "a symbolic link to nothing" });
+                continue;
+            }
+            if (!target.startsWith(realDir + path.sep)) {
+                skipped.push({ file: e.name, reason: "a symbolic link to a file outside the flows directory" });
+                continue;
+            }
+            if (!fs.statSync(target).isFile()) {
+                skipped.push({ file: e.name, reason: "a symbolic link to something that is not a file" });
+                continue;
+            }
+        }
+        else if (!e.isFile()) {
+            skipped.push({ file: e.name, reason: "not a regular file" });
+            continue;
+        }
+        files.push(e.name);
+    }
+    if (files.length > MAX_FLOWS)
+        throw new Error(`${dir} holds ${files.length} flows; a check replays at most ${MAX_FLOWS}`);
+    const flows = files.map((file) => {
+        const parsed = parseFlow(fs.readFileSync(path.join(dir, file), "utf8"), file);
+        if (!parsed.ok)
+            throw new Error(`flow ${parsed.error}`);
+        return parsed.flow;
+    });
+    return { flows, skipped };
+}
+/**
+ * Which flows directory a check uses: the one named, none for "off", and
+ * otherwise the project's own when it exists. A directory that was named and
+ * is missing is an error, never a quiet check with no flows.
+ */
+export function resolveFlowsDir(flag, projectDir, exists) {
+    if (flag === "off")
+        return { dir: null };
+    if (flag !== undefined)
+        return exists(flag) ? { dir: flag } : { error: `--flows: no directory at ${flag}` };
+    const own = path.join(projectDir, FLOWS_SUBDIR);
+    return { dir: exists(own) ? own : null };
+}
+/** A step as the report names it. */
+export function describeStep(step) {
+    switch (step.action) {
+        case "navigate":
+        case "click":
+        case "select":
+            return `${step.action} ${step.target}`;
+        case "type":
+            return `type into ${step.target}${step.pressEnter ? " and press Enter" : ""}`;
+        case "press":
+            return `press ${step.value}`;
+        case "expect-text":
+            return `expect text ${JSON.stringify(step.text)}`;
+        case "expect-url":
+            return `expect the URL to match /${step.pattern}/`;
+        case "expect-request":
+            return `expect ${step.request} → ${step.status}`;
+    }
+}
+/** Whether a step acts on the page (and so opens a new window for expect-request) rather than reading it. */
+export function isAction(step) {
+    return !step.action.startsWith("expect-");
+}
+/** Path, query and hash: what an expect-url pattern is tested against, so the same flow passes on a laptop and a preview deployment. */
+export function urlMatches(pattern, url) {
+    let tail;
+    try {
+        const u = new URL(url);
+        tail = `${u.pathname}${u.search}${u.hash}`;
+    }
+    catch {
+        tail = url;
+    }
+    return new RegExp(pattern).test(tail);
+}
+/** A path pattern against a request's path: equal segment for segment, where `*` stands for any one segment. A trailing slash is ignored. */
+export function requestPathMatches(pattern, pathname) {
+    const split = (p) => p.replace(/\/+$/, "").split("/");
+    const want = split(pattern);
+    const got = split(pathname);
+    return want.length === got.length && want.every((seg, i) => seg === "*" || seg === got[i]);
+}
+export function statusMatches(expected, status) {
+    return typeof expected === "number" ? status === expected : String(status)[0] === expected[0];
+}
+/**
+ * Whether the requests seen since the last action include the one a step
+ * expects. The reason says what was seen instead, which is what a person
+ * fixing it needs: the right request with the wrong status, or none at all.
+ */
+export function matchRequest(expect, seen) {
+    const [, method, pattern] = REQUEST_RE.exec(expect.request) ?? [];
+    const same = seen.filter((r) => {
+        if (r.method.toUpperCase() !== method)
+            return false;
+        try {
+            return requestPathMatches(pattern, new URL(r.url).pathname);
+        }
+        catch {
+            return false;
+        }
+    });
+    if (same.some((r) => statusMatches(expect.status, r.status)))
+        return { ok: true };
+    if (same.length > 0) {
+        const statuses = [...new Set(same.map((r) => r.status))].join(", ");
+        return { ok: false, reason: `${expect.request} answered ${statuses}, expected ${expect.status}` };
+    }
+    return { ok: false, reason: `no ${expect.request} request was sent since the last action` };
+}
+/**
+ * The request kinds a refused write may be excused from a step for: what
+ * navigator.sendBeacon and <a ping> send, under every engine's name for it
+ * (browsers.ts `beaconResourceType`). They are fire-and-forget by design, a
+ * page sends them on its own schedule, and no step waits for one.
+ */
+export const BACKGROUND_REQUEST_TYPES = new Set(["ping", ...BROWSER_ENGINES.map(beaconResourceType)]);
+/**
+ * Split what the write policy refused during a step into what the step is
+ * charged with and background requests, which are only listed. The rule is
+ * the request's kind, never its origin: an app's API is often on another port
+ * or subdomain, and deciding by origin either hid a step's own write or
+ * blamed one on the app. So a fetch, an XHR or a form post is charged to the
+ * step wherever it goes, telemetry sent with fetch included; only a beacon or
+ * a ping is excused. A refusal with no kind recorded is charged. The app's
+ * origin is dropped from what is charged, for a shorter reason.
+ */
+export function splitRefusals(refusals, appOrigin) {
+    const charged = [];
+    const background = [];
+    for (const { sig, type } of refusals) {
+        if (type !== undefined && BACKGROUND_REQUEST_TYPES.has(type))
+            background.push(sig);
+        else
+            charged.push(sig.split(appOrigin).join(""));
+    }
+    return { charged, background };
+}
+/** The step a run stopped at, as the evidence of its issue or the reason the check could not run. */
+export function flowStepEvidence(run) {
+    if (run.outcome.status === "passed")
+        return "";
+    return `flow "${run.name}" (${run.file}) step ${run.outcome.step} of ${run.steps}, ${run.outcome.did}: ${run.outcome.reason}`;
+}

package/dist/engine/memory.js

@@ -541,6 +541,8 @@ function sameFinding(a, b) {
 }
 /** Per-project directory for memory, session logs, and reports. */
 export const MEMORY_DIRNAME = ".scenescout";
+/** The one directory under it that is committed rather than ignored: saved flows (flow.ts). */
+export const SELF_IGNORE_KEEP = "flows";
 /** What the directory was called before the tool was renamed. */
 export const LEGACY_MEMORY_DIRNAME = ".scenecraft";
 /**
@@ -611,6 +613,10 @@ function liveOwnerPid(dir, isAlive) {
  * `git status`. That gets the same result without editing a file we do not
  * own, and it disappears cleanly when someone deletes the directory.
  *
+ * The one exception is `flows/`: saved flows are inputs a team commits so
+ * CI replays them, not output. A file written before that exception existed
+ * is still left alone; docs/ci.md says which two lines to add to it.
+ *
  * Best-effort and idempotent — a read-only checkout must never break attach,
  * and an existing file is left exactly as the user left it.
  */
@@ -621,7 +627,10 @@ export function writeSelfIgnore(dir) {
             return null;
         fs.writeFileSync(ignorePath, "# SceneScout exploratory-test artifacts: memory, session logs, status, report.\n" +
             "# Self-ignoring so a `git add -A` in the tested project can never commit them.\n" +
-            "*\n");
+            "*\n" +
+            "# Except the flows `scenescout check` replays: they are written to be committed.\n" +
+            `!${SELF_IGNORE_KEEP}/\n` +
+            `!${SELF_IGNORE_KEEP}/*.json\n`);
         return `Wrote ${MEMORY_DIRNAME}/.gitignore so these test artifacts stay out of your commits.`;
     }
     catch {

package/dist/engine/verify.js

@@ -23,6 +23,7 @@
  * Pure, so the ordering and the wording can be table-tested.
  */
 import { normalizePath } from "./fingerprint.js";
+import { failingSignatures } from "./memory.js";
 /** What a re-test found. */
 export const VERDICTS = ["gone", "present", "changed"];
 export const VERDICT_MEANING = {
@@ -140,3 +141,121 @@ export function sayVerification(f) {
         return ` · re-tested ${on}, behaviour has changed since it was filed`;
     return ` · re-tested ${on}`;
 }
+// ---------------------------------------------------------------------------
+// Re-testing by page load, for `scenescout check`.
+//
+// The campaign above hands findings to an agent, who re-walks them with
+// judgement. A check has none, so it re-tests only the findings whose
+// reproduction is mechanical: the evidence names a GET that failed with a
+// status, and the finding's repro shows nothing done on its page but looking
+// at it. Loading that page again and watching the same request is then the
+// whole reproduction, and its answer is a fact rather than an opinion.
+// ---------------------------------------------------------------------------
+/**
+ * Actions that only look at a page. A repro whose steps on the finding's page
+ * are all of these was reproduced by loading the page; any other step (a
+ * click, a typed value, a replayed request, anything this list does not know)
+ * means loading it is not the reproduction.
+ */
+const LOOK_ONLY = new Set([
+    "attach",
+    "navigate",
+    "plan:navigate",
+    "crawl",
+    "back",
+    "snapshot",
+    "screenshot",
+    "design-audit",
+    "task",
+    "journey:start",
+    "journey:end",
+    "record:full",
+]);
+/** How the finding's page is reached again, or null when a check cannot re-test it deterministically. */
+export function loadRetest(item) {
+    const signatures = [...failingSignatures(item.evidence)];
+    // A POST that failed is not re-sent by loading a page, and a check would refuse to send it anyway.
+    if (signatures.length === 0 || signatures.some((s) => !s.startsWith("GET ")))
+        return null;
+    // The first repro line is how the page was reached; every later one happened on it.
+    if (item.repro.length === 0)
+        return null;
+    for (const line of item.repro.slice(1)) {
+        if (!LOOK_ONLY.has(line.split(/\s/, 1)[0]))
+            return null;
+    }
+    let url;
+    try {
+        url = new URL(item.url);
+    }
+    catch {
+        return null;
+    }
+    // A secret was redacted out of the stored URL: loading what is left is a different page.
+    if (/redacted/i.test(item.url))
+        return null;
+    return { item, path: `${url.pathname}${url.search}`, signatures };
+}
+export const RETEST_VERDICTS = ["reproduces", "possibly-fixed", "not-reached"];
+/**
+ * Each re-testable finding against the page load that re-tests it.
+ *
+ * `reproduces` when the load saw one of the finding's failing requests fail
+ * with the same status; `possibly-fixed` when the page loaded and none did —
+ * possibly, because the page may simply not have asked this time; and
+ * `not-reached` when the page did not load or bounced to sign-in, which says
+ * nothing about the finding either way.
+ */
+export function retestResults(candidates, pages) {
+    return candidates.map(({ item, path, signatures }) => {
+        const base = { id: item.id, severity: item.severity, title: item.title, path, signatures };
+        const page = pages.find((p) => p.path === path);
+        if (!page)
+            return { ...base, verdict: "not-reached", note: "the page was not visited" };
+        if (page.loadError !== undefined)
+            return { ...base, verdict: "not-reached", note: "the page did not load" };
+        if (page.loginRedirect)
+            return { ...base, verdict: "not-reached", note: "the page sent the browser to sign-in" };
+        const seen = new Set();
+        for (const detail of page.httpErrors)
+            for (const s of failingSignatures(detail))
+                seen.add(s);
+        // The document's own failure is left out of the violations (withoutOwnResponse), so it is added back here.
+        if (page.status !== null && page.status >= 400)
+            for (const s of failingSignatures(`GET ${page.path.split("?")[0]} ${page.status}`))
+                seen.add(s);
+        return { ...base, verdict: signatures.some((s) => seen.has(s)) ? "reproduces" : "possibly-fixed" };
+    });
+}
+/**
+ * The findings a check re-tests: the open ones it can reproduce by loading a
+ * page, in the campaign's order and within its cap, plus how many open
+ * findings there are in all, so the report can say how many it left to a run.
+ */
+export function checkRetestPlan(findings) {
+    const open = findings.filter((f) => (f.status ?? "open") !== "resolved");
+    const eligible = open.filter((f) => loadRetest(item(f)) !== null);
+    const candidates = verifyWorklist(eligible)
+        .map(loadRetest)
+        .filter((c) => c !== null);
+    return { candidates, open: open.length };
+}
+/**
+ * The entries of a memory file's findings list a re-test can read. The file is
+ * JSON someone may have edited; an entry missing the fields the rules read is
+ * left out rather than failing the check, which only ever reports re-tests.
+ */
+export function wellFormedFindings(list) {
+    return list.filter((f) => {
+        if (typeof f !== "object" || f === null)
+            return false;
+        const x = f;
+        return (typeof x.id === "string" &&
+            typeof x.url === "string" &&
+            typeof x.title === "string" &&
+            typeof x.severity === "string" &&
+            (x.evidence === undefined || typeof x.evidence === "string") &&
+            Array.isArray(x.repro) &&
+            x.repro.every((r) => typeof r === "string"));
+    });
+}

package/dist/mcp-server.js

@@ -849,7 +849,7 @@ server.registerTool("scout_crawl", {
     }
 }, 600_000));
 server.registerTool("scout_run_plan", {
-    description: "Execute up to 20 actions in ONE call — use for mechanical sequences (fill a form, walk a wizard) so each step doesn't cost a round-trip. Targets resolve at execution time by semantic locator: 'testid=…', 'text=…', or 'label=…' (never snapshot refs). An `upload` step attaches a file as scout_upload does (target required — the file input or the control that opens its chooser; value = a fixture kind or a project-relative path). The plan ABORTS at the first NEW oracle violation, policy refusal, or failed step, returning a transcript of how far it got; repeats of already-reported violations do not abort (they stay logged for the report).",
+    description: "Execute up to 20 actions in ONE call — use for mechanical sequences (fill a form, walk a wizard) so each step doesn't cost a round-trip. Targets resolve at execution time by semantic locator: 'testid=…', 'text=…', 'label=…' or 'role=button[name=Save]' (never snapshot refs). The same steps, saved to .scenescout/flows/<name>.json with expect-text / expect-url / expect-request steps added, are replayed by `scenescout check` on every pull request. An `upload` step attaches a file as scout_upload does (target required — the file input or the control that opens its chooser; value = a fixture kind or a project-relative path). The plan ABORTS at the first NEW oracle violation, policy refusal, or failed step, returning a transcript of how far it got; repeats of already-reported violations do not abort (they stay logged for the report).",
     inputSchema: {
         steps: z
             .array(z.object({
@@ -857,7 +857,7 @@ server.registerTool("scout_run_plan", {
             target: z
                 .string()
                 .optional()
-                .describe("testid=…, text=…, label=… (or a path for navigate; 'top'/'bottom'/±px for scroll; for upload: the file input or the control that opens its chooser)"),
+                .describe("testid=…, text=…, label=…, role=<role>[name=\"…\"] (or a path for navigate; 'top'/'bottom'/±px for scroll; for upload: the file input or the control that opens its chooser)"),
             value: z
                 .string()
                 .optional()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scenescout",
-  "version": "3.10.0",
+  "version": "3.11.0",
   "description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
   "license": "MIT",
   "author": "brunoboto96",
@@ -64,6 +64,7 @@
     "format:check": "prettier --check .",
     "demo": "npm run build && tsx scripts/demo.ts",
     "demo:serve": "node demo-app/server.mjs",
+    "holdout:serve": "node holdout-app/server.mjs",
     "dev": "tsx src/cli.ts",
     "serve": "node dist/mcp-server.js",
     "smoke": "npm run build && npm run smoke:run",
@@ -71,7 +72,7 @@
     "mcp-check": "npm run build && npm run mcp-check:run",
     "mcp-check:run": "tsx scripts/mcp-check.ts",
     "test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
-    "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run hygiene-test",
+    "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run contract-test && npm run pace-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test",
     "scan-test": "tsx scripts/scan-test.ts",
     "oracle-test": "tsx --test scripts/oracle-test.ts",
     "policy-test": "tsx --test scripts/policy-test.ts",
@@ -85,6 +86,7 @@
     "install-test": "tsx --test scripts/install-test.ts",
     "live-test": "tsx --test scripts/live-test.ts",
     "demo-test": "tsx --test scripts/demo-test.ts",
+    "holdout-test": "tsx --test scripts/holdout-test.ts",
     "hygiene-test": "tsx --test scripts/hygiene-test.ts",
     "pace-test": "tsx --test scripts/pace-test.ts",
     "request-test": "tsx --test scripts/request-test.ts",

package/skills/scenescout/SKILL.md

@@ -33,7 +33,7 @@ You are the brain of an exploratory UI tester. The SceneScout MCP server gives y
 
 1. **`scout_crawl` first, always.** One call visits every known route (pass `paths` to sweep a specific subset instead), records coverage, and returns per-route health. This is the whole breadth pass — do not visit routes one-by-one with navigate+snapshot.
 2. **Investigate what the crawl flagged.** For each problem route (violations, dead-ends, auth-redirects): navigate there, `scout_snapshot`, reproduce, then `scout_finding`.
-3. **Run journeys with `scout_run_plan {steps}`.** Mechanical sequences (fill form → submit → check) go in ONE plan call — `steps` is an ordered list of `{action, target, value}` — with `testid=`/`text=`/`label=` targets — not one LLM turn per click. The plan aborts at the first violation and tells you where; that's your cue to investigate interactively.
+3. **Run journeys with `scout_run_plan {steps}`.** Mechanical sequences (fill form → submit → check) go in ONE plan call — `steps` is an ordered list of `{action, target, value}` — with `testid=`/`text=`/`label=`/`role=button[name="Save"]` targets — not one LLM turn per click. The plan aborts at the first violation and tells you where; that's your cue to investigate interactively. When the user wants a flow kept working, save the plan that walked it as `.scenescout/flows/<name>.json` — `{"name", "steps"}`, the same steps, starting with a `navigate`, plus `{"action":"expect-text","text"}`, `{"action":"expect-url","pattern"}` or `{"action":"expect-request","request":"GET /api/…","status":200}` where the outcome shows — and `scenescout check` replays it on every pull request. Save only flows that read: a check replays flows under observe's rule unless the project passes `--flow-writes allow`, so a step that sends a write is reported as "could not run" and the check exits 2.
 4. **Snapshot economics:** `scout_snapshot` after landing somewhere new; re-snapshots of the same route return *diffs* with stable refs — "No element changes" costs you almost nothing. `scout_screenshot` ONLY for suspected pixel-native issues (a canvas, a rendering glitch); geometry problems (overlap, off-screen, a covered control) are already in the snapshot as GEOMETRY issues, and images that failed to load are listed under BROKEN IMAGES — file those, quoting the line. Controls inside embeds (`<iframe>`) are listed with the page's, each marked `⟨in … frame⟩`, and act by ref like any other. A frame of the app's own origin is the app: test it fully. A frame of another site (`cross-origin`) is someone else's system that the user has not authorised you to test: click and type there as a user would, to see the embed render and respond, but never send it hostile input — the engine refuses markup, values over 200 characters, control characters, repeated-click probes and uploads there, and you must not try SQL, template or other injection shapes either — and file what you find there as the embed's behaviour, naming its origin, not as the app's bug. A failing request such a frame sent outside the app says so (`in an embed of …`) and is kept at medium; console errors cannot be told apart by frame and stay the app's; its controls are counted apart and never enter the app's coverage or gap ledger. Its container text is masked, and the writes it sends outside the app are refused in every mode but destructive — unless the user names that origin as a trusted embed (a provider in test mode, say): pass it in `scout_attach {trustedEmbeds: ["https://…"]}` and, in safe-write only, its writes go out. Never add an origin the user did not name. The FRAMES line names frames that were not read; say in your summary that their contents were not explored.
 5. **Native-user behaviours.** `scout_type {ref, textValue}` (or its alias `value`, matching `scout_select` and a plan step) APPENDS when a field already has content (menu clicks often insert @-mention chips or commands into composers — appending preserves them; the result reports what was already there); pass `replace=true` only to deliberately clear, and `pressEnter=true` to submit from the field the way a user would. Before concluding a badge, icon, or "N errors" indicator *does nothing*, `scout_hover` it — tooltips and hover cards are invisible to snapshots and clicks, and hover output includes what appeared. In HEADED mode (`scout_attach {headed:true}`, which the user asks for when they want to watch) the user's physical mouse competes with the synthetic pointer: if a hover reveals nothing and the finding matters, ask the user to move their mouse off the browser window and retry before filing. **Scroll long pages with `scout_scroll`** — the design audit and snapshot measure at the current scroll position, so judge deep sections by scrolling then re-auditing; it refuses to scroll where a real user couldn't and reports SCROLL LOCKED (the leaked modal scroll-lock that silently amputates everything below the fold — snapshots also flag it passively as an OVERLAY line), and scrolling triggers lazy-loaded content whose failures surface as fresh oracle violations. Elements fully clipped inside an overflow-hidden container are flagged UNREACHABLE in GEOMETRY issues — no amount of scrolling reveals them; that's a high-value layout bug, distinct from merely below-the-fold content. **A page can hold SEVERAL independent scroll regions** and plain `scout_scroll` moves the largest one, so a sidebar nav beside a taller main pane never budges: pass `scout_scroll {target:"testid=…"}` to scroll one region. Never report a nav item, tab or list row as missing/truncated until you have scrolled ITS container — content scrolled out of a secondary pane looks exactly like content that was cut off.
 6. **The rest of the input vocabulary.** `scout_select` sets a `<select>` option by value or visible label — use it rather than clicking a native dropdown open, which does not render as page DOM. `scout_press` sends a real key to the focused element (`Escape` to dismiss a modal, `Tab` to walk focus order, `Enter` to submit from a field); it is also how the keyboard-only pass at `extensive` is performed, and it vets the focused control first so a destructive action cannot be triggered blind in read-only mode. **`scout_upload {ref}` attaches a file the way a user does** — `ref` is a visible `<input type=file>` (snapshots list these with role `file`; `scout_type` on one redirects here) OR the button/label/dropzone that opens the file chooser (the chooser is intercepted and answered — that is how the hidden input behind a styled "Choose file" control is reached); omit `ref` when the page has exactly one file input, hidden or not (snapshots disclose hidden ones on a FILE INPUTS line). Nothing needs to exist on disk: a small VALID fixture is generated in memory, its kind inferred from the input's `accept` attribute or chosen with `fixture` (`pdf`, `png`, `txt`, `csv`, `json`); `filePath` uploads a real file but must live inside the attached project (fenced, like navigation is fenced to the origin); `name` overrides the filename. The result flags a file that violates `accept` (a mismatch the app then ACCEPTS is a validation finding), warns when the app cleared the input after selection, and says whether a state-changing request fired on selection — if none did, either click the form's submit or read the next snapshot for a client-side rejection. Plans take `{action:"upload", target, value:"pdf"}` steps (`target` required). When the input or its trigger was addressed by `ref`, the gap ledger counts an attached-but-unsent file as filled-never-submitted; the ref-less path has no listed element to mark.