scenescout 3.15.0 → 3.17.0

package/dist/engine/report.js

@@ -1,11 +1,16 @@
 import fs from "node:fs";
 import path from "node:path";
-import { SHARED_CHROME_ROUTE, isEmbedKey, isWorthALook, judgedMergesOf } from "./memory.js";
+import { baseRoute, reachedRoutes, routeIdentity, isEmbedKey, isWorthALook, judgedMergesOf, seenOnOf, } from "./memory.js";
 import { sayVerification } from "./verify.js";
+import { describePicture, readFindingPicture } from "./capture.js";
 import { feedForSession } from "./live.js";
 import { buildReplayHtml, evidenceFor } from "./replay.js";
 import { calibrate, formatCalibration } from "./calibration.js";
 import { formatPace, measurePace } from "./pace.js";
+import { formatNeverSubmittedEmpty } from "./forms.js";
+import { DEFAULT_REPORT_AUDIENCE, formatPlainSection, pictureOf } from "./plain.js";
+import { formatTicketsPlain, formatTicketsTechnical, ticketSummaryLine } from "./tickets.js";
+import { COLLECTOR_CAP } from "./collector.js";
 function playwrightSkeleton(f) {
     const routeClass = f.state.split("#")[0].split("?")[0];
     let gotoPath = routeClass;
@@ -126,11 +131,22 @@ export function formatWorthALook(items, sessionStart) {
         lines.push(`- **A defect only if** your project uses ${f.convention ?? "a convention the finding does not name"}`);
         if (f.evidence)
             lines.push(`- **Seen:** \`${f.evidence}\``);
-        lines.push(`- **Where:** \`${f.state}\` (${f.url})`);
+        lines.push(`- **Where:** \`${f.state}\` (${f.url})${alsoSeenOn(f)}`);
+        lines.push(...pictureLine(f));
         lines.push(``, f.detail, ``);
     }
     return lines;
 }
+/** A finding's picture as a bullet, its path relative to report.md, or nothing when it has none. */
+export function pictureLine(f) {
+    const p = readFindingPicture(f);
+    return p ? [`- **Picture:** \`${p.file}\` (${describePicture(p)})`] : [];
+}
+/** The other routes a merged finding was filed on, as the end of its Where line, or "". */
+function alsoSeenOn(f) {
+    const routes = seenOnOf(f);
+    return routes.length > 0 ? `; also seen on ${routes.map((r) => `\`${r}\``).join(", ")}` : "";
+}
 /**
  * How long ago a finding was last seen, for the historical index. A finding
  * nobody has re-confirmed in four months is a different thing from one seen
@@ -173,9 +189,22 @@ export function findingEvidence(memory, sessions) {
         // running at once, the run's whole log interleaves them, and the steps
         // before a finding would come from whichever lane acted last.
         const own = f.session ? sessions.find((s) => s.session === f.session) : undefined;
-        return { id: f.id, frames: evidenceFor(own ? own.steps : all, f.foundAt) };
+        const picture = readFindingPicture(f);
+        return {
+            id: f.id,
+            frames: evidenceFor(own ? own.steps : all, f.foundAt),
+            ...(picture
+                ? {
+                    picture: {
+                        file: picture.file,
+                        ...(picture.width && picture.height ? { width: picture.width, height: picture.height } : {}),
+                        caption: describePicture(picture),
+                    },
+                }
+                : {}),
+        };
     })
-        .filter((e) => e.frames.length > 0);
+        .filter((e) => e.frames.length > 0 || !!e.picture);
 }
 /**
  * The whole run as one page, for the live view to serve at its own address.
@@ -278,13 +307,10 @@ function offersSubmit(elements) {
         return toks.some((t) => SUBMIT_TOKENS.has(t)) && !toks.some((t) => FILTER_TOKENS.has(t));
     });
 }
-/**
- * The collector stops at 150 elements, so on a dense page the submit control
- * may simply not be in the element list. "No submit found" then means "we did
- * not look far enough", not "there is nothing to submit" — never suppress on
- * that basis.
- */
-const COLLECTOR_CAP = 150;
+// The collector stops at COLLECTOR_CAP elements, so on a dense page the
+// submit control may simply not be in the element list. "No submit found"
+// then means "we did not look far enough", not "there is nothing to submit" —
+// never suppress on that basis.
 /**
  * Make app-controlled text safe inside a Markdown table cell. The backslash
  * must be escaped FIRST: escaping only the pipe turns an input of `\|` into
@@ -395,6 +421,134 @@ export function formatUnchosenOptions(dropdowns) {
         ...(dropdowns.length > 15 ? [`  … +${dropdowns.length - 15} more`] : []),
     ];
 }
+/**
+ * What scout_coverage prints, in one of two scopes. "session" is one lane's
+ * own remaining work: the routes it reached this run and the forms it saw.
+ * In a parallel run the project view lists every lane's gaps, including
+ * routes this lane never opened and forms its role cannot reach, and a lane
+ * reading those as its own either wastes turns on them or reports itself
+ * incomplete. "project" is everything the memory holds, each route and form
+ * tagged with the sessions that reached it this run. In the session scope
+ * each route lists only the controls on states this session recorded. `routeLine`
+ * is the route contract's line (formatRouteCoverage), labelled as the
+ * project's in the session scope.
+ */
+export function coverageView(memory, session, scope, routeLine) {
+    const routes = scope === "session" ? memory.routesVisitedBy(session) : null;
+    // A session's own controls are the ones on the states it recorded: another
+    // role's table on the same route is not this session's work.
+    const cov = memory.coverage(routes ? { routes, states: memory.statesVisitedBy(session) } : undefined);
+    const inScope = (route) => routes === null || routes.has(route);
+    const tag = (route) => {
+        if (routes)
+            return "";
+        const by = memory.sessionsOnRoute(route);
+        return by.length > 0 ? ` (this run: ${by.join(", ")})` : "";
+    };
+    const head = routes === null
+        ? []
+        : routes.size === 0
+            ? [`Scope: session ${session}, which has reached no route this run yet. scope:"project" lists every session's coverage.`]
+            : [`Scope: session ${session} — the ${routes.size} route(s) it reached this run and the forms it saw. scope:"project" lists every session's.`];
+    return [
+        ...head,
+        `States known: ${cov.states} · Elements exercised: ${cov.elementsExercised}/${cov.elementsTotal}${cov.embeds.total > 0 ? ` (plus ${cov.embeds.exercised}/${cov.embeds.total} inside other sites' frames, not counted)` : ""}`,
+        // The route contract is the project's, every session's and every run's; a session's own figure is the head's.
+        routes ? `Project route contract (every session, every run): ${routeLine}` : routeLine,
+        `Unexercised elements by route:`,
+        ...cov.unexercised
+            .slice(0, 25)
+            .map((u) => `  ${u.state}: ${u.keys.slice(0, 6).join(", ")}${u.keys.length > 6 ? ` … +${u.keys.length - 6}` : ""}${tag(u.state)}`),
+        ...formatUnchosenOptions(memory.unchosenOptions().filter((d) => inScope(d.route))),
+        ...formatNeverSubmittedEmpty(routes ? memory.formsNeverSubmittedEmpty(session).map(({ route, key }) => ({ route, key })) : memory.formsNeverSubmittedEmpty()),
+    ];
+}
+/**
+ * The gap-ledger line for pages whose scripts sent a POST that observe
+ * refused, or null. Some apps read through POST (a search, a report query); a
+ * page loading its data that way cannot be tested in observe until the user
+ * names the endpoint in `readPosts`. The engine cannot tell a read from a
+ * write, so it names the endpoints and leaves the choice to the user.
+ */
+export function observeRefusedPostsGap(pages) {
+    const list = (pages ?? []).filter((p) => p.endpoints.length > 0);
+    if (list.length === 0)
+        return null;
+    const shown = list
+        .slice(0, 8)
+        .map((p) => `${p.route} (${p.endpoints.join(", ")})`)
+        .join("; ");
+    return (`${list.length} page(s) sent a POST that observe refused, so whatever it loads is untested: ${shown}${list.length > 8 ? " …" : ""}. ` +
+        `If one of these only reads (a search or query sent as POST), the user can name it in readPosts ("POST /path") and the page can be tested in observe; anything that writes stays refused.`);
+}
+/**
+ * The route lines of the gap ledger: never visited, visited but nothing
+ * exercised, never design-audited. All three count over ONE set, the route
+ * contract (`knownRoutes`), and say so, so their figures can be compared with
+ * each other and with the route line of scout_coverage.
+ *
+ * A state route folds into its contract route, and a UI-state query variant
+ * (`/things/:id?section=history`) into its base path: a tab clicked on the
+ * base route is recorded on the base route's state, and a variant reached by
+ * URL is the same page. A visited route the contract does not hold (a path
+ * typed by hand that does not exist) is not a route of the app.
+ *
+ * "Nothing exercised" and "never audited" judge the routes this run reached
+ * (`ledgerScope`), since a route an earlier run looked at and left says
+ * nothing about this run. Whether a control was exercised, or a route
+ * audited, still counts from any run.
+ */
+function routeLedger(memory, extras) {
+    const lines = [];
+    const runStates = memory.runStates;
+    const scope = extras?.ledgerScope ?? (runStates.size > 0 ? "run" : "project");
+    const states = scope === "run" ? runStates : undefined;
+    const when = scope === "run" ? "this run" : "in any run";
+    const visited = new Set();
+    for (const [fp, st] of Object.entries(memory.states))
+        if (!states || states.has(fp))
+            visited.add(routeIdentity(st.route));
+    const hasContract = (extras?.knownRoutes?.length ?? 0) > 0;
+    const contract = hasContract ? [...new Set(extras.knownRoutes)] : [...visited];
+    const of = `of ${contract.length} ${hasContract ? "known" : "visited"} route(s)`;
+    const unvisited = extras?.unvisitedRoutes ?? [];
+    if (unvisited.length > 0) {
+        lines.push(`${unvisited.length} ${hasContract ? `${of} ` : "route(s) "}never visited in any run: ${unvisited.slice(0, 10).join(", ")}${unvisited.length > 10 ? " …" : ""}`);
+    }
+    // `total` and `exercised` come from coverage() so they are the SAME deduped,
+    // chrome-stripped count the unexercised list is drawn from. Recomputing them
+    // from raw state elements (as this once did) counts every state's copy of a
+    // shared element, and a genuinely untouched route vanished from the ledger.
+    // The shared-chrome pseudo-route is no state's route, so it is never visited.
+    const cov = memory.coverage(states ? { states } : undefined);
+    const groups = new Map();
+    for (const [route, c] of cov.perRoute) {
+        if (!visited.has(route))
+            continue;
+        const g = groups.get(baseRoute(route)) ?? { total: 0, exercised: 0 };
+        g.total += c.total;
+        g.exercised += c.exercised;
+        groups.set(baseRoute(route), g);
+    }
+    const auditedBases = new Set(Object.entries(memory.routeFacts)
+        .filter(([, f]) => f.audited)
+        .map(([r]) => baseRoute(routeIdentity(r))));
+    // A contract route counts as reached when it was itself, or (being a base
+    // path) when one of its tabs or sections was.
+    const reached = contract.filter(reachedRoutes(visited));
+    const untouched = reached.filter((r) => {
+        const g = groups.get(baseRoute(routeIdentity(r)));
+        return g !== undefined && g.total > 0 && g.exercised === 0;
+    });
+    if (untouched.length > 0) {
+        lines.push(`${untouched.length} ${of} visited ${when} but NOTHING exercised (looked at, never touched): ${untouched.slice(0, 8).join(", ")}${untouched.length > 8 ? " …" : ""}`);
+    }
+    const unaudited = reached.filter((r) => !auditedBases.has(baseRoute(routeIdentity(r))));
+    if (unaudited.length > 0) {
+        lines.push(`${unaudited.length} ${of} visited ${when} and never design-audited: ${unaudited.slice(0, 8).join(", ")}${unaudited.length > 8 ? " …" : ""}`);
+    }
+    return lines;
+}
 /**
  * The GAP LEDGER — an explicit enumeration of what was NOT tested. This is
  * what turns "extensive" from a vibe into a verifiable claim: a run is only
@@ -403,33 +557,8 @@ export function formatUnchosenOptions(dropdowns) {
  */
 export function computeGaps(memory, extras) {
     const gaps = [];
-    const cov = memory.coverage();
     const facts = memory.routeFacts;
-    const visitedRoutes = [...new Set(Object.values(memory.states).map((st) => st.route))];
-    if (extras?.unvisitedRoutes?.length) {
-        gaps.push(`${extras.unvisitedRoutes.length} route(s) never visited: ${extras.unvisitedRoutes.slice(0, 10).join(", ")}${extras.unvisitedRoutes.length > 10 ? " …" : ""}`);
-    }
-    // `total` comes from coverage() so it is the SAME deduped, chrome-stripped
-    // denominator that `keys` is a subset of. Recomputing it from raw state
-    // elements (as this once did) counts every state's copy of a shared element,
-    // so any route with two states had total > keys.length, the equality never
-    // held, and a genuinely untouched route vanished from the ledger.
-    // SHARED_CHROME_ROUTE is a pseudo-route: no state carries it, so it cannot be
-    // "visited" and there is nothing to navigate to in order to clear it. Before
-    // coverage() reported a total for it, it fell out of this filter by accident
-    // (total === 0); excluding it explicitly keeps the ledger to entries a tester
-    // can actually act on.
-    const untouched = cov.unexercised.filter((u) => u.state !== SHARED_CHROME_ROUTE && u.total > 0 && u.keys.length === u.total);
-    if (untouched.length > 0) {
-        gaps.push(`${untouched.length} route(s) visited but NOTHING exercised (looked at, never touched): ${untouched
-            .slice(0, 8)
-            .map((u) => u.state)
-            .join(", ")}${untouched.length > 8 ? " …" : ""}`);
-    }
-    const unaudited = visitedRoutes.filter((r) => !facts[r]?.audited);
-    if (unaudited.length > 0) {
-        gaps.push(`${unaudited.length}/${visitedRoutes.length} visited route(s) never design-audited: ${unaudited.slice(0, 8).join(", ")}${unaudited.length > 8 ? " …" : ""}`);
-    }
+    gaps.push(...routeLedger(memory, extras));
     // Filled in, never committed. `mutated` records that a state-changing request
     // actually left the page; a route where someone typed, picked an option or
     // attached a file but nothing was ever submitted is a form that was looked
@@ -456,6 +585,10 @@ export function computeGaps(memory, extras) {
                 ? ` — expected in observe mode, which blocks every form submission by design; what the server does with these forms is untested. Cover them in read-only mode against an environment where creating records is acceptable.`
                 : ""));
     }
+    // From project memory, not a session: lanes close theirs before the report is written.
+    const refusedPosts = observeRefusedPostsGap(memory.observeRefusedPosts);
+    if (refusedPosts)
+        gaps.push(refusedPosts);
     const journeyTotal = Object.values(facts).reduce((a, f) => a + (f.journeysCompleted ?? 0), 0);
     if (journeyTotal === 0) {
         gaps.push(`no COMPLETED scout_journey measurements — task EASE is untested. An abandoned journey is a finding, not coverage: it proves a task is blocked, not that it was measured.`);
@@ -465,6 +598,40 @@ export function computeGaps(memory, extras) {
         gaps.push(`single-role run (${roles.join(", ") || "no role recorded"}) — permission boundaries and role capability gaps are untested`);
     return gaps;
 }
+/** The last frame on screen before each finding was filed, by id: its picture on a recorded run. */
+function lastFrames(memory) {
+    // A finding can carry evidence with no recorded frames (its own picture
+    // only), so it has no last frame to offer.
+    return new Map(findingEvidence(memory, replaySessions(memory))
+        .filter((e) => e.frames.length > 0)
+        .map((e) => [e.id, e.frames[e.frames.length - 1].frame]));
+}
+/**
+ * The document for an audience. `technical` is the report's lines, its title
+ * and date before the first section. The plain section is built only when it
+ * is printed: "both" puts it between the date and the technical report, which
+ * gets a heading of its own, and "qa" prints it alone.
+ */
+export function withAudience(technical, audience, plain) {
+    if (audience === "dev")
+        return technical.join("\n");
+    // The title and date are everything before the first section.
+    const first = technical.findIndex((l) => l.startsWith("## "));
+    const split = first < 0 ? technical.length : first;
+    const head = technical.slice(0, split);
+    const body = technical.slice(split);
+    if (audience === "qa")
+        return [...head, ...plain()].join("\n");
+    return [
+        ...head,
+        ...plain(),
+        `## Technical detail`,
+        ``,
+        `Everything below is for developers: requests, oracles, routes and ids, the gap ledger and a test skeleton for each finding.`,
+        ``,
+        ...body,
+    ].join("\n");
+}
 /**
  * The report as the run stands now. `write` is what scout_report does at the
  * end; the live view renders the same document on request without touching
@@ -508,6 +675,10 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
         lines.push(`| Finding dedup | ${judged.replace(/\|/g, "/")} |`);
     lines.push(`| Elements exercised (informational — denominator grows with every state) | ${cov.elementsExercised}/${cov.elementsTotal} |`);
     lines.push(``);
+    // ---- The tickets the run was given, criterion by criterion. ----
+    const ticketRun = memory.ticketsThisRun();
+    const ticketInput = { tickets: ticketRun.tickets, verdicts: ticketRun.verdicts, findings: memory.findings };
+    lines.push(...formatTicketsTechnical(ticketInput));
     // ---- Page quality scores, worst first — the cross-page comparator. ----
     // Scores persist across runs, so a table sorted purely by number ranks
     // yesterday's un-fixed measurement above today's re-audit and reports a page
@@ -602,7 +773,8 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
         lines.push(`- **Id:** \`${f.id}\` · **Category:** ${f.category}`);
         if (f.evidence)
             lines.push(`- **Evidence:** \`${f.evidence}\``);
-        lines.push(`- **Where:** \`${f.state}\` (${f.url})`);
+        lines.push(`- **Where:** \`${f.state}\` (${f.url})${alsoSeenOn(f)}`);
+        lines.push(...pictureLine(f));
         lines.push(`- **Seen in runs:** ${f.runs}`);
         // A merge the model made is shown with what was filed, so a wrong one can be seen, and refiled as its own defect (ADR 4).
         for (const m of judgedMergesOf(f)) {
@@ -703,6 +875,17 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
             lines.push(`- \`${o}\``);
         lines.push(``);
     }
+    if (extras?.readPosts && extras.readPosts.length > 0) {
+        lines.push(`## Read POSTs`);
+        lines.push(``);
+        lines.push(extras.mode === "observe"
+            ? `Named by the user as endpoints that only read, so observe let them out unless the path or body looked destructive or the body was a GraphQL mutation:`
+            : `Named as reads, but not applied: they only count in observe mode, and this run was ${extras.mode ?? "read-only"}:`);
+        lines.push(``);
+        for (const e of extras.readPosts)
+            lines.push(`- \`${e}\``);
+        lines.push(``);
+    }
     lines.push(...violationRollup(oracleLog.filter((v) => !v.embed)));
     lines.push(...embedSection(oracleLog.filter((v) => v.embed), cov.embeds));
     if (cov.unexercised.length > 0) {
@@ -725,7 +908,24 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
     // Only on a run that used lanes, and only once enough of them have been
     // judged for the number to mean anything; formatCalibration decides both.
     lines.push(...formatCalibration(calibrate(memory.laneDecisions, memory.findings)));
-    const markdown = lines.join("\n");
+    const markdown = withAudience(lines, extras?.report ?? DEFAULT_REPORT_AUDIENCE, () => {
+        const frames = lastFrames(memory);
+        return formatPlainSection({
+            current,
+            historical: historical.length,
+            worthALook: worthALook.length,
+            violations: oracleLog.filter((v) => !v.embed),
+            routes: extras && extras.routesTotal > 0 ? { visited: extras.routesVisited, total: extras.routesTotal } : undefined,
+            gaps,
+            lastFrames: frames,
+            audience: extras?.report ?? DEFAULT_REPORT_AUDIENCE,
+            tickets: formatTicketsPlain({
+                ...ticketInput,
+                problemNumber: new Map(current.map((f, i) => [f.id, i + 1])),
+                pictureOf: (f) => pictureOf(f, frames.get(f.id)),
+            }),
+        });
+    });
     const outPath = path.join(memory.dir, "report.md");
     const htmlPath = path.join(memory.dir, "report.html");
     let htmlWritten = false;
@@ -774,6 +974,7 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
                 : []),
         ``,
         `OPEN FINDINGS: ${open.length} (${open.filter((f) => f.severity === "high").length} high) — ${current.length} this session, ${historical.length} historical${resolved.length ? `, ${resolved.length} resolved` : ""}`,
+        ...[ticketSummaryLine(ticketRun.tickets, ticketRun.verdicts)].filter((l) => l !== null),
         ...(extras && extras.routesTotal > 0
             ? [
                 `COVERAGE: routes ${extras.routesVisited}/${extras.routesTotal} · ${cov.states} states · ${cov.elementsExercised}/${cov.elementsTotal} elements exercised`,
@@ -796,5 +997,5 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
         `Gap ledger${gaps.length === 0 ? ": EMPTY — nothing known left untested" : ` (${gaps.length}):`}`,
         ...gaps.map((g) => `  ⚠ ${g}`),
     ];
-    return { markdown, path: outPath, summary: summaryLines.join("\n") };
+    return { markdown, path: outPath, summary: summaryLines.join("\n"), ...(htmlWritten ? { html: htmlPath } : {}) };
 }