scenescout 3.14.1 → 3.15.0

package/dist/engine/memory.js

@@ -26,6 +26,26 @@ export const FINDING_CATEGORIES = [
     "missing-testid",
     "other",
 ];
+/** Most judged merges one finding keeps; the oldest go first. */
+export const MAX_JUDGED_MERGES = 10;
+/**
+ * A finding's judged merges that are well formed. The file is read back
+ * without a schema, and the report prints each of these, so a hand-edited
+ * entry that is not one is dropped, not trusted.
+ */
+export function judgedMergesOf(f) {
+    if (!Array.isArray(f.judgedMerges))
+        return [];
+    return f.judgedMerges.filter((m) => !!m &&
+        typeof m === "object" &&
+        typeof m.title === "string" &&
+        typeof m.category === "string" &&
+        typeof m.at === "string" &&
+        ["high", "medium", "low"].includes(m.severity) &&
+        typeof m.pSame === "number" &&
+        Number.isFinite(m.pSame) &&
+        (m.evidence === undefined || typeof m.evidence === "string"));
+}
 /**
  * Whether a finding is in the "worth a look" tier. Anything else a file holds
  * reads as a defect: a finding is never taken out of the defect count on a
@@ -217,6 +237,8 @@ export function mergeMemory(mine, theirs) {
                 // Keep whichever action was actually recorded; ours wins a tie.
                 ...((el.lastAction ?? prev?.lastAction) ? { lastAction: el.lastAction ?? prev?.lastAction } : {}),
                 absentStreak: Math.min(prev?.absentStreak ?? 0, el.absentStreak ?? 0),
+                // Ours is the latest reading of whether a user can act on it.
+                ...(el.inert ? { inert: true } : {}),
             };
         }
         out.states[fp] = {
@@ -251,6 +273,12 @@ export function mergeMemory(mine, theirs) {
         Object.assign(withoutTier(merged), tier);
         if (!isWorthALook(merged) && isWorthALook(newer))
             merged.severity = older.severity;
+        // Each side's judged merges are a record of filings, not later knowledge: kept from both.
+        const trail = unionJudgedMerges(f, other);
+        if (trail)
+            merged.judgedMerges = trail;
+        else
+            delete merged.judgedMerges;
         byId.set(f.id, merged);
     }
     out.findings = [...byId.values()];
@@ -307,6 +335,16 @@ export function mergeMemory(mine, theirs) {
         delete out.laneRoutes;
     return out;
 }
+/** Two findings' judged merges as one list, each filing once, oldest first, at most MAX_JUDGED_MERGES. Idempotent. */
+function unionJudgedMerges(a, b) {
+    const all = [...judgedMergesOf(b), ...judgedMergesOf(a)];
+    if (all.length === 0)
+        return undefined;
+    const byKey = new Map();
+    for (const m of all)
+        byKey.set(`${m.at}\u0000${m.title}`, m);
+    return [...byKey.values()].sort((x, y) => x.at.localeCompare(y.at)).slice(-MAX_JUDGED_MERGES);
+}
 /** Most routes kept per lane. A lane report carries at most a few hundred; this keeps a long-lived project's file small. */
 export const MAX_LANE_ROUTES = 255;
 /**
@@ -374,6 +412,10 @@ function findingJaccard(a, b) {
     const union = new Set([...a, ...b]).size;
     return union === 0 ? 0 : inter / union;
 }
+/** How alike two titles are, 0 to 1, by the token overlap the rule's last tier uses. Orders the dedup judge's candidates. */
+export function titleSimilarity(a, b) {
+    return findingJaccard(findingTokens(a), findingTokens(b));
+}
 /**
  * Distinctive literals — quoted or parenthesized fragments like
  * `"(role not recorded)"` or `"Document not found"`. Two findings on the same
@@ -809,7 +851,20 @@ export class MemoryStore {
         this.auditsThisRun = 0;
         this.selectChoices.clear();
         this.emptySubmits.clear();
+        this.dedupJudge = null;
+        this.dedupChoice = undefined;
+        this.dedupOff = undefined;
     }
+    /**
+     * The dedup judge filings go through (fileFinding), or null for the rule
+     * alone. Per run, like the probes: the server sets it when a session
+     * attaches, and a run that never asks for it dedups by the rule only.
+     */
+    dedupJudge = null;
+    /** What an attach of this run asked for by name; unset means the server's environment decides. */
+    dedupChoice;
+    /** Why the judge this run asked for could not be set up (no key), for the report; unset when it was not asked for or is on. */
+    dedupOff;
     /**
      * Each dropdown's options and the ones chosen in THIS run, by any session,
      * keyed by route and element. A select counts as exercised after one choice,
@@ -953,6 +1008,10 @@ export class MemoryStore {
                 dupOf.runs += f.runs;
                 if (!dupOf.evidence && f.evidence)
                     dupOf.evidence = f.evidence;
+                // Each copy's judged merges are filings: folding the copies keeps them all.
+                const trail = unionJudgedMerges(dupOf, f);
+                if (trail)
+                    dupOf.judgedMerges = trail;
                 if (f.status === "resolved")
                     dupOf.status = "resolved";
                 const promoted = isWorthALook(dupOf) && !isWorthALook(f);
@@ -1435,8 +1494,12 @@ export class MemoryStore {
         this.actionLog.push(full);
         fs.appendFileSync(this.sessionLogPath, JSON.stringify(full) + "\n");
     }
-    /** Record a visit to a state; returns whether it was new. */
-    visitState(fingerprint, url, route, elementKeys) {
+    /**
+     * Record a visit to a state; returns whether it was new. `inertKeys` are the
+     * listed keys a user cannot act on (collector inertKeys): they stay known,
+     * so a click on one still registers, but coverage does not count them.
+     */
+    visitState(fingerprint, url, route, elementKeys, inertKeys = []) {
         let rec = this.data.states[fingerprint];
         const isNew = !rec;
         if (!rec) {
@@ -1446,10 +1509,15 @@ export class MemoryStore {
         rec.visits += 1;
         rec.lastSeen = new Date().toISOString();
         const present = new Set(elementKeys);
+        const inert = new Set(inertKeys);
         for (const key of elementKeys) {
             if (!rec.elements[key])
                 rec.elements[key] = { exercised: false };
             rec.elements[key].absentStreak = 0;
+            if (inert.has(key))
+                rec.elements[key].inert = true;
+            else
+                delete rec.elements[key].inert;
         }
         // Prune ghosts: dynamic elements (list rows, ordinal-suffixed duplicates)
         // that vanish for 3 consecutive visits would otherwise make coverage
@@ -1492,9 +1560,65 @@ export class MemoryStore {
      */
     /** Returns the finding, whether it is new, and whether an existing worth-a-look was just promoted to a defect by it. */
     addFinding(input) {
-        // Redact BEFORE the id is derived, so a re-found finding whose quoted
-        // secret differs by a character still hashes to the same id.
-        const f = {
+        const f = this.redacted(input);
+        const id = findingId(f);
+        const existing = this.data.findings.find((x) => isDuplicateFinding(x, f));
+        if (existing)
+            return [existing, false, this.mergeInto(existing, f, id)];
+        return [this.append(f, id), true, false];
+    }
+    /**
+     * File a finding, asking the dedup judge when one is set (dedupJudge) and
+     * the rule keeps the filing apart from everything stored. The rule decides
+     * first, exactly as addFinding: a merge the rule makes is never put to the
+     * model, so the judge can only add merges, never undo one, and the rule's
+     * merge at the next load (retroMerge) cannot split what the judge joined.
+     * Without a judge this is addFinding.
+     */
+    async fileFinding(input) {
+        const f = this.redacted(input);
+        const id = findingId(f);
+        const byRule = this.data.findings.find((x) => isDuplicateFinding(x, f));
+        if (byRule)
+            return { finding: byRule, isNew: false, promoted: this.mergeInto(byRule, f, id) };
+        const judge = this.dedupJudge;
+        if (!judge)
+            return { finding: this.append(f, id), isNew: true, promoted: false };
+        const before = new Set(this.data.findings.map((x) => x.id));
+        let verdict = null;
+        let judgeError;
+        try {
+            verdict = await judge.judge(f, [...this.data.findings]);
+        }
+        catch (err) {
+            judgeError = err instanceof Error ? err.message : String(err);
+        }
+        const extra = judgeError ? { judgeError } : {};
+        // The list may have changed while the judge was asked. A filing that
+        // landed meanwhile is compared by the rule, as it would have been with no
+        // judge; the finding the judge chose is looked up again by id, since a
+        // merge with another process's memory may have replaced its object.
+        const landed = this.data.findings.find((x) => !before.has(x.id) && isDuplicateFinding(x, f));
+        if (landed)
+            return { finding: landed, isNew: false, promoted: this.mergeInto(landed, f, id), ...extra };
+        // Only what a judge may answer is taken: an open finding on the filing's page, called the same at better than even.
+        const route = f.state.split("#")[0];
+        const chosen = verdict && verdict.pSame >= 0.5
+            ? this.data.findings.find((x) => x.id === verdict.sameAs && x.status !== "resolved" && x.state.split("#")[0] === route)
+            : undefined;
+        if (chosen && verdict) {
+            const judged = { pSame: verdict.pSame };
+            return { finding: chosen, isNew: false, promoted: this.mergeInto(chosen, f, id, judged), judged, ...extra };
+        }
+        return { finding: this.append(f, id), isNew: true, promoted: false, ...extra };
+    }
+    /**
+     * The filing with secrets redacted. BEFORE the id is derived, so a re-found
+     * finding whose quoted secret differs by a character still hashes to the
+     * same id; and before the dedup judge is shown it.
+     */
+    redacted(input) {
+        return {
             ...input,
             title: redactSecrets(input.title),
             detail: redactSecrets(input.detail),
@@ -1503,37 +1627,54 @@ export class MemoryStore {
             url: redactSecrets(input.url),
             evidence: input.evidence ? redactSecrets(input.evidence) : input.evidence,
         };
-        const id = findingId(f);
-        const existing = this.data.findings.find((x) => isDuplicateFinding(x, f));
-        if (existing) {
-            existing.runs += 1;
-            existing.foundAt = new Date().toISOString();
-            if (!existing.evidence && f.evidence)
-                existing.evidence = f.evidence;
-            // A worth-a-look filed again as a defect is promoted, at the severity the defect was filed at.
-            const promoted = isWorthALook(existing) && !isWorthALook(f);
-            const tier = mergeTier(existing, f);
-            Object.assign(withoutTier(existing), tier);
-            if (promoted)
-                existing.severity = f.severity;
-            // Re-finding a RESOLVED finding is a regression — reopen it loudly
-            // rather than letting it hide in the report's completed section. But a
-            // FUZZY match must never resurrect a fixed bug: telling someone a
-            // regression landed when it did not is far more costly than carrying a
-            // visible duplicate, and it corrupts the one signal that says whether a
-            // fix held. Demand an exact identity match (same category+title+route) or
-            // an identical evidence signature before reopening.
-            const exactMatch = existing.id === id ||
-                (!!existing.evidence &&
-                    !!f.evidence &&
-                    existing.evidence.toLowerCase().replace(/\s+/g, " ").trim() === f.evidence.toLowerCase().replace(/\s+/g, " ").trim());
-            if (existing.status === "resolved" && exactMatch) {
-                existing.status = "open";
-                existing.regressedAt = existing.foundAt;
-            }
-            this.flush();
-            return [existing, false, promoted];
+    }
+    /**
+     * Fold a filing into the finding it duplicates. Returns whether a
+     * worth-a-look was promoted to a defect by it. `judged`: the dedup judge
+     * made this merge, so the filing is kept on the finding (judgedMerges).
+     */
+    mergeInto(existing, f, id, judged) {
+        existing.runs += 1;
+        existing.foundAt = new Date().toISOString();
+        if (!existing.evidence && f.evidence)
+            existing.evidence = f.evidence;
+        // A worth-a-look filed again as a defect is promoted, at the severity the defect was filed at.
+        const promoted = isWorthALook(existing) && !isWorthALook(f);
+        const tier = mergeTier(existing, f);
+        Object.assign(withoutTier(existing), tier);
+        if (promoted)
+            existing.severity = f.severity;
+        // Re-finding a RESOLVED finding is a regression — reopen it loudly
+        // rather than letting it hide in the report's completed section. But a
+        // FUZZY match must never resurrect a fixed bug: telling someone a
+        // regression landed when it did not is far more costly than carrying a
+        // visible duplicate, and it corrupts the one signal that says whether a
+        // fix held. Demand an exact identity match (same category+title+route) or
+        // an identical evidence signature before reopening.
+        const exactMatch = existing.id === id ||
+            (!!existing.evidence &&
+                !!f.evidence &&
+                existing.evidence.toLowerCase().replace(/\s+/g, " ").trim() === f.evidence.toLowerCase().replace(/\s+/g, " ").trim());
+        if (existing.status === "resolved" && exactMatch) {
+            existing.status = "open";
+            existing.regressedAt = existing.foundAt;
         }
+        if (judged) {
+            const entry = {
+                title: f.title,
+                category: f.category,
+                severity: f.severity,
+                ...(f.evidence ? { evidence: f.evidence } : {}),
+                pSame: judged.pSame,
+                at: existing.foundAt,
+            };
+            existing.judgedMerges = [...(existing.judgedMerges ?? []), entry].slice(-MAX_JUDGED_MERGES);
+        }
+        this.flush();
+        return promoted;
+    }
+    /** Store a filing as a new finding, with its repro trace. */
+    append(f, id) {
         // Repro trace scoped to the finding's route: everything since the action
         // that landed there, not 12 lines of unrelated cross-module noise.
         const routeOf = (url) => {
@@ -1569,7 +1710,7 @@ export class MemoryStore {
         };
         this.data.findings.push(finding);
         this.flush();
-        return [finding, true, false];
+        return finding;
     }
     get findings() {
         return this.data.findings;
@@ -1684,6 +1825,9 @@ export class MemoryStore {
                 byRoute.set(rec.route, route);
             }
             for (const [key, v] of Object.entries(rec.elements)) {
+                // Not a control: nothing to exercise, so not counted as a gap or a total.
+                if (v.inert)
+                    continue;
                 route.set(key, (route.get(key) ?? false) || v.exercised);
             }
         }

package/dist/engine/oracles.js

@@ -106,6 +106,78 @@ export class EmbedRequestLog {
         return key === null ? null : (this.byKey.get(key) ?? null);
     }
 }
+/** How long after a replay ends the browser's console echo of its failure is still taken for the replay's. */
+export const REPLAY_ECHO_WINDOW_MS = 2000;
+/**
+ * Addresses the tester is calling with scout_request, so the browser's console
+ * echo of a refused probe ("Failed to load resource: … 403") is known as the
+ * tester's and not charged to the page. The request itself is matched by
+ * identity in the monitor; the echo carries only an address, so it is matched
+ * by address while the replay is in flight and for a short window after, which
+ * is when the browser prints it. Bounded, dropping the oldest.
+ */
+export class ReplayLog {
+    cap;
+    /** Address → until when an echo of it is the replay's (Infinity while in flight). */
+    until = new Map();
+    constructor(cap = 200) {
+        this.cap = cap;
+    }
+    begin(url) {
+        const key = requestKey(url);
+        this.until.delete(key);
+        this.until.set(key, Infinity);
+        if (this.until.size > this.cap)
+            this.until.delete(this.until.keys().next().value);
+    }
+    end(url, now) {
+        const key = requestKey(url);
+        if (this.until.has(key))
+            this.until.set(key, now + REPLAY_ECHO_WINDOW_MS);
+    }
+    /** Whether a console message is the browser's echo of a replay's failed load. */
+    echoes(text, locationUrl, pageUrl, now) {
+        const key = failedLoadEchoOf(text, locationUrl, pageUrl);
+        if (key === null)
+            return false;
+        const until = this.until.get(key);
+        if (until === undefined)
+            return false;
+        if (now > until) {
+            this.until.delete(key);
+            return false;
+        }
+        return true;
+    }
+}
+/**
+ * Wording of a router cancelling a route change on purpose. Some client-side
+ * routers can only stop a navigation (to keep a dirty form, say) by throwing
+ * from their route-change event, and the apps that use them filter this
+ * sentinel out of their own error monitoring. Narrow on purpose: it must name
+ * the route and the cancelling.
+ */
+const ROUTE_CANCEL_RE = /\brout(?:e|ing)\b.{0,40}\b(?:abort|cancel)|\b(?:abort|cancel)\w*\b.{0,40}\brout(?:e|ing)\b/i;
+/**
+ * Is this uncaught page error a router deliberately cancelling a route change,
+ * rather than a crash? Only for a click that left the URL where it was, and
+ * then only when the message says so, or when the click asked for a route
+ * change and the page put up a confirmation instead. Still reported, at
+ * medium with a note: the error is real and uncaught, but the user never sees
+ * it, and ranking it with crashes made every such page argue it away.
+ */
+export function isRouteCancellation(message, c) {
+    if (!c.byClick || c.urlChanged)
+        return false;
+    return ROUTE_CANCEL_RE.test(message) || (c.viaLink && c.dialogOpened);
+}
+/** Appended to a page error read as a route-change cancellation, saying which evidence it was read on. */
+export function routeCancelNote(message) {
+    const why = ROUTE_CANCEL_RE.test(message)
+        ? "the click left the URL unchanged and the error says the route change was cancelled"
+        : "the click on a link left the URL unchanged and the page opened a dialog instead";
+    return ` (likely a router cancelling the route change on purpose: ${why}; reported at medium, check what the page showed before filing)`;
+}
 /**
  * Invariant oracles: passive listeners that record violations regardless of
  * what the agent is doing. The engine drains the buffer after every action and
@@ -124,6 +196,11 @@ export class OracleMonitor {
             // Benign noise: failed favicon / source map fetches show up as console errors.
             if (/favicon|source map/i.test(text))
                 return;
+            // The browser's echo of the tester's own scout_request: its answer was in that tool's result.
+            if (this.replays.echoes(text, msg.location().url, page.url(), Date.now())) {
+                this.replayAttributed += 1;
+                return;
+            }
             this.record({
                 kind: "console_error",
                 severity: "high",
@@ -150,6 +227,10 @@ export class OracleMonitor {
                 return;
             if (BENIGN_URL_RE.test(req.url()))
                 return;
+            if (this.isReplay(req)) {
+                this.replayAttributed += 1;
+                return;
+            }
             if (this.refusedByPolicy(req)) {
                 this.policyAttributed += 1;
                 return;
@@ -170,6 +251,12 @@ export class OracleMonitor {
             // reported here on every page load teaches the driver to ignore http_error.
             if (BENIGN_URL_RE.test(res.url()))
                 return;
+            // The tester's own scout_request: a probe of a boundary is meant to be
+            // refused, and its answer was in that tool's result, not the page's.
+            if (this.isReplay(res.request())) {
+                this.replayAttributed += 1;
+                return;
+            }
             // The write policy's own stand-in refusal, matched by request identity
             // like a dropped one: the server never said this.
             if (this.refusedByPolicy(res.request())) {
@@ -205,6 +292,44 @@ export class OracleMonitor {
      * errors look the same.
      */
     policyAttributed = 0;
+    isReplay = () => false;
+    replays = new ReplayLog();
+    /** Failures of the tester's own scout_request calls, kept out of the page's violations and counted. */
+    replayAttributed = 0;
+    /**
+     * The engine knows which requests are its own scout_request replays, by
+     * identity; their failures are the tester's probes, not the page's.
+     */
+    setReplayCheck(check) {
+        this.isReplay = check;
+    }
+    /** A scout_request call to this address is starting: the browser's echo of its failure is the tester's. */
+    replayStarted(url) {
+        this.replays.begin(url);
+    }
+    /** That call has returned; its echo is still expected for a short window. */
+    replayEnded(url) {
+        this.replays.end(url, Date.now());
+    }
+    /** Whether a page error was recorded since `since` (ms) and not yet drained. */
+    hasPageErrorSince(since) {
+        return this.buffer.some((v) => v.kind === "page_error" && Date.parse(v.at) >= since);
+    }
+    /**
+     * Re-rank the page errors this click raised that read as a router
+     * cancelling a route change (isRouteCancellation). Done before the drain, so
+     * the action's result, the session log and the report all see one verdict.
+     */
+    downgradeRouteCancellations(since, c) {
+        for (const v of this.buffer) {
+            if (v.kind !== "page_error" || v.severity !== "high" || Date.parse(v.at) < since)
+                continue;
+            if (!isRouteCancellation(v.detail, c))
+                continue;
+            v.severity = "medium";
+            v.detail += routeCancelNote(v.detail);
+        }
+    }
     embedOfRequest = () => null;
     embedRequests = new EmbedRequestLog();
     /**
@@ -241,7 +366,7 @@ export class OracleMonitor {
      * it is reported through here rather than by a page event.
      */
     noteContradiction(c, url) {
-        this.record({ kind: c.kind, severity: "high", detail: c.detail, url });
+        this.record({ kind: c.kind, severity: c.severity ?? "high", detail: c.detail, url });
     }
     /**
      * The page posted a token-shaped value with targetOrigin "*" (postmessage.ts).

package/dist/engine/provider.js

@@ -3,7 +3,8 @@ export class MalformedReply extends Error {
 }
 /** The most output one model call may produce. Large enough for a plan of steps; small enough that one reply cannot spend the budget. */
 export const MAX_OUTPUT_TOKENS = 16_000;
-const obj = (v) => (v && typeof v === "object" && !Array.isArray(v) ? v : null);
+/** A plain object as a record, or null for anything else (an array, null, a string). */
+export const obj = (v) => (v && typeof v === "object" && !Array.isArray(v) ? v : null);
 const num = (v) => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0);
 function parseArgs(raw) {
     if (raw === undefined || raw === null || raw === "")
@@ -31,7 +32,7 @@ export class AnthropicConversation {
             headers: { "content-type": "application/json", "x-api-key": key, "anthropic-version": "2023-06-01" },
             body: {
                 model: this.o.model,
-                max_tokens: MAX_OUTPUT_TOKENS,
+                max_tokens: this.o.maxOutputTokens ?? MAX_OUTPUT_TOKENS,
                 // The method and the tools are the same on every call: cached once, read back at a tenth of the price.
                 system: [{ type: "text", text: this.o.system, cache_control: { type: "ephemeral" } }],
                 tools: this.o.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.parameters })),
@@ -102,7 +103,7 @@ export class OpenAIConversation {
                 input: this.input,
                 tools: this.o.tools.map((t) => ({ type: "function", name: t.name, description: t.description, parameters: t.parameters, strict: false })),
                 reasoning: { effort: this.o.effort },
-                max_output_tokens: MAX_OUTPUT_TOKENS,
+                max_output_tokens: this.o.maxOutputTokens ?? MAX_OUTPUT_TOKENS,
                 // Stateless: nothing of the tested app is kept on the provider's side
                 // between calls, and the reasoning comes back encrypted so it can be
                 // handed back with the rest of the turn.