scenescout 3.14.1 → 3.16.0

package/dist/engine/crawl.js

@@ -0,0 +1,49 @@
+/**
+ * What a crawl makes of one route it loaded: whether an explicitly requested
+ * path joins the route contract, and the summary line it prints. Kept out of
+ * browser.ts so both rules can be table-tested without a browser.
+ */
+import { isNonPageRoute } from "./fingerprint.js";
+/** The crawl flag for a main area showing only its state, or null. */
+export function mainStateFlag(state) {
+    return state === "error" ? "ERROR-VIEW" : state === "loading" ? "STILL-LOADING" : null;
+}
+/**
+ * The route an explicitly requested crawl path adds to the route contract,
+ * or null. A path someone asked for, that answered with a page and stayed
+ * where it was asked, is a route of the app whether or not any link names
+ * it: left out, a sweep of forty extra paths left "Routes visited: 37/37"
+ * unchanged and the report understated what was checked. A path that
+ * redirected is not added (the page it landed on is the route that exists),
+ * nor one that failed, nor a dead end (an app that answers 200 for every
+ * path), nor one whose main area showed only an error view or a loading
+ * placeholder (a client-rendered app answers 200 and draws its not-found
+ * view), nor an API or download path. Added routes are remembered across
+ * runs, so a probe of made-up paths must not become part of the contract.
+ */
+export function crawledRoute(o) {
+    if (typeof o.status !== "number" || o.status >= 400 || o.deadEnd || o.mainState)
+        return null;
+    if (o.loginRedirect || o.landedRoute !== o.requestedRoute)
+        return null;
+    if (isNonPageRoute(o.requestedRoute))
+        return null;
+    return o.requestedRoute;
+}
+/**
+ * The crawl's one line for a route. A path that ended on another route says
+ * so, as scout_navigate does: "/old — 200 · 94 el" read as the old page
+ * answering, when the 200 and the 94 elements were the page it landed on.
+ * The sign-in bounce has its own flag (AUTH-REDIRECT) and is not repeated.
+ * `main` is what the page's main area holds (collector mainRegionTag), so
+ * "41 el" alone cannot hide a main area that rendered nothing.
+ */
+export function crawlLine(o, counts, flags) {
+    const redirected = !o.loginRedirect && o.landedRoute !== o.requestedRoute;
+    const allFlags = [...(redirected ? [`REDIRECTED → ${o.landedRoute}`] : []), ...flags];
+    return (`${o.path} — ${o.status} · ${counts.elements} el` +
+        (counts.main ? ` · ${counts.main}` : "") +
+        (counts.missingTestid ? ` · ${counts.missingTestid} no-testid` : "") +
+        (counts.unnamed ? ` · ${counts.unnamed} unnamed` : "") +
+        (allFlags.length ? ` · ${allFlags.join(" ")}` : ""));
+}

package/dist/engine/dedup.js

@@ -1,6 +1,6 @@
 /**
- * Measuring the finding-dedup decision against the answer keys, and an
- * optional model-backed judge to measure beside it.
+ * Measuring the finding-dedup decision against the answer keys, and the
+ * model-backed judge the store can ask beside its rule.
  *
  * The store's dedup rule (memory.ts, isDuplicateFinding) decides whether a
  * newly filed finding is one it already records. Its mistakes are costly both
@@ -12,14 +12,19 @@
  *
  * The model judge asks for a verdict (same / different / unsure) and a
  * probability through one tool call, so its answer is structured, not parsed
- * from prose. It is measured here and not used by the store: the rule stays
- * the only thing that dedups until the judge's Brier beats it on these pairs.
- * Everything here is pure; the network is the caller's, through `ask`.
+ * from prose. Measured on these pairs it beat the rule on both apps
+ * (docs/benchmark.md), and every pair it won was a merge the rule missed, so
+ * the store asks it only about filings the rule keeps apart (DedupJudge,
+ * below): a `scenescout ci` run by default, the MCP server when told to.
+ * Nothing here touches the network; the caller's `ask` does.
  */
 import { createHash } from "node:crypto";
 import { archiveApp, classify, findingText } from "./bench.js";
 import { MIN_FOR_A_VERDICT } from "./calibration.js";
-import { findingId, isDuplicateFinding } from "./memory.js";
+import { addUsage, dedupModeFromEnv, judgeKeyConfig, NO_USAGE } from "./ci.js";
+import { withWatchdog } from "./dispatch.js";
+import { findingId, isDuplicateFinding, titleSimilarity } from "./memory.js";
+import { obj } from "./provider.js";
 const pagesOf = (e) => [e.route, ...(e.alsoOn ?? [])];
 const norm = (s) => (s ?? "").toLowerCase().replace(/\s+/g, " ").trim();
 /**
@@ -178,10 +183,12 @@ export function parseJudgement(turn) {
  * Decide one pair. With no judge the rule decides, as the store does. With a
  * judge, the model decides when it answers same or different; when it fails
  * for any reason, or is unsure, the rule decides and the note says why, so a
- * run never silently becomes a run of the rule.
+ * run never silently becomes a run of the rule. `ruleVerdict` is the rule's
+ * verdict when the caller already has it: the store asks only about pairs its
+ * rule keeps apart, and its rule reads more of a finding than a pair carries.
  */
-export async function decideDuplicate(p, ask) {
-    const rule = () => ruleJudgement(p).verdict === "same";
+export async function decideDuplicate(p, ask, ruleVerdict) {
+    const rule = ruleVerdict ?? (() => ruleJudgement(p).verdict === "same");
     if (!ask)
         return { duplicate: rule(), by: "rule" };
     let parsed;
@@ -203,6 +210,319 @@ export async function decideDuplicate(p, ask) {
         return { duplicate: rule(), by: "rule", judgement: parsed.judgement, note: "the model judge was unsure; the current rule decided" };
     return { duplicate: parsed.judgement.verdict === "same", by: "model", judgement: parsed.judgement };
 }
+// ── the judge the store asks ────────────────────────────────────────────────
+/** Calls made about one filing at most: its most alike open findings on the page. */
+export const JUDGE_MAX_CALLS = 3;
+/** The longest one judge call may take. */
+export const JUDGE_CALL_MS = 15_000;
+/**
+ * The longest the judge may spend on one filing, every call together. Under
+ * scout_finding's 60-second watchdog, so a slow provider costs a filing its
+ * judge and never the filing.
+ */
+export const JUDGE_FILING_MS = 40_000;
+/** Failed calls in a row after which the judge is switched off for the rest of the run. */
+export const JUDGE_MAX_FAILURES = 3;
+/** The most output one judge call may produce: the answer is one short tool call (24 to 26 tokens when measured). */
+export const JUDGE_MAX_OUTPUT_TOKENS = 2_000;
+/**
+ * The most of a title, and of evidence, the store's judge shows the model. A
+ * filing with a pasted log for evidence still makes a question of bounded
+ * size, which a client answering the judge accepts (MAX_JUDGE_KICKOFF_CHARS).
+ */
+export const JUDGE_TITLE_CHARS = 500;
+export const JUDGE_EVIDENCE_CHARS = 2_000;
+const routeOf = (state) => state.split("#")[0];
+/** A judge time limit as a log line says it: "15s", or "20ms" for a test's. */
+export const durationText = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`);
+/**
+ * The model judge as the store asks it (memory.ts DuplicateJudge). It is
+ * asked only about a filing the rule keeps apart from everything stored, and
+ * only against the open findings on the filing's own page: the most alike
+ * title first, at most `maxCalls` of them, within `filingMs`. "Same" merges;
+ * anything else (different, unsure, a contradiction, a failed or slow call)
+ * leaves the rule's decision, which is to keep the filing apart. So the judge
+ * only ever adds merges the rule missed, which is where every pair it won in
+ * the measurement was (docs/benchmark.md).
+ *
+ * Each kind of fall-back is logged once, the cap once per page, and
+ * JUDGE_MAX_FAILURES failed calls in a row switch the judge off for the rest
+ * of the run. `log` reaches the operator; the caller redacts keys from it.
+ */
+export class DedupJudge {
+    ask;
+    o;
+    counts = { calls: 0, same: 0, different: 0, fellBack: 0, capped: 0, usage: { ...NO_USAGE }, ms: 0 };
+    failuresInARow = 0;
+    offReason = null;
+    logged = new Set();
+    maxCalls;
+    callMs;
+    filingMs;
+    now;
+    redact;
+    constructor(ask, o) {
+        this.ask = ask;
+        this.o = o;
+        this.maxCalls = o.maxCalls ?? JUDGE_MAX_CALLS;
+        this.callMs = o.callMs ?? JUDGE_CALL_MS;
+        this.filingMs = o.filingMs ?? JUDGE_FILING_MS;
+        this.now = o.now ?? Date.now;
+        this.redact = o.redact ?? ((text) => text);
+    }
+    /** What the judge has done this run. */
+    get tally() {
+        return this.counts;
+    }
+    /** Why the judge is no longer asked, or null while it is. */
+    get off() {
+        return this.offReason;
+    }
+    async judge(incoming, stored) {
+        try {
+            return await this.decide(incoming, stored);
+        }
+        catch (err) {
+            // decideDuplicate catches whatever a call throws, so this is a fault in this
+            // class. It would recur on every filing, so the judge stops here, and says why;
+            // the log gets the stack, the report only the message.
+            const message = err instanceof Error ? err.message : String(err);
+            this.switchOff(`a fault in the judge: ${message}`, err instanceof Error ? err.stack : undefined);
+            return null;
+        }
+    }
+    async decide(incoming, stored) {
+        if (this.offReason)
+            return null;
+        const route = routeOf(incoming.state);
+        const candidates = stored
+            .map((x, i) => ({ x, i, alike: titleSimilarity(x.title, incoming.title) }))
+            .filter(({ x }) => x.status !== "resolved" && routeOf(x.state) === route)
+            // The most alike first; of two equally alike, the newer.
+            .sort((p, q) => q.alike - p.alike || q.i - p.i);
+        if (candidates.length > this.maxCalls) {
+            this.counts.capped += 1;
+            this.once(`cap\u0000${route}`, `dedup judge: ${candidates.length} open findings on ${route} could be the one just filed there; the judge is asked about the ` +
+                `${this.maxCalls} most alike, and the rule, which kept it apart from all of them, decides the rest ` +
+                `(at most ${this.maxCalls} calls per filing; logged once per page)`);
+        }
+        const started = this.now();
+        for (const { x } of candidates.slice(0, this.maxCalls)) {
+            if (this.offReason)
+                break;
+            const left = this.filingMs - (this.now() - started);
+            if (left <= 0) {
+                this.once("time", `dedup judge: the ${durationText(this.filingMs)} a filing may take ran out; the rule decided the pairs left (logged once)`);
+                break;
+            }
+            const d = await decideDuplicate({ route, a: shown(x), b: shown(incoming) }, this.timed(Math.min(this.callMs, left)), () => false);
+            this.count(d);
+            if (d.by === "model" && d.duplicate && d.judgement)
+                return { sameAs: x.id, pSame: d.judgement.pSame };
+        }
+        return null;
+    }
+    /** The ask, ended at `ms` and counted. */
+    timed(ms) {
+        return async (system, tools, kickoff) => {
+            const started = this.now();
+            this.counts.calls += 1;
+            try {
+                const turn = await withWatchdog("dedup judge", this.ask(system, tools, kickoff, ms), ms, () => null);
+                if (!turn)
+                    throw new Error(`no answer within ${durationText(ms)}`);
+                this.counts.usage = addUsage(this.counts.usage, turn.usage);
+                return turn;
+            }
+            finally {
+                this.counts.ms += this.now() - started;
+            }
+        };
+    }
+    count(d) {
+        if (d.by === "model") {
+            this.failuresInARow = 0;
+            if (d.duplicate)
+                this.counts.same += 1;
+            else
+                this.counts.different += 1;
+            return;
+        }
+        this.counts.fellBack += 1;
+        if (d.failure !== "error") {
+            // Unsure, or an answer that contradicted itself: the provider answered, so neither counts towards switching off.
+            this.failuresInARow = 0;
+            this.once(d.failure ?? "unsure", `dedup judge: ${d.note ?? "no usable answer"} (later ones are counted, not logged)`);
+            return;
+        }
+        this.failuresInARow += 1;
+        this.once("error", `dedup judge: ${d.note ?? "a call failed"} (later failures are counted, not logged)`);
+        if (this.failuresInARow >= JUDGE_MAX_FAILURES)
+            this.switchOff(`${this.failuresInARow} failed calls in a row, the last: ${d.note ?? "no detail"}`);
+    }
+    switchOff(reason, detail) {
+        this.offReason = this.redact(reason);
+        this.o.log(this.redact(`dedup judge: switched off after ${reason}; the rule decides every filing for the rest of this run${detail ? `\n${detail}` : ""}`));
+    }
+    once(kind, line) {
+        if (this.logged.has(kind))
+            return;
+        this.logged.add(kind);
+        this.o.log(this.redact(line));
+    }
+    describe() {
+        const t = this.counts;
+        if (t.calls === 0 && !this.offReason)
+            return null;
+        const tokens = t.usage.input + t.usage.output;
+        return (`the rule, then the model judge (${this.o.label}) for filings the rule kept apart: ${t.calls} call(s), ` +
+            `${t.same} same, ${t.different} different, ${t.fellBack} left to the rule` +
+            (t.capped ? `; ${t.capped} filing(s) had more open findings on their page than the ${this.maxCalls} it asks about` : "") +
+            (tokens > 0 ? `; ${tokens.toLocaleString("en-US")} tokens` : "") +
+            `; ${(t.ms / 1000).toFixed(1)}s` +
+            (this.offReason ? `; switched off after ${this.offReason}` : ""));
+    }
+}
+/** A finding as the store's judge shows it: what was filed, each field cut to a length the question can carry. */
+function shown(f) {
+    const cut = (text, max) => (text.length <= max ? text : `${text.slice(0, max)}…`);
+    return { title: cut(f.title, JUDGE_TITLE_CHARS), category: f.category, ...(f.evidence ? { evidence: cut(f.evidence, JUDGE_EVIDENCE_CHARS) } : {}) };
+}
+/** Whether a client answers the judge's sampling requests: sampling with tools, and DEDUP_JUDGE_CAPABILITY. */
+export function clientAnswersJudge(caps) {
+    return !!caps?.sampling?.tools && !!caps.experimental?.[DEDUP_JUDGE_CAPABILITY];
+}
+/**
+ * How the MCP server dedups for a run: what an attach of the run named, else
+ * DEDUP_ENV. A judge asks through the client when the client answers the
+ * judge (a `scenescout ci` run, which holds the key, so the server never
+ * does), else with a key from the server's environment; with neither it is
+ * off, and the note says why. A malformed DEDUP_ENV or DEDUP_PROVIDER_ENV is
+ * refused, naming the variable.
+ */
+export function planDedup(choice, env, caps) {
+    const mode = choice ?? dedupModeFromEnv(env);
+    if (mode === "rule")
+        return { mode, note: "" };
+    if (clientAnswersJudge(caps)) {
+        const label = "the CI run's model";
+        return {
+            mode,
+            via: "client",
+            label,
+            note: `\nFinding dedup: the rule, then ${label}, asked about a filing the rule keeps apart from everything recorded, against the open findings on its page.`,
+        };
+    }
+    const key = judgeKeyConfig(env);
+    if (!key.ok)
+        return { mode, via: "off", why: key.error, note: `\n⚠ DEDUP JUDGE OFF: ${key.error}. The rule decides duplicates.` };
+    const { provider, model, effort } = key.resolved;
+    const label = `${provider} ${model}, effort ${effort}`;
+    return {
+        mode,
+        via: "key",
+        resolved: key.resolved,
+        key: key.key,
+        label,
+        note: `\nFinding dedup: the rule, then a model judge (${label}), asked about a filing the rule keeps apart from everything recorded, against the open findings on its page. ` +
+            `Each pair it is asked about (titles, categories, evidence and the page's path) is sent to ${provider}.`,
+    };
+}
+// ── the judge through an MCP client ─────────────────────────────────────────
+/**
+ * The client capability, under `experimental`, by which a client says it
+ * answers the dedup judge's sampling requests. `scenescout ci` declares it:
+ * the server's judge then asks the run's model through the run, and the key
+ * never enters the server's process. No other client is sent a sampling
+ * request.
+ */
+export const DEDUP_JUDGE_CAPABILITY = "scenescout/dedup-judge";
+/**
+ * The longest question a client answering the judge accepts. The store's judge
+ * cuts titles and evidence (JUDGE_TITLE_CHARS, JUDGE_EVIDENCE_CHARS), so its
+ * questions always fit; a longer one is refused, and counted as a failed call.
+ */
+export const MAX_JUDGE_KICKOFF_CHARS = 20_000;
+/** The sampling request (MCP sampling/createMessage, with tools) that carries one judge call to the client. */
+export function judgeSamplingParams(system, tools, kickoff) {
+    return {
+        systemPrompt: system,
+        messages: [{ role: "user", content: { type: "text", text: kickoff } }],
+        tools: tools.map((t) => ({ name: t.name, description: t.description, inputSchema: { ...t.parameters, type: "object" } })),
+        toolChoice: { mode: "required" },
+        maxTokens: JUDGE_MAX_OUTPUT_TOKENS,
+        includeContext: "none",
+    };
+}
+/** An Ask that puts the judge's question to the client. `createMessage` is the server's sampling request. */
+export function samplingAsk(createMessage, callMs = JUDGE_CALL_MS) {
+    return async (system, tools, kickoff, limitMs) => turnFromSampling(await createMessage(judgeSamplingParams(system, tools, kickoff), { timeout: Math.min(callMs, limitMs ?? callMs) }));
+}
+/** The client's answer read back as a turn, for parseJudgement. Usage is the client's to count, so the turn carries none. */
+export function turnFromSampling(result) {
+    const r = obj(result);
+    const content = r ? (Array.isArray(r.content) ? r.content : [r.content]) : [];
+    const blocks = content.map(obj).filter((b) => b !== null);
+    const calls = blocks
+        .filter((b) => b.type === "tool_use")
+        .map((b, i) => ({ id: typeof b.id === "string" ? b.id : `call_${i}`, name: typeof b.name === "string" ? b.name : "", input: b.input }));
+    const text = blocks
+        .filter((b) => b.type === "text")
+        .map((b) => String(b.text ?? ""))
+        .join("\n")
+        .trim();
+    const note = calls.length > 0
+        ? undefined
+        : r?.stopReason === "maxTokens"
+            ? "the model's reply was cut at its output limit"
+            : text
+                ? `the model answered without calling a tool: ${text.slice(0, 200)}`
+                : undefined;
+    return { text, calls, usage: { ...NO_USAGE }, ...(note ? { note } : {}) };
+}
+/**
+ * The client's side: the text of a sampling request shaped as the judge's
+ * question, checked for shape and size only. A client answering the judge
+ * sends this text on as given, under its own JUDGE_SYSTEM, JUDGE_TOOL and
+ * output cap, so the server cannot choose the system prompt, the tools or how
+ * much the model may write.
+ */
+export function judgeKickoffOf(params) {
+    const p = obj(params);
+    if (!p)
+        return { ok: false, error: "the request has no parameters" };
+    const tools = Array.isArray(p.tools) ? p.tools.map(obj) : [];
+    if (tools.length !== 1 || tools[0]?.name !== JUDGE_TOOL.name)
+        return { ok: false, error: `the request does not offer exactly the ${JUDGE_TOOL.name} tool` };
+    const messages = Array.isArray(p.messages) ? p.messages.map(obj) : [];
+    if (messages.length !== 1 || messages[0]?.role !== "user")
+        return { ok: false, error: "the request is not one user message" };
+    const content = messages[0].content;
+    const blocks = (Array.isArray(content) ? content : [content]).map(obj);
+    const text = blocks.length === 1 && blocks[0]?.type === "text" && typeof blocks[0].text === "string" ? blocks[0].text : "";
+    if (!text.trim())
+        return { ok: false, error: "the message is not one block of text" };
+    if (text.length > MAX_JUDGE_KICKOFF_CHARS)
+        return { ok: false, error: `the question is longer than ${MAX_JUDGE_KICKOFF_CHARS} characters` };
+    return { ok: true, kickoff: text };
+}
+/** The client's answer: the judge's turn as a sampling result, its tool call as tool_use content. */
+export function samplingResultOf(turn, model) {
+    const content = [];
+    for (const c of turn.calls) {
+        const input = obj(c.input);
+        if (input && !c.argsError)
+            content.push({ type: "tool_use", id: c.id, name: c.name, input });
+        else
+            content.push({ type: "text", text: `${c.name || "a tool"} was called with arguments that could not be read: ${c.argsError ?? "not an object"}` });
+    }
+    const said = [turn.text, turn.note].filter((s) => !!s && s.trim()).join("\n");
+    if (said)
+        content.push({ type: "text", text: said });
+    if (content.length === 0)
+        content.push({ type: "text", text: "(no answer)" });
+    return { model, role: "assistant", content, stopReason: content.some((b) => b.type === "tool_use") ? "toolUse" : "endTurn" };
+}
 /**
  * Accuracy, Brier and ECE of a decider over labelled pairs. `judgements[i]`
  * answers `labels[i]`; null is a judge that gave no usable answer. The