scenescout 3.10.0 → 3.11.1

package/dist/engine/bench.js

@@ -79,6 +79,19 @@ const NonDefect = z
     counterExamples: z.array(z.string()).default([]),
 })
     .strict();
+const Contextual = z
+    .object({
+    id: z.string().min(1),
+    route: z.string().startsWith("/"),
+    title: z.string().min(1),
+    category: z.string().min(1),
+    /** The convention that decides it: where it would be a defect, and why the run cannot tell whether it holds here. */
+    reason: z.string().min(1),
+    match: z.array(Pattern).min(1),
+    examples: z.array(z.string()).default([]),
+    counterExamples: z.array(z.string()).default([]),
+})
+    .strict();
 const Key = z
     .object({
     app: z.string().min(1),
@@ -92,6 +105,14 @@ const Key = z
      */
     alsoReal: z.array(Entry).default([]),
     nonDefects: z.array(NonDefect).default([]),
+    /**
+     * Observations that are defects only under a convention the run cannot
+     * see — a spacing scale the project may or may not declare, say. Neither
+     * lane is wrong about them, so they are set aside: a finding matching one
+     * is neither correct nor a false positive, and a verdict on one is not
+     * scored for calibration.
+     */
+    contextual: z.array(Contextual).default([]),
 })
     .strict();
 /**
@@ -105,7 +126,7 @@ export function parseKey(raw) {
         const issue = parsed.error.issues[0];
         throw new Error(`The answer key is not valid at ${issue.path.join(".") || "(root)"}: ${issue.message}`);
     }
-    const ids = [...parsed.data.defects, ...parsed.data.alsoReal, ...parsed.data.nonDefects].map((e) => e.id);
+    const ids = [...parsed.data.defects, ...parsed.data.alsoReal, ...parsed.data.nonDefects, ...parsed.data.contextual].map((e) => e.id);
     const dup = ids.find((id, i) => ids.indexOf(id) !== i);
     if (dup)
         throw new Error(`The answer key uses the id ${JSON.stringify(dup)} twice.`);
@@ -119,7 +140,7 @@ export function parseKey(raw) {
 }
 /** A short, stable fingerprint of a key, so a scorecard says which key produced it. */
 export function keyHash(key) {
-    const canonical = JSON.stringify({ defects: key.defects, alsoReal: key.alsoReal, nonDefects: key.nonDefects });
+    const canonical = JSON.stringify({ defects: key.defects, alsoReal: key.alsoReal, nonDefects: key.nonDefects, contextual: key.contextual });
     return createHash("sha256").update(canonical).digest("hex").slice(0, 10);
 }
 const RANK = { minimal: 0, medium: 1, extensive: 2 };
@@ -138,12 +159,19 @@ function matches(patterns, text) {
  * match would quietly count a finding about two defects as one, and letting a
  * non-defect win everywhere turned realistic rewordings of real defects into
  * false positives with a confident explanation beside them.
+ *
+ * A contextual entry is one more claimant on the same terms as a real one: it
+ * wins over nothing, no non-defect can override it, and text it shares with
+ * any other entry is ambiguous. Setting a finding aside removes it from
+ * precision, so a pattern that let it win would hide real defects from the
+ * score as quietly as a greedy non-defect once did.
  */
 export function classify(text, key) {
     const nd = key.nonDefects.filter((e) => matches(e.match, text));
     const real = [
         ...key.defects.filter((e) => matches(e.match, text)).map((entry) => ({ kind: "defect", entry })),
         ...key.alsoReal.filter((e) => matches(e.match, text)).map((entry) => ({ kind: "alsoReal", entry })),
+        ...key.contextual.filter((e) => matches(e.match, text)).map((entry) => ({ kind: "contextual", entry })),
     ];
     if (nd.length > 1)
         return { kind: "ambiguous", ids: [...nd, ...real.map((m) => m.entry)].map((e) => e.id) };
@@ -171,7 +199,7 @@ export function decisionText(d) {
  * A key that disagrees with its own examples is wrong before any run is
  * scored. Returns every title or example — non-defects' included — that
  * classifies anywhere but its own entry, and every counter-example that
- * classifies TO its entry.
+ * classifies TO its entry. Contextual entries are held to the same rule.
  */
 export function lintKey(key) {
     const problems = [];
@@ -195,11 +223,16 @@ export function lintKey(key) {
                 check(e.id, kind, t, false);
         }
     }
-    for (const e of key.nonDefects) {
-        for (const t of [e.title, ...e.examples])
-            check(e.id, "nonDefect", t, true);
-        for (const t of e.counterExamples)
-            check(e.id, "nonDefect", t, false);
+    for (const [kind, list] of [
+        ["nonDefect", key.nonDefects],
+        ["contextual", key.contextual],
+    ]) {
+        for (const e of list) {
+            for (const t of [e.title, ...e.examples])
+                check(e.id, kind, t, true);
+            for (const t of e.counterExamples)
+                check(e.id, kind, t, false);
+        }
     }
     return problems;
 }
@@ -208,8 +241,8 @@ export function lintKey(key) {
  *
  * "defect" on a planted or also-real defect is right, and on a known
  * non-defect is wrong. "not_a_defect" is the reverse. An "unsure" verdict, a
- * dismissal as another lane's, a decision the key does not name, and one it
- * names ambiguously are not scored — the same rule the in-product calibration
+ * dismissal as another lane's, a decision the key does not name, one it
+ * names ambiguously, and one about a contextual entry are not scored — the same rule the in-product calibration
  * follows, for the same reason.
  */
 export function judgeDecision(d, key) {
@@ -218,7 +251,7 @@ export function judgeDecision(d, key) {
     if (isScopeDismissal(d))
         return null;
     const m = classify(decisionText(d), key);
-    if (!m || m.kind === "ambiguous")
+    if (!m || m.kind === "ambiguous" || m.kind === "contextual")
         return null;
     const isDefect = m.kind !== "nonDefect";
     return d.verdict === "defect" ? isDefect : !isDefect;
@@ -233,7 +266,19 @@ export function calibrateAgainstKey(decisions, key) {
     if (decisions.length === 0)
         return null;
     const buckets = BUCKET_EDGES.map(() => ({ n: 0, conf: 0, right: 0 }));
-    const out = { judged: 0, correct: 0, notInKey: 0, ambiguous: 0, badConfidence: 0, unsure: 0, outOfScope: 0, buckets: [], ece: 0, brier: 0 };
+    const out = {
+        judged: 0,
+        correct: 0,
+        notInKey: 0,
+        ambiguous: 0,
+        badConfidence: 0,
+        unsure: 0,
+        outOfScope: 0,
+        contextual: 0,
+        buckets: [],
+        ece: 0,
+        brier: 0,
+    };
     for (const d of decisions) {
         if (d.verdict === "unsure") {
             out.unsure += 1;
@@ -257,6 +302,10 @@ export function calibrateAgainstKey(decisions, key) {
             out.ambiguous += 1;
             continue;
         }
+        if (m.kind === "contextual") {
+            out.contextual += 1;
+            continue;
+        }
         const right = d.verdict === "defect" ? m.kind !== "nonDefect" : m.kind === "nonDefect";
         out.judged += 1;
         if (right)
@@ -290,6 +339,7 @@ export function score(key, findings, decisions, level = "medium") {
     const falsePositives = [];
     const unknown = [];
     const ambiguous = [];
+    const contextual = [];
     let correct = 0;
     for (const f of findings) {
         const m = classify(findingText(f), key);
@@ -305,6 +355,10 @@ export function score(key, findings, decisions, level = "medium") {
             falsePositives.push({ id: m.entry.id, title: f.title, why: m.entry.why, severity: f.severity });
             continue;
         }
+        if (m.kind === "contextual") {
+            contextual.push({ id: m.entry.id, title: f.title });
+            continue;
+        }
         correct += 1;
         if (m.kind === "defect") {
             const list = hitsByDefect.get(m.entry.id) ?? [];
@@ -355,6 +409,7 @@ export function score(key, findings, decisions, level = "medium") {
         findings: findings.length,
         correct,
         falsePositives,
+        contextual,
         unknown,
         ambiguous,
         duplicates,
@@ -367,12 +422,14 @@ const pct = (n, d) => (d === 0 ? "—" : `${Math.round((n / d) * 100)}%`);
  * Precision, with its bounds. The labelled ratio leaves unlabelled findings out
  * of the denominator, so on its own it cannot move when a change adds five new
  * false claims nobody has judged yet. The bounds can: the lower one counts
- * every open finding as wrong, the upper one as right.
+ * every open finding as wrong, the upper one as right. Findings set aside as
+ * contextual are in neither: they are not claims the run could get right.
  */
 export function precisionBounds(c) {
     const labelled = c.correct + c.falsePositives.length;
     const open = c.unknown.length + c.ambiguous.length;
-    return { labelled: `${c.correct}/${labelled} (${pct(c.correct, labelled)})`, low: pct(c.correct, c.findings), high: pct(c.correct + open, c.findings) };
+    const scored = c.findings - c.contextual.length;
+    return { labelled: `${c.correct}/${labelled} (${pct(c.correct, labelled)})`, low: pct(c.correct, scored), high: pct(c.correct + open, scored) };
 }
 /** The scorecard as a person reads it. Leads with the two numbers, then says what each is made of. */
 export function formatScorecard(c) {
@@ -383,9 +440,13 @@ export function formatScorecard(c) {
         ``,
         `Recall     ${c.found.length}/${c.expected} (${pct(c.found.length, c.expected)}) of the planted defects expected at this level`,
         `Precision  ${p.labelled} of the findings the key can label` +
-            (open ? ` — ${open} of ${c.findings} unlabelled, so between ${p.low} and ${p.high} of all findings` : ""),
-        ``,
+            (open
+                ? ` — ${open} of ${c.findings - c.contextual.length} unlabelled, so between ${p.low} and ${p.high} of ${c.contextual.length ? "the findings not set aside" : "all findings"}`
+                : ""),
     ];
+    if (c.contextual.length)
+        lines.push(`Set aside  ${c.contextual.length} of ${c.findings} finding(s): defects only under a convention the run cannot see, so in neither count`);
+    lines.push(``);
     if (c.missed.length)
         lines.push(`Missed: ${c.missed.join(", ")}`);
     if (c.judgedNotFiled.length)
@@ -397,6 +458,11 @@ export function formatScorecard(c) {
         for (const fp of c.falsePositives)
             lines.push(`  [${fp.severity}] ${fp.title} — ${fp.why}`);
     }
+    if (c.contextual.length) {
+        lines.push(``, `Set aside as contextual (${c.contextual.length}):`);
+        for (const x of c.contextual)
+            lines.push(`  ${x.title} (${x.id})`);
+    }
     if (c.ambiguous.length) {
         lines.push(``, `Ambiguous (${c.ambiguous.length}) — the key claims each twice; sharpen it:`);
         for (const a of c.ambiguous)
@@ -425,6 +491,7 @@ export function formatScorecard(c) {
             k.badConfidence && `${k.badConfidence} with an unusable confidence`,
             k.unsure && `${k.unsure} unsure`,
             k.outOfScope && `${k.outOfScope} dismissed as another lane's`,
+            k.contextual && `${k.contextual} about things that are defects only under a convention the run cannot see`,
         ].filter(Boolean);
         lines.push(``, k.judged === 0
             ? `Lane calibration against the key: nothing the key could judge` + (skipped.length ? ` (${skipped.join(", ")})` : "")
@@ -460,6 +527,13 @@ export function runDate(decisions, given) {
         throw new Error(`The run's last decision was made on ${last}; it cannot be dated ${given}.`);
     return given ?? last;
 }
+/**
+ * The order `--all` lists runs in: by date, then by name with its numbers
+ * compared as numbers, so run-10 follows run-9 rather than run-1.
+ */
+export function byRunOrder(a, b) {
+    return a.date.localeCompare(b.date) || a.run.localeCompare(b.run, undefined, { numeric: true });
+}
 /**
  * Paths into a machine's own directories, which a finding's evidence can pick
  * up from a stack trace or an upload. An archive is committed, and a home
@@ -469,8 +543,9 @@ const LOCAL_PATH_RE = /(?:\/Users\/[^/\s"']+|\/home\/[^/\s"']+|\/private\/tmp|\/
 export function sanitize(text) {
     return text.replace(LOCAL_PATH_RE, "<path>");
 }
-export function toArchive(run, date, note, findings, decisions) {
+export function toArchive(run, date, note, findings, decisions, app) {
     return {
+        app,
         run,
         date,
         note,
@@ -483,3 +558,65 @@ export function toArchive(run, date, note, findings, decisions) {
         decisions: decisions.map((d) => ({ ...d, observation: sanitize(d.observation), evidence: d.evidence === null ? null : sanitize(d.evidence) })),
     };
 }
+// ── which app, and so which key ─────────────────────────────────────────────
+/**
+ * The app a run is for when nothing says otherwise. Every archive made before
+ * there was a second benchmark app is this one's, and carries no `app`.
+ */
+export const DEFAULT_APP = "demo";
+/** The app an archive was made against. */
+export function archiveApp(a) {
+    return a.app ?? DEFAULT_APP;
+}
+/**
+ * The app whose key scores a run. Scoring a run against another app's key
+ * produces a plausible wrong number — every planted defect "missed", every
+ * finding "unlabelled" — so a request that disagrees with what the archive
+ * records is refused rather than obeyed.
+ *
+ * `requested` is what the caller asked for (`--app`), `archived` what the
+ * run's archive records, and `known` the apps that have a key.
+ */
+export function chooseApp(opts) {
+    const { requested, archived, known } = opts;
+    for (const [what, app] of [
+        ["--app", requested],
+        ["The archive's app", archived],
+    ]) {
+        if (app !== undefined && !known.includes(app))
+            throw new Error(`${what} ${JSON.stringify(app)} is not a benchmark app; the apps are ${known.join(", ")}.`);
+    }
+    if (requested !== undefined && archived !== undefined && requested !== archived)
+        throw new Error(`This run was archived for the ${archived} app; scoring it against the ${requested} app's key would score the wrong answers.`);
+    return requested ?? archived ?? DEFAULT_APP;
+}
+/**
+ * Which benchmark app a key file belongs to, by the name the key gives its
+ * app, or undefined for a key no benchmark app ships (a draft, a test key).
+ * A copy of one app's key, edited or not, still names that app.
+ */
+export function appOfKey(key, keys) {
+    return Object.keys(keys).find((app) => keys[app].app === key.app);
+}
+/**
+ * Refuse a key file that belongs to another benchmark app than the archive
+ * being scored: `--key` naming the held-out key scored a demo archive against
+ * the wrong answers without a word.
+ */
+export function checkKeyForArchive(key, archive, keys) {
+    const keyApp = appOfKey(key, keys);
+    const runApp = archiveApp(archive);
+    if (keyApp !== undefined && keyApp !== runApp)
+        throw new Error(`This run was archived for the ${runApp} app, and that key is the ${keyApp} app's; scoring one against the other would score the wrong answers.`);
+}
+/** Archives grouped by app, in the order each app is first met, each group in `byRunOrder`. */
+export function groupByApp(archives) {
+    const groups = new Map();
+    for (const a of archives) {
+        const app = archiveApp(a);
+        groups.set(app, [...(groups.get(app) ?? []), a]);
+    }
+    for (const list of groups.values())
+        list.sort(byRunOrder);
+    return groups;
+}