scenescout 3.11.1 → 3.13.0

package/dist/engine/calibration.js

@@ -21,7 +21,7 @@
  * Pure, so every rule here is table-tested.
  */
 import { requestsPair, statedRequests, templatedPathsMatch } from "./lane.js";
-import { failingSignatures } from "./memory.js";
+import { failingSignatures, isWorthALook } from "./memory.js";
 /**
  * Upper edge of each confidence bucket. Five is enough to see a shape and few
  * enough that each holds a usable count on a run of a few dozen decisions;
@@ -87,7 +87,8 @@ function usableConfidence(value) {
  */
 export function calibrate(decisions, findings) {
     const filedKeys = new Map();
-    for (const f of findings) {
+    // Filed as worth a look is not filed as a defect: a lane that called it a defect was not agreed with.
+    for (const f of findings.filter((x) => !isWorthALook(x))) {
         if (!f.evidence)
             continue;
         // A finding verified as still present is the most informative match, so it
@@ -102,8 +103,11 @@ export function calibrate(decisions, findings) {
     const claims = decisions.filter((d) => d.verdict === "defect" && d.evidence !== null);
     const checkable = claims.filter((d) => usableConfidence(d.confidence) !== null && joinKeys(d.evidence).size > 0);
     const unjoinable = claims.length - checkable.length;
+    const worthALook = decisions.filter((d) => d.verdict === "worth_a_look").length;
     if (checkable.length === 0)
-        return unjoinable > 0 ? { checkable: 0, filed: 0, stated: 0, buckets: [], ece: 0, verified: { present: 0, gone: 0, changed: 0 }, unjoinable } : null;
+        return unjoinable > 0 || worthALook > 0
+            ? { checkable: 0, filed: 0, stated: 0, buckets: [], ece: 0, verified: { present: 0, gone: 0, changed: 0 }, unjoinable, worthALook }
+            : null;
     const buckets = BUCKET_EDGES.map(() => ({ n: 0, conf: 0, hits: 0 }));
     const verified = { present: 0, gone: 0, changed: 0 };
     const matched = new Map();
@@ -147,7 +151,7 @@ export function calibrate(decisions, findings) {
         ece += (b.n / n) * Math.abs(meanConf - rate);
         out.push({ label: bucketLabel(i), decisions: b.n, stated: meanConf, filed: rate });
     });
-    return { checkable: n, filed, stated: stated / n, buckets: out, ece, verified, unjoinable };
+    return { checkable: n, filed, stated: stated / n, buckets: out, ece, verified, unjoinable, worthALook };
 }
 const pct = (x) => `${Math.round(x * 100)}%`;
 /**
@@ -163,7 +167,7 @@ export function formatCalibration(c) {
         // and a run that recorded none look identical when the section simply
         // vanishes, and the reader concludes the feature is broken — which is this
         // project's own definition of a silent path.
-        const had = c.checkable + c.unjoinable;
+        const had = c.checkable + c.unjoinable + c.worthALook;
         if (had === 0)
             return [];
         return [
@@ -171,6 +175,7 @@ export function formatCalibration(c) {
             ``,
             `Not enough to say yet: ${c.checkable} lane decision(s) could be checked${c.unjoinable > 0 ? ` (and ${c.unjoinable} could not be looked up at all)` : ""}, ` +
                 `and ${MIN_FOR_A_VERDICT} are needed before a calibration figure survives one of them changing.`,
+            ...worthALookNote(c),
             ``,
         ];
     }
@@ -194,6 +199,7 @@ export function formatCalibration(c) {
     if (c.unjoinable > 0) {
         lines.push(``, `${c.unjoinable} further decision(s) called a defect but named no failing endpoint, so nothing could be looked up for them. They are excluded above rather than counted as wrong.`);
     }
+    lines.push(...worthALookNote(c));
     const seen = c.verified.present + c.verified.gone + c.verified.changed;
     if (seen > 0) {
         lines.push(``, `${seen} of the findings those decisions matched ${seen === 1 ? "has" : "have"} since been re-tested with \`scout_verify\`: ${c.verified.present} still present, ${c.verified.gone} gone, ${c.verified.changed} changed. ` +
@@ -202,6 +208,16 @@ export function formatCalibration(c) {
     lines.push(``);
     return lines;
 }
+/** The line saying how many "worth a look" decisions were left out, and why; nothing when there were none. */
+function worthALookNote(c) {
+    if (c.worthALook === 0)
+        return [];
+    return [
+        ``,
+        `${c.worthALook} decision(s) were marked worth a look: real, and a defect only under a convention of the project the run cannot see. ` +
+            `Whether one was filed says nothing about whether the lane was right, so they are not scored.`,
+    ];
+}
 /** Below this, the number swings on a single decision and is worse than no number. */
 export const MIN_FOR_A_VERDICT = 8;
 /**
@@ -259,7 +275,8 @@ export const MAX_UNFILED_NAMED = 10;
 export function unfiledDefects(decisions, findings) {
     const keys = [];
     const filed = [];
-    for (const f of findings) {
+    // A defect filed only as worth a look is not in the report's findings, so it is still unfiled.
+    for (const f of findings.filter((x) => !isWorthALook(x))) {
         const text = `${f.evidence ?? ""} ${f.title}`;
         const requests = statedRequests(f.evidence ?? "");
         // The store's signatures, and the finding's own failing requests as

package/dist/engine/check.js

@@ -81,7 +81,31 @@ export const CHECK_RULES = {
         help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
     },
 };
-export const CHECK_RULE_IDS = Object.keys(CHECK_RULES);
+/**
+ * Rules whose measurement is exact and whose meaning depends on a convention
+ * of the project the check cannot see. SceneScout is used against any app, so
+ * it does not decide those conventions: these are listed as "worth a look",
+ * each naming the convention that would make it a defect, and never counted,
+ * given a severity or gated on, at any --fail-on. SARIF reports them at level
+ * "note". `convention` finishes the sentence "a defect only if your project
+ * uses …". --ignore takes them like any other rule.
+ */
+export const WORTH_A_LOOK_RULES = {
+    "off-grid-spacing": {
+        title: "Spacing off a 4px grid",
+        help: "More than a fifth of the page's paddings or vertical margins are not multiples of 4px. That matters where a project keeps a 4px spacing scale, and not where it uses another scale or none.",
+        convention: "a 4px spacing scale",
+    },
+    "indistinct-link": {
+        title: "Link styled like body text",
+        help: "Links with no underline, in the same colour as the page's body text. In running text a reader cannot tell them from the text around them; in navigation this styling is common, and the check cannot tell the two apart.",
+        convention: "a visible link style (an underline or a distinct colour) wherever links appear, navigation included",
+    },
+};
+export const CHECK_RULE_IDS = [...Object.keys(CHECK_RULES), ...Object.keys(WORTH_A_LOOK_RULES)];
+export function isWorthALookRule(rule) {
+    return Object.prototype.hasOwnProperty.call(WORTH_A_LOOK_RULES, rule);
+}
 const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
 /** Snapshot refs (`e12`) are numbered per run; evidence carrying them would never match itself twice. */
 function stripRefs(line) {
@@ -145,27 +169,36 @@ export function geometryRule(line) {
  * oracles caught while it ran. The second kind goes through the same rules as
  * a crawled page's, so a request that fails on load and again inside a flow is
  * one issue, not two.
+ *
+ * A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
+ * the same way; the two lists never share an entry.
  */
-export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
+export function checkFindings(routes, origin, ignore = [], flows = []) {
     const byKey = new Map();
+    const looks = new Map();
     const add = (rule, evidence, route, opts = {}) => {
         if (ignore.includes(rule))
             return;
         const clean = redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
         const key = `${rule}\u0000${clean}`;
-        const found = byKey.get(key);
+        const found = byKey.get(key) ?? looks.get(key);
         if (found) {
             if (!found.routes.includes(route))
                 found.routes.push(route);
             return;
         }
+        const fingerprint = createHash("sha256").update(key).digest("hex").slice(0, 32);
+        if (isWorthALookRule(rule)) {
+            looks.set(key, { rule, evidence: clean, routes: [route], convention: WORTH_A_LOOK_RULES[rule].convention, fingerprint });
+            return;
+        }
         byKey.set(key, {
             rule,
             severity: opts.embed ? "medium" : (opts.severity ?? CHECK_RULES[rule].severity),
             evidence: clean,
             routes: [route],
             ...(opts.embed ? { embed: opts.embed } : {}),
-            fingerprint: createHash("sha256").update(key).digest("hex").slice(0, 32),
+            fingerprint,
         });
     };
     for (const r of routes) {
@@ -206,7 +239,14 @@ export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
         if (f.outcome.status === "failed")
             add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
     }
-    return [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule));
+    return {
+        issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
+        worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
+    };
+}
+/** The defect tier of `checkFindings`: what counts, and what the gate reads. */
+export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
+    return checkFindings(routes, origin, ignore, flows).issues;
 }
 /**
  * Why a check has nothing to give a verdict on, or null when it measured the
@@ -383,7 +423,7 @@ export function parseCheckArgs(args, cwd) {
     if (paths?.some((p) => !p.startsWith("/")))
         return { ok: false, error: "--paths are paths on the app, each starting with /" };
     const ignore = list(flags.get("ignore")) ?? [];
-    const unknownRules = ignore.filter((r) => !(r in CHECK_RULES));
+    const unknownRules = ignore.filter((r) => !CHECK_RULE_IDS.includes(r));
     if (unknownRules.length > 0)
         return { ok: false, error: `unknown rule(s) in --ignore: ${unknownRules.join(", ")}. Rules: ${CHECK_RULE_IDS.join(", ")}` };
     const retest = flags.get("retest") ?? "on";
@@ -489,8 +529,17 @@ const RETEST_SARIF_RULE = {
     help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
     defaultConfiguration: { level: "error" },
 };
+function sarifLocation(route) {
+    return route === "(shared chrome)"
+        ? {
+            physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
+            message: { text: "the app's shared shell, on every page that renders it" },
+        }
+        : { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
+}
 export function toSarif(result, toolVersion) {
     const used = [...new Set(result.issues.map((i) => i.rule))];
+    const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
     const base = new URL(result.url);
     const reproducing = retestGateFailures(result);
     const refused = result.flows.filter((f) => f.outcome.status === "refused");
@@ -500,14 +549,20 @@ export function toSarif(result, toolVersion) {
         message: {
             text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.routes.length > 1 ? ` (on ${i.routes.length} routes)` : ""}${i.embed ? ` (in an embed of ${i.embed})` : ""}`,
         },
-        locations: i.routes.slice(0, 10).map((r) => r === "(shared chrome)"
-            ? {
-                physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
-                message: { text: "the app's shared shell, on every page that renders it" },
-            }
-            : { physicalLocation: { artifactLocation: { uri: r.replace(/^\//, ""), uriBaseId: "APP" } } }),
+        locations: i.routes.slice(0, 10).map(sarifLocation),
         partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
     }));
+    // Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
+    const lookResults = result.worthALook.map((o) => ({
+        ruleId: o.rule,
+        level: "note",
+        message: {
+            text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${o.routes.length > 1 ? ` (on ${o.routes.length} routes)` : ""}. A defect only if your project uses ${o.convention}.`,
+        },
+        locations: o.routes.slice(0, 10).map(sarifLocation),
+        partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
+        properties: { tier: "worth-a-look", convention: o.convention },
+    }));
     // Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
     const retestResults = reproducing.map((r) => ({
         ruleId: RETEST_SARIF_RULE.id,
@@ -537,6 +592,14 @@ export function toSarif(result, toolVersion) {
                                 help: { text: CHECK_RULES[id].help },
                                 defaultConfiguration: { level: SARIF_LEVEL[CHECK_RULES[id].severity] },
                             })),
+                            ...lookRules.map((id) => ({
+                                id,
+                                name: WORTH_A_LOOK_RULES[id].title,
+                                shortDescription: { text: WORTH_A_LOOK_RULES[id].title },
+                                help: { text: `${WORTH_A_LOOK_RULES[id].help} Worth a look: a defect only if your project uses ${WORTH_A_LOOK_RULES[id].convention}.` },
+                                defaultConfiguration: { level: "note" },
+                                properties: { tags: ["worth-a-look"] },
+                            })),
                             ...(reproducing.length > 0 ? [RETEST_SARIF_RULE] : []),
                         ],
                     },
@@ -553,7 +616,7 @@ export function toSarif(result, toolVersion) {
                     },
                 ],
                 originalUriBaseIds: { APP: { uri: `${base.origin}/` } },
-                results: [...issueResults, ...retestResults],
+                results: [...issueResults, ...lookResults, ...retestResults],
             },
         ],
     };
@@ -567,8 +630,10 @@ function cell(text) {
     return text
         .replace(/\\/g, "\\\\")
         .replace(/\|/g, "\\|")
-        .replace(/\s*\n\s*/g, " ");
+        .replace(/\s*[\r\n]\s*/g, " ");
 }
+/** The same escaping, for other markdown tables: the CI run's summary. */
+export const markdownCell = cell;
 /** A code span that the text cannot close: its fence is one backtick longer than any run of backticks inside it. */
 function code(text) {
     const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
@@ -596,7 +661,8 @@ export function formatCheck(result) {
         (passed
             ? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
             : `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
-        (unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : ""));
+        (unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
+        (result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
     // Right under the verdict: what a green check was allowed to do is part of what it means.
     lines.push("", `Settings — ${describeSettings(result)}`);
     for (const sev of CHECK_SEVERITIES) {
@@ -608,6 +674,12 @@ export function formatCheck(result) {
             lines.push(`- **${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`);
         }
     }
+    if (result.worthALook.length > 0) {
+        lines.push("", `## Worth a look (${result.worthALook.length})`, "", "Measured exactly, and defects only under a convention of your project that the check cannot see. They are not counted above and never fail the gate, at any --fail-on.", "");
+        for (const o of result.worthALook) {
+            lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
+        }
+    }
     lines.push("", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|");
     for (const r of result.routes) {
         const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
@@ -701,5 +773,7 @@ export function toSummaryJson(result, toolVersion) {
         skippedFlows: result.skippedFlows,
         retest: result.retest,
         issues: result.issues,
+        // Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
+        worthALook: result.worthALook,
     };
 }