scenescout 3.14.1 → 3.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +94 -0
  2. package/README.md +90 -18
  3. package/dist/browsers.js +28 -0
  4. package/dist/check-run.js +211 -24
  5. package/dist/ci-run.js +374 -62
  6. package/dist/cli.js +221 -47
  7. package/dist/commands.js +42 -4
  8. package/dist/engine/baseline.js +377 -0
  9. package/dist/engine/brief.js +16 -7
  10. package/dist/engine/browser.js +1660 -386
  11. package/dist/engine/calibration.js +61 -30
  12. package/dist/engine/capture.js +164 -0
  13. package/dist/engine/check.js +258 -68
  14. package/dist/engine/ci-lanes.js +215 -0
  15. package/dist/engine/ci.js +233 -18
  16. package/dist/engine/claims.js +234 -12
  17. package/dist/engine/collector.js +851 -78
  18. package/dist/engine/crawl.js +49 -0
  19. package/dist/engine/dedup.js +329 -9
  20. package/dist/engine/design.js +355 -43
  21. package/dist/engine/export.js +877 -0
  22. package/dist/engine/fingerprint.js +92 -4
  23. package/dist/engine/flow.js +18 -6
  24. package/dist/engine/forms.js +198 -18
  25. package/dist/engine/journey.js +29 -1
  26. package/dist/engine/lane.js +13 -3
  27. package/dist/engine/launch.js +45 -6
  28. package/dist/engine/limits.js +7 -0
  29. package/dist/engine/live-page.js +49 -2
  30. package/dist/engine/live.js +4 -1
  31. package/dist/engine/memory.js +668 -70
  32. package/dist/engine/open.js +118 -0
  33. package/dist/engine/oracles.js +167 -2
  34. package/dist/engine/plain.js +268 -0
  35. package/dist/engine/png.js +127 -0
  36. package/dist/engine/policy.js +379 -9
  37. package/dist/engine/probes.js +3 -2
  38. package/dist/engine/profiles.js +45 -9
  39. package/dist/engine/project-folder.js +191 -0
  40. package/dist/engine/provider.js +4 -3
  41. package/dist/engine/refresh.js +338 -25
  42. package/dist/engine/replay.js +63 -10
  43. package/dist/engine/report.js +248 -40
  44. package/dist/engine/request.js +317 -23
  45. package/dist/engine/sarif.js +120 -0
  46. package/dist/engine/scripted-login.js +342 -62
  47. package/dist/engine/settle.js +67 -0
  48. package/dist/engine/signed-in.js +256 -0
  49. package/dist/engine/tickets.js +671 -0
  50. package/dist/engine/unload.js +3 -2
  51. package/dist/export-run.js +633 -0
  52. package/dist/first-run.js +581 -0
  53. package/dist/installer.js +267 -3
  54. package/dist/intake.js +104 -0
  55. package/dist/login-run.js +386 -71
  56. package/dist/mcp-server.js +637 -68
  57. package/dist/playbook.js +5 -0
  58. package/package.json +6 -3
  59. package/skills/scenescout/SKILL.md +51 -18
@@ -12,9 +12,11 @@
12
12
  */
13
13
  import { createHash } from "node:crypto";
14
14
  import { BROWSER_ENGINES } from "../browsers.js";
15
+ import { BASELINE_MODES, baselineEvidence, baselineFingerprint, countStatuses, describeChange, parseThreshold, VISUAL_RULE, } from "./baseline.js";
15
16
  import { flowStepEvidence } from "./flow.js";
16
17
  import { parseLimitFlag } from "./limits.js";
17
- import { redactSecrets } from "./memory.js";
18
+ import { redactRoute, redactSecrets } from "./memory.js";
19
+ import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifFileFor, sarifLocation } from "./sarif.js";
18
20
  import { httpErrorDetail, httpStatusOf } from "./oracles.js";
19
21
  export const CHECK_SEVERITIES = ["high", "medium", "low"];
20
22
  export const FAIL_ON = [...CHECK_SEVERITIES, "never"];
@@ -65,7 +67,11 @@ export const CHECK_RULES = {
65
67
  title: "Control covered by pinned chrome",
66
68
  help: "A fixed or sticky element sits on top of the control, so clicks land on it instead.",
67
69
  },
68
- "clipped-control": { severity: "medium", title: "Control unreachable", help: "The control is clipped inside a container that cannot scroll." },
70
+ "clipped-control": {
71
+ severity: "medium",
72
+ title: "Control unreachable",
73
+ help: "The control is clipped inside a container that cannot scroll and has no control beside it to page it.",
74
+ },
69
75
  "offpage-control": { severity: "medium", title: "Control outside the page", help: "The control is laid out where no scrolling can reach it." },
70
76
  "overlapping-controls": { severity: "low", title: "Controls overlap", help: "Two controls in the same layer cover most of each other." },
71
77
  "broken-image": { severity: "medium", title: "Broken image", help: "The browser could not render the image." },
@@ -78,7 +84,11 @@ export const CHECK_RULES = {
78
84
  contrast: { severity: "low", title: "Text contrast below WCAG", help: "Text needs 4.5:1 (3:1 when large) against its background." },
79
85
  "focus-indicator": { severity: "low", title: "No visible focus indicator", help: "Tabbing to the control changes nothing on screen." },
80
86
  "horizontal-scroll": { severity: "medium", title: "Page scrolls sideways", help: "Content is wider than the viewport." },
81
- "tiny-target": { severity: "low", title: "Small click target", help: "Below the 24px WCAG 2.2 target-size minimum." },
87
+ "tiny-target": {
88
+ severity: "low",
89
+ title: "Small click target",
90
+ help: "Below the 24px WCAG 2.2 target-size minimum, measured on what the user clicks (a native input's label or drop zone), with another target inside its 24px circle.",
91
+ },
82
92
  "clipped-text": { severity: "low", title: "Text clipped", help: "Text is wider than its box and cut off without an ellipsis." },
83
93
  "image-aspect": { severity: "low", title: "Image distorted", help: "The rendered box does not match the image's proportions." },
84
94
  "flow-step-failed": {
@@ -86,6 +96,11 @@ export const CHECK_RULES = {
86
96
  title: "Saved flow broke",
87
97
  help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
88
98
  },
99
+ [VISUAL_RULE]: {
100
+ severity: "high",
101
+ title: "Differs from its visual baseline",
102
+ help: "A page or element listed in the baselines' targets.json no longer looks like its baseline (--baseline compare): more of its pixels changed than --baseline-threshold allows, or its size changed. It is also filed when the page or element could not be captured at all, or its baseline cannot be used (half there, unreadable, or taken with other settings). An intended change is approved by running the check with --baseline update (and committing the new baseline, where the project keeps baselines in a folder it commits).",
103
+ },
89
104
  };
90
105
  /**
91
106
  * Rules whose measurement is exact and whose meaning depends on a convention
@@ -107,12 +122,19 @@ export const WORTH_A_LOOK_RULES = {
107
122
  help: "Links with no underline, in the same colour as the page's body text. In running text a reader cannot tell them from the text around them; in navigation this styling is common, and the check cannot tell the two apart.",
108
123
  convention: "a visible link style (an underline or a distinct colour) wherever links appear, navigation included",
109
124
  },
125
+ "scrolled-out-controls": {
126
+ title: "Controls scrolled out of view sideways",
127
+ help: "Controls inside a horizontally scrolling container (a wide table's last column, say) lie outside its visible width at this viewport, so only a sideways scroll of the container shows them. A wide table that scrolls is a common, deliberate layout; it matters where a project keeps every row's actions in view.",
128
+ convention: "row actions and other controls that stay in view without a sideways scroll of their container at this viewport width",
129
+ },
110
130
  };
111
131
  export const CHECK_RULE_IDS = [...Object.keys(CHECK_RULES), ...Object.keys(WORTH_A_LOOK_RULES)];
112
132
  export function isWorthALookRule(rule) {
113
133
  return Object.prototype.hasOwnProperty.call(WORTH_A_LOOK_RULES, rule);
114
134
  }
115
- const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
135
+ export const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
136
+ /** The route a defect of the app's shared shell is charged to: one fix, however many pages show it. */
137
+ export const SHARED_CHROME_ROUTE = "(shared chrome)";
116
138
  /** Snapshot refs (`e12`) are numbered per run; evidence carrying them would never match itself twice. */
117
139
  function stripRefs(line) {
118
140
  return line.replace(/\be\d+\s+/g, "").trim();
@@ -166,6 +188,8 @@ export function geometryRule(line) {
166
188
  return "offpage-control";
167
189
  if (/ overlaps /.test(line))
168
190
  return "overlapping-controls";
191
+ if (/ scrolled out of view inside a horizontally scrolling container /.test(line))
192
+ return "scrolled-out-controls";
169
193
  return "layout-issue";
170
194
  }
171
195
  /**
@@ -180,16 +204,28 @@ export function geometryRule(line) {
180
204
  *
181
205
  * A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
182
206
  * the same way; the two lists never share an entry.
207
+ *
208
+ * Visual baselines add one issue per target that is not met (baseline.ts
209
+ * decides which). Each target is its own fact already, so these skip the
210
+ * dedup, and their fingerprint comes from the target, not the evidence: the
211
+ * evidence carries this run's percentage, and the same target changing by a
212
+ * different amount is still the same alert.
183
213
  */
184
- export function checkFindings(routes, origin, ignore = [], flows = []) {
214
+ export function checkFindings(routes, origin, ignore = [], flows = [], baselines = null) {
185
215
  const byKey = new Map();
186
216
  const looks = new Map();
217
+ /** Evidence as it is written: no origin, no secret, and bounded. */
218
+ const cleanEvidence = (evidence) => redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
187
219
  const add = (rule, evidence, route, opts = {}) => {
188
220
  if (ignore.includes(rule))
189
221
  return;
190
- const clean = redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
222
+ const clean = cleanEvidence(evidence);
191
223
  const key = `${rule}\u0000${clean}`;
192
- const found = byKey.get(key) ?? looks.get(key);
224
+ const issue = byKey.get(key);
225
+ // A fact a crawled page and a flow both show points at the flow: the flow is the file that reproduces it.
226
+ if (issue && opts.flow && !issue.flow)
227
+ issue.flow = opts.flow;
228
+ const found = issue ?? looks.get(key);
193
229
  if (found) {
194
230
  if (!found.routes.includes(route))
195
231
  found.routes.push(route);
@@ -206,6 +242,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
206
242
  evidence: clean,
207
243
  routes: [route],
208
244
  ...(opts.embed ? { embed: opts.embed } : {}),
245
+ ...(opts.flow ? { flow: opts.flow } : {}),
209
246
  fingerprint,
210
247
  });
211
248
  };
@@ -238,17 +275,32 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
238
275
  for (const p of r.placeholderOnly)
239
276
  add("placeholder-only-label", p, route);
240
277
  for (const d of r.design)
241
- add(d.rule, d.detail, d.chrome ? "(shared chrome)" : route);
278
+ add(d.rule, d.detail, d.chrome ? SHARED_CHROME_ROUTE : route);
242
279
  }
243
280
  for (const f of flows) {
244
281
  for (const { path, violation } of f.violations)
245
- add(violationRule(violation), violation.detail, path, { embed: violation.embed });
282
+ add(violationRule(violation), violation.detail, path, { embed: violation.embed, flow: f.file });
246
283
  // A refused step is not the app's defect: the check reports it as "could not run" (refusedFlowReason).
247
284
  if (f.outcome.status === "failed")
248
- add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
285
+ add("flow-step-failed", flowStepEvidence(f), f.outcome.path, { flow: f.file });
286
+ }
287
+ const visual = [];
288
+ if (baselines && !ignore.includes(VISUAL_RULE)) {
289
+ for (const r of baselines.results) {
290
+ const evidence = baselineEvidence(r, baselines);
291
+ if (evidence === null)
292
+ continue;
293
+ visual.push({
294
+ rule: VISUAL_RULE,
295
+ severity: CHECK_RULES[VISUAL_RULE].severity,
296
+ evidence: cleanEvidence(evidence),
297
+ routes: [r.path],
298
+ fingerprint: baselineFingerprint(baselines.engine, r),
299
+ });
300
+ }
249
301
  }
250
302
  return {
251
- issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
303
+ issues: [...byKey.values(), ...visual].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
252
304
  worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
253
305
  };
254
306
  }
@@ -290,15 +342,8 @@ export function withoutOwnResponse(r) {
290
342
  const own = httpErrorDetail("GET", r.url.replace(/#.*$/, ""), r.status);
291
343
  return { ...r, violations: r.violations.filter((v) => !(v.kind === "http_error" && v.detail === own)) };
292
344
  }
293
- /**
294
- * Routes are links the app printed, and a link can carry a token
295
- * (`?reset=…`, `?api_key=…`). Evidence is redacted where issues are built;
296
- * this does the same for every route string before anything is written.
297
- */
298
- export function redactRoute(route) {
299
- // The trailing "[n secrets redacted]" note belongs to prose; in a route it would read as part of the path.
300
- return redactSecrets(route).replace(/ \[\d+ secrets? redacted\]$/, "");
301
- }
345
+ /** Lives beside redactSecrets so the baselines' file names (baseline.ts) use it too; exported here as before. */
346
+ export { redactRoute };
302
347
  export function redactRoutes(routes) {
303
348
  const clean = redactRoute;
304
349
  return routes.map((r) => ({
@@ -308,6 +353,16 @@ export function redactRoutes(routes) {
308
353
  violations: r.violations.map((v) => ({ ...v, url: clean(v.url) })),
309
354
  }));
310
355
  }
356
+ /**
357
+ * The same redaction for visual baselines: a target's path, and the reason a
358
+ * picture could not be taken, which can quote a URL the app redirected to.
359
+ */
360
+ export function redactBaselineRun(run) {
361
+ return {
362
+ ...run,
363
+ results: run.results.map((r) => ({ ...r, path: redactRoute(r.path), ...(r.detail !== undefined ? { detail: redactSecrets(r.detail) } : {}) })),
364
+ };
365
+ }
311
366
  /** The same redaction for what flows measured: the page paths and the request URLs their violations quote. */
312
367
  export function redactFlowRuns(runs) {
313
368
  return runs.map((f) => ({
@@ -365,9 +420,19 @@ export const CHECK_OPTION_NAMES = [
365
420
  "gate-retests",
366
421
  "action-timeout-ms",
367
422
  "nav-timeout-ms",
423
+ "baseline",
424
+ "baselines",
425
+ "baseline-threshold",
426
+ "sarif-file-anchor",
368
427
  ];
369
428
  export const MAX_CHECK_ROUTES = 150;
429
+ /** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
430
+ export const MAX_DISCOVERY_ROUNDS = 6;
370
431
  export const DEFAULT_CHECK_ROUTES = 50;
432
+ /** A path given on the command line, resolved from the directory the command runs in; the same on every platform. */
433
+ export function resolveArgPath(cwd, p) {
434
+ return p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`;
435
+ }
371
436
  /** Parse `scenescout check` arguments. Every mistake is a sentence, never a half-configured run. */
372
437
  export function parseCheckArgs(args, cwd) {
373
438
  const positional = [];
@@ -461,7 +526,23 @@ export function parseCheckArgs(args, cwd) {
461
526
  const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
462
527
  if (!gateRetests)
463
528
  return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
464
- const resolve = (p) => (p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`);
529
+ const baseline = oneOf("baseline", BASELINE_MODES, "off");
530
+ if (!baseline)
531
+ return { ok: false, error: `--baseline must be one of ${BASELINE_MODES.join(", ")}` };
532
+ const threshold = parseThreshold(flags.get("baseline-threshold"));
533
+ if (!threshold.ok)
534
+ return threshold;
535
+ const baselinesDir = flags.get("baselines");
536
+ if (baselinesDir !== undefined && baselinesDir.trim() === "")
537
+ return { ok: false, error: "--baselines needs a directory" };
538
+ // A setting that would do nothing is a mistake to point out, not to ignore: the check would run without the baselines asked for.
539
+ if (baseline === "off" && (baselinesDir !== undefined || flags.has("baseline-threshold"))) {
540
+ return { ok: false, error: "--baselines and --baseline-threshold apply only with --baseline compare or --baseline update" };
541
+ }
542
+ const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
543
+ if (anchor && !anchor.ok)
544
+ return anchor;
545
+ const resolve = (p) => resolveArgPath(cwd, p);
465
546
  return {
466
547
  ok: true,
467
548
  options: {
@@ -482,6 +563,10 @@ export function parseCheckArgs(args, cwd) {
482
563
  flowWrites,
483
564
  onRefusedStep,
484
565
  gateRetests,
566
+ baseline,
567
+ ...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
568
+ baselineThreshold: threshold.value,
569
+ ...(anchor ? { sarifFileAnchor: anchor.value } : {}),
485
570
  },
486
571
  };
487
572
  }
@@ -535,10 +620,13 @@ export function describeSettings(result) {
535
620
  }
536
621
  const SARIF_LEVEL = { high: "error", medium: "warning", low: "note" };
537
622
  /**
538
- * SARIF 2.1.0, for code-scanning dashboards. A UI check has no source file to
539
- * point at, so each result's location is the route, relative to the checked
540
- * app (the APP base id). Fingerprints come from the evidence, not the
541
- * location, so a preview deployment on a new URL does not reopen every alert.
623
+ * SARIF 2.1.0, for code-scanning dashboards. Code scanning keeps a result
624
+ * only when its location is a repository file, so each result points at one
625
+ * (sarif.ts says which) and carries its routes in logical locations, in
626
+ * `properties.routes` and in its message. Fingerprints come from the
627
+ * evidence, not the location, so a preview deployment on a new URL does not
628
+ * reopen every alert; an unmet visual baseline's comes from its target and
629
+ * browser instead.
542
630
  */
543
631
  const RETEST_SARIF_RULE = {
544
632
  id: "open-finding-reproduces",
@@ -547,39 +635,39 @@ const RETEST_SARIF_RULE = {
547
635
  help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
548
636
  defaultConfiguration: { level: "error" },
549
637
  };
550
- function sarifLocation(route) {
551
- return route === "(shared chrome)"
552
- ? {
553
- physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
554
- message: { text: "the app's shared shell, on every page that renders it" },
555
- }
556
- : { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
638
+ function routeLocation(file, route) {
639
+ return route === SHARED_CHROME_ROUTE ? sarifLocation(file, route, "the app's shared shell, on every page that renders it") : sarifLocation(file, route);
557
640
  }
558
- export function toSarif(result, toolVersion) {
641
+ /** Where a result was seen, for its message: the first three routes and how many more. */
642
+ function onRoutes(routes) {
643
+ return ` — on ${routes.slice(0, 3).join(", ")}${routes.length > 3 ? ` and ${routes.length - 3} more` : ""}`;
644
+ }
645
+ /** `files` defaults to package.json as the anchor, with no flows directory; the CLI always passes what it resolved. */
646
+ export function toSarif(result, toolVersion, files = { anchor: SARIF_ANCHOR_FALLBACKS[0] }) {
559
647
  const used = [...new Set(result.issues.map((i) => i.rule))];
560
648
  const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
561
- const base = new URL(result.url);
562
649
  const reproducing = retestGateFailures(result);
563
650
  const refused = result.flows.filter((f) => f.outcome.status === "refused");
564
651
  const issueResults = result.issues.map((i) => ({
565
652
  ruleId: i.rule,
566
653
  level: SARIF_LEVEL[i.severity],
567
654
  message: {
568
- text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.routes.length > 1 ? ` (on ${i.routes.length} routes)` : ""}${i.embed ? ` (in an embed of ${i.embed})` : ""}`,
655
+ text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.embed ? ` (in an embed of ${i.embed})` : ""}${onRoutes(i.routes)}`,
569
656
  },
570
- locations: i.routes.slice(0, 10).map(sarifLocation),
657
+ locations: i.routes.slice(0, 10).map((route) => routeLocation(sarifFileFor(files, i.flow), route)),
571
658
  partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
659
+ properties: { routes: i.routes, ...(i.flow ? { flow: i.flow } : {}) },
572
660
  }));
573
661
  // Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
574
662
  const lookResults = result.worthALook.map((o) => ({
575
663
  ruleId: o.rule,
576
664
  level: "note",
577
665
  message: {
578
- text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${o.routes.length > 1 ? ` (on ${o.routes.length} routes)` : ""}. A defect only if your project uses ${o.convention}.`,
666
+ text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${onRoutes(o.routes)}. A defect only if your project uses ${o.convention}.`,
579
667
  },
580
- locations: o.routes.slice(0, 10).map(sarifLocation),
668
+ locations: o.routes.slice(0, 10).map((route) => routeLocation(files.anchor, route)),
581
669
  partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
582
- properties: { tier: "worth-a-look", convention: o.convention },
670
+ properties: { tier: "worth-a-look", convention: o.convention, routes: o.routes },
583
671
  }));
584
672
  // Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
585
673
  const retestResults = reproducing.map((r) => ({
@@ -587,9 +675,10 @@ export function toSarif(result, toolVersion) {
587
675
  // The severity the finding was filed at, as the page rules map theirs; an unknown one reads as the worst.
588
676
  level: SARIF_LEVEL[CHECK_SEVERITIES.includes(r.severity) ? r.severity : "high"],
589
677
  message: {
590
- text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")} (gated by --gate-retests ${result.settings.gateRetests})`,
678
+ text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")}${onRoutes([r.path])} (gated by --gate-retests ${result.settings.gateRetests})`,
591
679
  },
592
- locations: [{ physicalLocation: { artifactLocation: { uri: r.path.replace(/^\//, ""), uriBaseId: "APP" } } }],
680
+ locations: [routeLocation(files.anchor, r.path)],
681
+ properties: { routes: [r.path] },
593
682
  partialFingerprints: { "scenescoutCheck/v1": createHash("sha256").update(`retest\u0000${r.id}`).digest("hex").slice(0, 32) },
594
683
  }));
595
684
  return {
@@ -633,7 +722,7 @@ export function toSarif(result, toolVersion) {
633
722
  })),
634
723
  },
635
724
  ],
636
- originalUriBaseIds: { APP: { uri: `${base.origin}/` } },
725
+ properties: { app: new URL(result.url).origin },
637
726
  results: [...issueResults, ...lookResults, ...retestResults],
638
727
  },
639
728
  ],
@@ -659,10 +748,126 @@ function code(text) {
659
748
  const pad = longest > 0 ? " " : "";
660
749
  return `${fence}${pad}${text}${pad}${fence}`;
661
750
  }
751
+ /** The first three routes as code spans, and how many more. */
662
752
  function routeList(routes) {
663
753
  const shown = routes.slice(0, 3).map(code);
664
754
  return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
665
755
  }
756
+ const BASELINE_MARK = {
757
+ matches: "✓",
758
+ changed: "✗",
759
+ "no-baseline": "○",
760
+ unusable: "✗",
761
+ "not-captured": "⊘",
762
+ updated: "↻",
763
+ };
764
+ /** One target's line in the report. */
765
+ function baselineLine(r, run) {
766
+ const what = `${BASELINE_MARK[r.status]} ${code(r.element)} on ${code(r.path)}`;
767
+ const notes = [r.platformNote, r.partial]
768
+ .filter(Boolean)
769
+ .map((n) => ` _(${cell(n)})_`)
770
+ .join("");
771
+ switch (r.status) {
772
+ case "matches":
773
+ return `- ${what}: ${run.mode === "update" ? `within the threshold of its baseline (${r.diff?.percent ?? 0}% changed), which was left as it was` : `matches (${r.diff?.percent ?? 0}% changed)`}${notes}`;
774
+ case "changed": {
775
+ const files = r.files ? ` — expected ${code(r.files.expected)}, now ${code(r.files.actual)}, diff ${code(r.files.diff)}` : "";
776
+ return `- ${what}: ${r.diff ? describeChange(r.diff, run.threshold) : "it no longer matches its baseline"}${files}${notes}`;
777
+ }
778
+ case "no-baseline":
779
+ return `- ${what}: no baseline yet${notes}`;
780
+ case "unusable":
781
+ return `- ${what}: its baseline cannot be used: ${cell(r.detail ?? "no reason given")}`;
782
+ case "not-captured":
783
+ return `- ${what}: could not be captured: ${cell(r.detail ?? "no reason given")}${run.mode === "update" ? "; its baseline was not written" : ""}`;
784
+ case "updated":
785
+ return `- ${what}: baseline written to ${code(r.baseline)} (${cell(r.detail ?? "updated")})${notes}`;
786
+ }
787
+ }
788
+ /** The report's section on visual baselines: every target and what became of it. */
789
+ function formatBaselines(run) {
790
+ const c = countStatuses(run.results);
791
+ const counts = [
792
+ ["changed", "changed"],
793
+ ["unusable", "unusable"],
794
+ ["not-captured", "not captured"],
795
+ ["no-baseline", "no baseline yet"],
796
+ ["updated", "updated"],
797
+ ["matches", run.mode === "update" ? "left as they were" : "match"],
798
+ ]
799
+ .filter(([k]) => c[k] > 0)
800
+ .map(([k, label]) => `${c[k]} ${label}`)
801
+ .join(" · ");
802
+ const lines = [
803
+ `## Visual baselines (${run.results.length})`,
804
+ "",
805
+ `${run.mode === "update" ? "Updated" : "Compared"} in ${run.engine}, baselines in ${code(run.dir)} · ${run.threshold}% of a picture's pixels may change · ${counts}`,
806
+ "",
807
+ ...run.results.map((r) => baselineLine(r, run)),
808
+ ];
809
+ if (c.changed > 0)
810
+ lines.push("", "The pictures are beside this report, under `visual/` (in the job's artifact on CI): the baseline, the picture now, and the changed pixels in red.");
811
+ if (c["no-baseline"] > 0 || c.unusable > 0) {
812
+ lines.push("", "A target with no baseline yet is listed and never fails; one whose baseline cannot be used fails until it is taken again. Run the check with `--baseline update` to take them.");
813
+ }
814
+ if (run.results.some((r) => r.platformNote)) {
815
+ lines.push("", "_Some baselines were taken on another operating system. Each one draws text differently, so a change there may be the system's rather than the app's: take baselines where the check runs._");
816
+ }
817
+ return lines;
818
+ }
819
+ /** One markdown list item for an issue: its title, rule, evidence and routes. */
820
+ export function issueLine(i) {
821
+ return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
822
+ }
823
+ /** The issues, one section per severity, worst first. Shared by the check's report and the first run's. */
824
+ export function issueSections(issues) {
825
+ const lines = [];
826
+ for (const sev of CHECK_SEVERITIES) {
827
+ const of = issues.filter((i) => i.severity === sev);
828
+ if (of.length === 0)
829
+ continue;
830
+ lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
831
+ for (const i of of)
832
+ lines.push(`- ${issueLine(i)}`);
833
+ }
834
+ return lines;
835
+ }
836
+ const WORTH_A_LOOK_INTRO = "Measured exactly, and defects only under a convention of your project that the check cannot see.";
837
+ /** The worth-a-look section, or nothing when there is none. `gated`: the report is a gate's, so say these never fail it. */
838
+ export function worthALookSection(observations, gated = true) {
839
+ if (observations.length === 0)
840
+ return [];
841
+ const lines = [
842
+ "",
843
+ `## Worth a look (${observations.length})`,
844
+ "",
845
+ `${WORTH_A_LOOK_INTRO} They are not counted above${gated ? " and never fail the gate, at any --fail-on" : ""}.`,
846
+ "",
847
+ ];
848
+ for (const o of observations) {
849
+ lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
850
+ }
851
+ return lines;
852
+ }
853
+ /** The table of routes measured, with each one's status, controls and issue count. */
854
+ export function routesTable(result) {
855
+ const lines = ["", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|"];
856
+ for (const r of result.routes) {
857
+ const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
858
+ const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
859
+ (r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
860
+ // Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
861
+ lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
862
+ }
863
+ return lines;
864
+ }
865
+ /** The routes known and not visited, as one line; nothing when every known route was visited. */
866
+ export function unvisitedLine(unvisited, why) {
867
+ if (unvisited.length === 0)
868
+ return [];
869
+ return ["", `Not visited (${why}): ${unvisited.slice(0, 20).map(code).join(", ")}${unvisited.length > 20 ? " …" : ""}`];
870
+ }
666
871
  /** The human report: the verdict first, then what failed it, then everything else. */
667
872
  export function formatCheck(result) {
668
873
  const { passed, counts, failing, retestsFailing, couldNotRun } = summarise(result);
@@ -672,6 +877,8 @@ export function formatCheck(result) {
672
877
  lines.push(`${result.url} · ${result.routes.length} route(s) · mode ${result.mode} · gate: ${result.failOn === "never" ? "report only" : `fail on ${result.failOn}${result.failOn === "low" ? "" : " or worse"}`}`);
673
878
  lines.push("");
674
879
  const unaudited = unauditedRoutes(result.routes).length;
880
+ // Nothing compared is not a match: a target with no baseline yet is named beside the verdict, as an unmeasured design is.
881
+ const uncompared = result.baselines?.mode === "compare" ? countStatuses(result.baselines.results)["no-baseline"] : 0;
675
882
  const failingText = retestsFailing > 0
676
883
  ? `${failing} failing the gate (${failing - retestsFailing} issue(s), ${retestsFailing} re-tested finding(s))`
677
884
  : `${failing} issue(s) at the gate's severity`;
@@ -680,35 +887,14 @@ export function formatCheck(result) {
680
887
  ? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
681
888
  : `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
682
889
  (unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
890
+ (uncompared > 0 ? ` · ${uncompared} visual target(s) not compared: no baseline yet` : "") +
683
891
  (result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
684
892
  // Right under the verdict: what a green check was allowed to do is part of what it means.
685
893
  lines.push("", `Settings — ${describeSettings(result)}`);
686
- for (const sev of CHECK_SEVERITIES) {
687
- const of = result.issues.filter((i) => i.severity === sev);
688
- if (of.length === 0)
689
- continue;
690
- lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
691
- for (const i of of) {
692
- lines.push(`- **${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`);
693
- }
694
- }
695
- if (result.worthALook.length > 0) {
696
- lines.push("", `## Worth a look (${result.worthALook.length})`, "", "Measured exactly, and defects only under a convention of your project that the check cannot see. They are not counted above and never fail the gate, at any --fail-on.", "");
697
- for (const o of result.worthALook) {
698
- lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
699
- }
700
- }
701
- lines.push("", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|");
702
- for (const r of result.routes) {
703
- const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
704
- const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
705
- (r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
706
- // Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
707
- lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
708
- }
709
- if (result.unvisited.length > 0) {
710
- lines.push("", `Not visited (over --max-routes): ${result.unvisited.slice(0, 20).map(code).join(", ")}${result.unvisited.length > 20 ? " …" : ""}`);
711
- }
894
+ lines.push(...issueSections(result.issues));
895
+ lines.push(...worthALookSection(result.worthALook));
896
+ lines.push(...routesTable(result));
897
+ lines.push(...unvisitedLine(result.unvisited, "over --max-routes"));
712
898
  if (result.flows.length > 0 || result.skippedFlows.length > 0) {
713
899
  lines.push("", `## Flows (${result.flows.length})`, "");
714
900
  for (const f of result.flows) {
@@ -726,6 +912,8 @@ export function formatCheck(result) {
726
912
  for (const s of result.skippedFlows)
727
913
  lines.push(`- skipped ${code(s.file)}: ${cell(s.reason)}`);
728
914
  }
915
+ if (result.baselines)
916
+ lines.push("", ...formatBaselines(result.baselines));
729
917
  if (result.retest && result.retest.open > 0) {
730
918
  const { open, results } = result.retest;
731
919
  lines.push("", `## Open findings re-tested (${results.length} of ${open})`, "");
@@ -778,6 +966,7 @@ export function toSummaryJson(result, toolVersion) {
778
966
  ...(r.auditError !== undefined ? { designNotMeasured: r.auditError } : {}),
779
967
  })),
780
968
  unvisited: result.unvisited,
969
+ ...(result.timeBudget ? { timeBudget: result.timeBudget } : {}),
781
970
  ignored: result.ignored,
782
971
  flows: result.flows.map((f) => ({
783
972
  name: f.name,
@@ -790,6 +979,7 @@ export function toSummaryJson(result, toolVersion) {
790
979
  })),
791
980
  skippedFlows: result.skippedFlows,
792
981
  retest: result.retest,
982
+ baselines: result.baselines ?? null,
793
983
  issues: result.issues,
794
984
  // Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
795
985
  worthALook: result.worthALook,