scenescout 3.15.0 → 3.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +75 -0
  2. package/README.md +64 -15
  3. package/dist/browsers.js +28 -0
  4. package/dist/check-run.js +190 -14
  5. package/dist/ci-run.js +268 -52
  6. package/dist/cli.js +104 -43
  7. package/dist/commands.js +3 -2
  8. package/dist/engine/baseline.js +377 -0
  9. package/dist/engine/brief.js +16 -7
  10. package/dist/engine/browser.js +1147 -286
  11. package/dist/engine/calibration.js +61 -30
  12. package/dist/engine/capture.js +164 -0
  13. package/dist/engine/check.js +182 -39
  14. package/dist/engine/ci-lanes.js +215 -0
  15. package/dist/engine/ci.js +136 -18
  16. package/dist/engine/claims.js +159 -3
  17. package/dist/engine/collector.js +561 -30
  18. package/dist/engine/crawl.js +49 -0
  19. package/dist/engine/design.js +281 -38
  20. package/dist/engine/export.js +877 -0
  21. package/dist/engine/fingerprint.js +92 -4
  22. package/dist/engine/flow.js +18 -6
  23. package/dist/engine/forms.js +181 -18
  24. package/dist/engine/journey.js +29 -1
  25. package/dist/engine/lane.js +13 -3
  26. package/dist/engine/launch.js +45 -6
  27. package/dist/engine/limits.js +7 -0
  28. package/dist/engine/live-page.js +49 -2
  29. package/dist/engine/live.js +4 -1
  30. package/dist/engine/memory.js +501 -47
  31. package/dist/engine/open.js +118 -0
  32. package/dist/engine/oracles.js +41 -1
  33. package/dist/engine/plain.js +268 -0
  34. package/dist/engine/png.js +127 -0
  35. package/dist/engine/policy.js +379 -9
  36. package/dist/engine/probes.js +3 -2
  37. package/dist/engine/profiles.js +45 -9
  38. package/dist/engine/project-folder.js +191 -0
  39. package/dist/engine/refresh.js +68 -3
  40. package/dist/engine/replay.js +63 -10
  41. package/dist/engine/report.js +241 -40
  42. package/dist/engine/request.js +317 -23
  43. package/dist/engine/sarif.js +120 -0
  44. package/dist/engine/settle.js +67 -0
  45. package/dist/engine/signed-in.js +256 -0
  46. package/dist/engine/tickets.js +671 -0
  47. package/dist/engine/unload.js +3 -2
  48. package/dist/export-run.js +633 -0
  49. package/dist/first-run.js +4 -0
  50. package/dist/installer.js +267 -3
  51. package/dist/intake.js +104 -0
  52. package/dist/login-run.js +250 -36
  53. package/dist/mcp-server.js +567 -57
  54. package/dist/playbook.js +5 -0
  55. package/package.json +6 -3
  56. package/skills/scenescout/SKILL.md +49 -16
@@ -12,9 +12,11 @@
12
12
  */
13
13
  import { createHash } from "node:crypto";
14
14
  import { BROWSER_ENGINES } from "../browsers.js";
15
+ import { BASELINE_MODES, baselineEvidence, baselineFingerprint, countStatuses, describeChange, parseThreshold, VISUAL_RULE, } from "./baseline.js";
15
16
  import { flowStepEvidence } from "./flow.js";
16
17
  import { parseLimitFlag } from "./limits.js";
17
- import { redactSecrets } from "./memory.js";
18
+ import { redactRoute, redactSecrets } from "./memory.js";
19
+ import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifFileFor, sarifLocation } from "./sarif.js";
18
20
  import { httpErrorDetail, httpStatusOf } from "./oracles.js";
19
21
  export const CHECK_SEVERITIES = ["high", "medium", "low"];
20
22
  export const FAIL_ON = [...CHECK_SEVERITIES, "never"];
@@ -65,7 +67,11 @@ export const CHECK_RULES = {
65
67
  title: "Control covered by pinned chrome",
66
68
  help: "A fixed or sticky element sits on top of the control, so clicks land on it instead.",
67
69
  },
68
- "clipped-control": { severity: "medium", title: "Control unreachable", help: "The control is clipped inside a container that cannot scroll." },
70
+ "clipped-control": {
71
+ severity: "medium",
72
+ title: "Control unreachable",
73
+ help: "The control is clipped inside a container that cannot scroll and has no control beside it to page it.",
74
+ },
69
75
  "offpage-control": { severity: "medium", title: "Control outside the page", help: "The control is laid out where no scrolling can reach it." },
70
76
  "overlapping-controls": { severity: "low", title: "Controls overlap", help: "Two controls in the same layer cover most of each other." },
71
77
  "broken-image": { severity: "medium", title: "Broken image", help: "The browser could not render the image." },
@@ -78,7 +84,11 @@ export const CHECK_RULES = {
78
84
  contrast: { severity: "low", title: "Text contrast below WCAG", help: "Text needs 4.5:1 (3:1 when large) against its background." },
79
85
  "focus-indicator": { severity: "low", title: "No visible focus indicator", help: "Tabbing to the control changes nothing on screen." },
80
86
  "horizontal-scroll": { severity: "medium", title: "Page scrolls sideways", help: "Content is wider than the viewport." },
81
- "tiny-target": { severity: "low", title: "Small click target", help: "Below the 24px WCAG 2.2 target-size minimum." },
87
+ "tiny-target": {
88
+ severity: "low",
89
+ title: "Small click target",
90
+ help: "Below the 24px WCAG 2.2 target-size minimum, measured on what the user clicks (a native input's label or drop zone), with another target inside its 24px circle.",
91
+ },
82
92
  "clipped-text": { severity: "low", title: "Text clipped", help: "Text is wider than its box and cut off without an ellipsis." },
83
93
  "image-aspect": { severity: "low", title: "Image distorted", help: "The rendered box does not match the image's proportions." },
84
94
  "flow-step-failed": {
@@ -86,6 +96,11 @@ export const CHECK_RULES = {
86
96
  title: "Saved flow broke",
87
97
  help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
88
98
  },
99
+ [VISUAL_RULE]: {
100
+ severity: "high",
101
+ title: "Differs from its visual baseline",
102
+ help: "A page or element listed in the baselines' targets.json no longer looks like its baseline (--baseline compare): more of its pixels changed than --baseline-threshold allows, or its size changed. It is also filed when the page or element could not be captured at all, or its baseline cannot be used (half there, unreadable, or taken with other settings). An intended change is approved by running the check with --baseline update (and committing the new baseline, where the project keeps baselines in a folder it commits).",
103
+ },
89
104
  };
90
105
  /**
91
106
  * Rules whose measurement is exact and whose meaning depends on a convention
@@ -189,16 +204,28 @@ export function geometryRule(line) {
189
204
  *
190
205
  * A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
191
206
  * the same way; the two lists never share an entry.
207
+ *
208
+ * Visual baselines add one issue per target that is not met (baseline.ts
209
+ * decides which). Each target is its own fact already, so these skip the
210
+ * dedup, and their fingerprint comes from the target, not the evidence: the
211
+ * evidence carries this run's percentage, and the same target changing by a
212
+ * different amount is still the same alert.
192
213
  */
193
- export function checkFindings(routes, origin, ignore = [], flows = []) {
214
+ export function checkFindings(routes, origin, ignore = [], flows = [], baselines = null) {
194
215
  const byKey = new Map();
195
216
  const looks = new Map();
217
+ /** Evidence as it is written: no origin, no secret, and bounded. */
218
+ const cleanEvidence = (evidence) => redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
196
219
  const add = (rule, evidence, route, opts = {}) => {
197
220
  if (ignore.includes(rule))
198
221
  return;
199
- const clean = redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
222
+ const clean = cleanEvidence(evidence);
200
223
  const key = `${rule}\u0000${clean}`;
201
- const found = byKey.get(key) ?? looks.get(key);
224
+ const issue = byKey.get(key);
225
+ // A fact a crawled page and a flow both show points at the flow: the flow is the file that reproduces it.
226
+ if (issue && opts.flow && !issue.flow)
227
+ issue.flow = opts.flow;
228
+ const found = issue ?? looks.get(key);
202
229
  if (found) {
203
230
  if (!found.routes.includes(route))
204
231
  found.routes.push(route);
@@ -215,6 +242,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
215
242
  evidence: clean,
216
243
  routes: [route],
217
244
  ...(opts.embed ? { embed: opts.embed } : {}),
245
+ ...(opts.flow ? { flow: opts.flow } : {}),
218
246
  fingerprint,
219
247
  });
220
248
  };
@@ -251,13 +279,28 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
251
279
  }
252
280
  for (const f of flows) {
253
281
  for (const { path, violation } of f.violations)
254
- add(violationRule(violation), violation.detail, path, { embed: violation.embed });
282
+ add(violationRule(violation), violation.detail, path, { embed: violation.embed, flow: f.file });
255
283
  // A refused step is not the app's defect: the check reports it as "could not run" (refusedFlowReason).
256
284
  if (f.outcome.status === "failed")
257
- add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
285
+ add("flow-step-failed", flowStepEvidence(f), f.outcome.path, { flow: f.file });
286
+ }
287
+ const visual = [];
288
+ if (baselines && !ignore.includes(VISUAL_RULE)) {
289
+ for (const r of baselines.results) {
290
+ const evidence = baselineEvidence(r, baselines);
291
+ if (evidence === null)
292
+ continue;
293
+ visual.push({
294
+ rule: VISUAL_RULE,
295
+ severity: CHECK_RULES[VISUAL_RULE].severity,
296
+ evidence: cleanEvidence(evidence),
297
+ routes: [r.path],
298
+ fingerprint: baselineFingerprint(baselines.engine, r),
299
+ });
300
+ }
258
301
  }
259
302
  return {
260
- issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
303
+ issues: [...byKey.values(), ...visual].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
261
304
  worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
262
305
  };
263
306
  }
@@ -299,15 +342,8 @@ export function withoutOwnResponse(r) {
299
342
  const own = httpErrorDetail("GET", r.url.replace(/#.*$/, ""), r.status);
300
343
  return { ...r, violations: r.violations.filter((v) => !(v.kind === "http_error" && v.detail === own)) };
301
344
  }
302
- /**
303
- * Routes are links the app printed, and a link can carry a token
304
- * (`?reset=…`, `?api_key=…`). Evidence is redacted where issues are built;
305
- * this does the same for every route string before anything is written.
306
- */
307
- export function redactRoute(route) {
308
- // The trailing "[n secrets redacted]" note belongs to prose; in a route it would read as part of the path.
309
- return redactSecrets(route).replace(/ \[\d+ secrets? redacted\]$/, "");
310
- }
345
+ /** Lives beside redactSecrets so the baselines' file names (baseline.ts) use it too; exported here as before. */
346
+ export { redactRoute };
311
347
  export function redactRoutes(routes) {
312
348
  const clean = redactRoute;
313
349
  return routes.map((r) => ({
@@ -317,6 +353,16 @@ export function redactRoutes(routes) {
317
353
  violations: r.violations.map((v) => ({ ...v, url: clean(v.url) })),
318
354
  }));
319
355
  }
356
+ /**
357
+ * The same redaction for visual baselines: a target's path, and the reason a
358
+ * picture could not be taken, which can quote a URL the app redirected to.
359
+ */
360
+ export function redactBaselineRun(run) {
361
+ return {
362
+ ...run,
363
+ results: run.results.map((r) => ({ ...r, path: redactRoute(r.path), ...(r.detail !== undefined ? { detail: redactSecrets(r.detail) } : {}) })),
364
+ };
365
+ }
320
366
  /** The same redaction for what flows measured: the page paths and the request URLs their violations quote. */
321
367
  export function redactFlowRuns(runs) {
322
368
  return runs.map((f) => ({
@@ -374,6 +420,10 @@ export const CHECK_OPTION_NAMES = [
374
420
  "gate-retests",
375
421
  "action-timeout-ms",
376
422
  "nav-timeout-ms",
423
+ "baseline",
424
+ "baselines",
425
+ "baseline-threshold",
426
+ "sarif-file-anchor",
377
427
  ];
378
428
  export const MAX_CHECK_ROUTES = 150;
379
429
  /** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
@@ -476,6 +526,22 @@ export function parseCheckArgs(args, cwd) {
476
526
  const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
477
527
  if (!gateRetests)
478
528
  return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
529
+ const baseline = oneOf("baseline", BASELINE_MODES, "off");
530
+ if (!baseline)
531
+ return { ok: false, error: `--baseline must be one of ${BASELINE_MODES.join(", ")}` };
532
+ const threshold = parseThreshold(flags.get("baseline-threshold"));
533
+ if (!threshold.ok)
534
+ return threshold;
535
+ const baselinesDir = flags.get("baselines");
536
+ if (baselinesDir !== undefined && baselinesDir.trim() === "")
537
+ return { ok: false, error: "--baselines needs a directory" };
538
+ // A setting that would do nothing is a mistake to point out, not to ignore: the check would run without the baselines asked for.
539
+ if (baseline === "off" && (baselinesDir !== undefined || flags.has("baseline-threshold"))) {
540
+ return { ok: false, error: "--baselines and --baseline-threshold apply only with --baseline compare or --baseline update" };
541
+ }
542
+ const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
543
+ if (anchor && !anchor.ok)
544
+ return anchor;
479
545
  const resolve = (p) => resolveArgPath(cwd, p);
480
546
  return {
481
547
  ok: true,
@@ -497,6 +563,10 @@ export function parseCheckArgs(args, cwd) {
497
563
  flowWrites,
498
564
  onRefusedStep,
499
565
  gateRetests,
566
+ baseline,
567
+ ...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
568
+ baselineThreshold: threshold.value,
569
+ ...(anchor ? { sarifFileAnchor: anchor.value } : {}),
500
570
  },
501
571
  };
502
572
  }
@@ -550,10 +620,13 @@ export function describeSettings(result) {
550
620
  }
551
621
  const SARIF_LEVEL = { high: "error", medium: "warning", low: "note" };
552
622
  /**
553
- * SARIF 2.1.0, for code-scanning dashboards. A UI check has no source file to
554
- * point at, so each result's location is the route, relative to the checked
555
- * app (the APP base id). Fingerprints come from the evidence, not the
556
- * location, so a preview deployment on a new URL does not reopen every alert.
623
+ * SARIF 2.1.0, for code-scanning dashboards. Code scanning keeps a result
624
+ * only when its location is a repository file, so each result points at one
625
+ * (sarif.ts says which) and carries its routes in logical locations, in
626
+ * `properties.routes` and in its message. Fingerprints come from the
627
+ * evidence, not the location, so a preview deployment on a new URL does not
628
+ * reopen every alert; an unmet visual baseline's comes from its target and
629
+ * browser instead.
557
630
  */
558
631
  const RETEST_SARIF_RULE = {
559
632
  id: "open-finding-reproduces",
@@ -562,39 +635,39 @@ const RETEST_SARIF_RULE = {
562
635
  help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
563
636
  defaultConfiguration: { level: "error" },
564
637
  };
565
- function sarifLocation(route) {
566
- return route === SHARED_CHROME_ROUTE
567
- ? {
568
- physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
569
- message: { text: "the app's shared shell, on every page that renders it" },
570
- }
571
- : { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
638
+ function routeLocation(file, route) {
639
+ return route === SHARED_CHROME_ROUTE ? sarifLocation(file, route, "the app's shared shell, on every page that renders it") : sarifLocation(file, route);
640
+ }
641
+ /** Where a result was seen, for its message: the first three routes and how many more. */
642
+ function onRoutes(routes) {
643
+ return ` — on ${routes.slice(0, 3).join(", ")}${routes.length > 3 ? ` and ${routes.length - 3} more` : ""}`;
572
644
  }
573
- export function toSarif(result, toolVersion) {
645
+ /** `files` defaults to package.json as the anchor, with no flows directory; the CLI always passes what it resolved. */
646
+ export function toSarif(result, toolVersion, files = { anchor: SARIF_ANCHOR_FALLBACKS[0] }) {
574
647
  const used = [...new Set(result.issues.map((i) => i.rule))];
575
648
  const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
576
- const base = new URL(result.url);
577
649
  const reproducing = retestGateFailures(result);
578
650
  const refused = result.flows.filter((f) => f.outcome.status === "refused");
579
651
  const issueResults = result.issues.map((i) => ({
580
652
  ruleId: i.rule,
581
653
  level: SARIF_LEVEL[i.severity],
582
654
  message: {
583
- text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.routes.length > 1 ? ` (on ${i.routes.length} routes)` : ""}${i.embed ? ` (in an embed of ${i.embed})` : ""}`,
655
+ text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.embed ? ` (in an embed of ${i.embed})` : ""}${onRoutes(i.routes)}`,
584
656
  },
585
- locations: i.routes.slice(0, 10).map(sarifLocation),
657
+ locations: i.routes.slice(0, 10).map((route) => routeLocation(sarifFileFor(files, i.flow), route)),
586
658
  partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
659
+ properties: { routes: i.routes, ...(i.flow ? { flow: i.flow } : {}) },
587
660
  }));
588
661
  // Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
589
662
  const lookResults = result.worthALook.map((o) => ({
590
663
  ruleId: o.rule,
591
664
  level: "note",
592
665
  message: {
593
- text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${o.routes.length > 1 ? ` (on ${o.routes.length} routes)` : ""}. A defect only if your project uses ${o.convention}.`,
666
+ text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${onRoutes(o.routes)}. A defect only if your project uses ${o.convention}.`,
594
667
  },
595
- locations: o.routes.slice(0, 10).map(sarifLocation),
668
+ locations: o.routes.slice(0, 10).map((route) => routeLocation(files.anchor, route)),
596
669
  partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
597
- properties: { tier: "worth-a-look", convention: o.convention },
670
+ properties: { tier: "worth-a-look", convention: o.convention, routes: o.routes },
598
671
  }));
599
672
  // Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
600
673
  const retestResults = reproducing.map((r) => ({
@@ -602,9 +675,10 @@ export function toSarif(result, toolVersion) {
602
675
  // The severity the finding was filed at, as the page rules map theirs; an unknown one reads as the worst.
603
676
  level: SARIF_LEVEL[CHECK_SEVERITIES.includes(r.severity) ? r.severity : "high"],
604
677
  message: {
605
- text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")} (gated by --gate-retests ${result.settings.gateRetests})`,
678
+ text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")}${onRoutes([r.path])} (gated by --gate-retests ${result.settings.gateRetests})`,
606
679
  },
607
- locations: [{ physicalLocation: { artifactLocation: { uri: r.path.replace(/^\//, ""), uriBaseId: "APP" } } }],
680
+ locations: [routeLocation(files.anchor, r.path)],
681
+ properties: { routes: [r.path] },
608
682
  partialFingerprints: { "scenescoutCheck/v1": createHash("sha256").update(`retest\u0000${r.id}`).digest("hex").slice(0, 32) },
609
683
  }));
610
684
  return {
@@ -648,7 +722,7 @@ export function toSarif(result, toolVersion) {
648
722
  })),
649
723
  },
650
724
  ],
651
- originalUriBaseIds: { APP: { uri: `${base.origin}/` } },
725
+ properties: { app: new URL(result.url).origin },
652
726
  results: [...issueResults, ...lookResults, ...retestResults],
653
727
  },
654
728
  ],
@@ -679,6 +753,69 @@ function routeList(routes) {
679
753
  const shown = routes.slice(0, 3).map(code);
680
754
  return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
681
755
  }
756
+ const BASELINE_MARK = {
757
+ matches: "✓",
758
+ changed: "✗",
759
+ "no-baseline": "○",
760
+ unusable: "✗",
761
+ "not-captured": "⊘",
762
+ updated: "↻",
763
+ };
764
+ /** One target's line in the report. */
765
+ function baselineLine(r, run) {
766
+ const what = `${BASELINE_MARK[r.status]} ${code(r.element)} on ${code(r.path)}`;
767
+ const notes = [r.platformNote, r.partial]
768
+ .filter(Boolean)
769
+ .map((n) => ` _(${cell(n)})_`)
770
+ .join("");
771
+ switch (r.status) {
772
+ case "matches":
773
+ return `- ${what}: ${run.mode === "update" ? `within the threshold of its baseline (${r.diff?.percent ?? 0}% changed), which was left as it was` : `matches (${r.diff?.percent ?? 0}% changed)`}${notes}`;
774
+ case "changed": {
775
+ const files = r.files ? ` — expected ${code(r.files.expected)}, now ${code(r.files.actual)}, diff ${code(r.files.diff)}` : "";
776
+ return `- ${what}: ${r.diff ? describeChange(r.diff, run.threshold) : "it no longer matches its baseline"}${files}${notes}`;
777
+ }
778
+ case "no-baseline":
779
+ return `- ${what}: no baseline yet${notes}`;
780
+ case "unusable":
781
+ return `- ${what}: its baseline cannot be used: ${cell(r.detail ?? "no reason given")}`;
782
+ case "not-captured":
783
+ return `- ${what}: could not be captured: ${cell(r.detail ?? "no reason given")}${run.mode === "update" ? "; its baseline was not written" : ""}`;
784
+ case "updated":
785
+ return `- ${what}: baseline written to ${code(r.baseline)} (${cell(r.detail ?? "updated")})${notes}`;
786
+ }
787
+ }
788
+ /** The report's section on visual baselines: every target and what became of it. */
789
+ function formatBaselines(run) {
790
+ const c = countStatuses(run.results);
791
+ const counts = [
792
+ ["changed", "changed"],
793
+ ["unusable", "unusable"],
794
+ ["not-captured", "not captured"],
795
+ ["no-baseline", "no baseline yet"],
796
+ ["updated", "updated"],
797
+ ["matches", run.mode === "update" ? "left as they were" : "match"],
798
+ ]
799
+ .filter(([k]) => c[k] > 0)
800
+ .map(([k, label]) => `${c[k]} ${label}`)
801
+ .join(" · ");
802
+ const lines = [
803
+ `## Visual baselines (${run.results.length})`,
804
+ "",
805
+ `${run.mode === "update" ? "Updated" : "Compared"} in ${run.engine}, baselines in ${code(run.dir)} · ${run.threshold}% of a picture's pixels may change · ${counts}`,
806
+ "",
807
+ ...run.results.map((r) => baselineLine(r, run)),
808
+ ];
809
+ if (c.changed > 0)
810
+ lines.push("", "The pictures are beside this report, under `visual/` (in the job's artifact on CI): the baseline, the picture now, and the changed pixels in red.");
811
+ if (c["no-baseline"] > 0 || c.unusable > 0) {
812
+ lines.push("", "A target with no baseline yet is listed and never fails; one whose baseline cannot be used fails until it is taken again. Run the check with `--baseline update` to take them.");
813
+ }
814
+ if (run.results.some((r) => r.platformNote)) {
815
+ lines.push("", "_Some baselines were taken on another operating system. Each one draws text differently, so a change there may be the system's rather than the app's: take baselines where the check runs._");
816
+ }
817
+ return lines;
818
+ }
682
819
  /** One markdown list item for an issue: its title, rule, evidence and routes. */
683
820
  export function issueLine(i) {
684
821
  return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
@@ -740,6 +877,8 @@ export function formatCheck(result) {
740
877
  lines.push(`${result.url} · ${result.routes.length} route(s) · mode ${result.mode} · gate: ${result.failOn === "never" ? "report only" : `fail on ${result.failOn}${result.failOn === "low" ? "" : " or worse"}`}`);
741
878
  lines.push("");
742
879
  const unaudited = unauditedRoutes(result.routes).length;
880
+ // Nothing compared is not a match: a target with no baseline yet is named beside the verdict, as an unmeasured design is.
881
+ const uncompared = result.baselines?.mode === "compare" ? countStatuses(result.baselines.results)["no-baseline"] : 0;
743
882
  const failingText = retestsFailing > 0
744
883
  ? `${failing} failing the gate (${failing - retestsFailing} issue(s), ${retestsFailing} re-tested finding(s))`
745
884
  : `${failing} issue(s) at the gate's severity`;
@@ -748,6 +887,7 @@ export function formatCheck(result) {
748
887
  ? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
749
888
  : `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
750
889
  (unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
890
+ (uncompared > 0 ? ` · ${uncompared} visual target(s) not compared: no baseline yet` : "") +
751
891
  (result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
752
892
  // Right under the verdict: what a green check was allowed to do is part of what it means.
753
893
  lines.push("", `Settings — ${describeSettings(result)}`);
@@ -772,6 +912,8 @@ export function formatCheck(result) {
772
912
  for (const s of result.skippedFlows)
773
913
  lines.push(`- skipped ${code(s.file)}: ${cell(s.reason)}`);
774
914
  }
915
+ if (result.baselines)
916
+ lines.push("", ...formatBaselines(result.baselines));
775
917
  if (result.retest && result.retest.open > 0) {
776
918
  const { open, results } = result.retest;
777
919
  lines.push("", `## Open findings re-tested (${results.length} of ${open})`, "");
@@ -837,6 +979,7 @@ export function toSummaryJson(result, toolVersion) {
837
979
  })),
838
980
  skippedFlows: result.skippedFlows,
839
981
  retest: result.retest,
982
+ baselines: result.baselines ?? null,
840
983
  issues: result.issues,
841
984
  // Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
842
985
  worthALook: result.worthALook,
@@ -0,0 +1,215 @@
1
+ /**
2
+ * Lanes for `scenescout ci` (--lanes): the app split between several model
3
+ * loops that explore at once, each in its own browser session and its own
4
+ * part of the app, drawing on the run's one budget, with their findings folded
5
+ * into the one report. This file holds the rules, none of which needs a
6
+ * browser or a network: which routes the planning crawl found, how they are
7
+ * split (brief.ts, the split scout_lane_brief makes), what each lane is told,
8
+ * and how the lanes' endings become the run's. The loop that runs them is
9
+ * src/ci-run.ts; the shared budget is engine/ci.ts's. Why: ADR 20.
10
+ */
11
+ import { LANE_RULES, planLanes } from "./brief.js";
12
+ import { CI_TOOLS } from "./ci.js";
13
+ /** The session the run attaches first. It plans, holds no lane, and writes the report once every lane is done. */
14
+ export const PLANNER_SESSION = "default";
15
+ /** Crawls the planning step makes at most: each visits the routes the pages of the one before linked to. */
16
+ export const PLAN_CRAWL_ROUNDS = 3;
17
+ /** The most routes a lane's first message lists, and the most crawl lines it is handed. */
18
+ const LIST_MAX = 40;
19
+ /**
20
+ * A lane's tools: an exploring run's, less scout_report. The run writes one
21
+ * report for every lane once all are done; a lane that wrote its own would
22
+ * report the others' work as unfinished.
23
+ */
24
+ export const LANE_TOOLS = CI_TOOLS.filter((t) => t !== "scout_report");
25
+ // ── what the planning crawl found ───────────────────────────────────────────
26
+ /**
27
+ * A route line of a scout_crawl result: `<path> — <status> · <n> el …`, or
28
+ * `<path> — LOAD FAILED`. The path is what the crawl visited, as the browser
29
+ * opened it.
30
+ */
31
+ const CRAWL_LINE = /^(\/\S*) — (.+)$/;
32
+ /** The heading of the crawl's problem list, whose entries name a route and say what is wrong on it. */
33
+ const PROBLEMS_HEADING = /^PROBLEM ROUTES \(\d+\):$/;
34
+ /** A problem entry's first line: the route, then ` → …`, `: <why it did not load>`, or nothing (its detail lines follow, indented). */
35
+ const PROBLEM_LINE = /^(\/\S*?)(?: →.*|: .*)?$/;
36
+ /**
37
+ * What a scout_crawl result says about each route it visited: the route's own
38
+ * line, then whatever its problem list says about it. Routes it skipped as
39
+ * off-origin are not the app's, and are left out.
40
+ */
41
+ export function crawlNotes(text) {
42
+ const notes = new Map();
43
+ const add = (route, line) => {
44
+ const list = notes.get(route);
45
+ if (list)
46
+ list.push(line);
47
+ else
48
+ notes.set(route, [line]);
49
+ };
50
+ let inProblems = false;
51
+ let current = null;
52
+ for (const raw of text.split(/\r?\n/)) {
53
+ const line = raw.trimEnd();
54
+ if (PROBLEMS_HEADING.test(line)) {
55
+ inProblems = true;
56
+ continue;
57
+ }
58
+ if (!inProblems) {
59
+ const m = CRAWL_LINE.exec(line);
60
+ if (m && !/^SKIPPED/.test(m[2]))
61
+ add(m[1], line);
62
+ continue;
63
+ }
64
+ if (line === "") {
65
+ // A blank line ends the entry it follows, not the list.
66
+ current = null;
67
+ continue;
68
+ }
69
+ if (!/^(\/|\s)/.test(line)) {
70
+ // A line that is neither a route nor a detail (the coverage line) ends the list.
71
+ inProblems = false;
72
+ current = null;
73
+ continue;
74
+ }
75
+ if (/^\s/.test(line)) {
76
+ if (current)
77
+ add(current, line);
78
+ continue;
79
+ }
80
+ const p = PROBLEM_LINE.exec(line);
81
+ current = p && notes.has(p[1]) ? p[1] : null;
82
+ if (current)
83
+ add(current, line);
84
+ }
85
+ return notes;
86
+ }
87
+ /** Whether a crawl had nothing left to visit, so another round would find nothing either. */
88
+ export function crawlFoundNothing(text) {
89
+ return /^(?:\[session [^\]]*\]\s*)?(Nothing to crawl|Nothing new to crawl|No routes to crawl yet)/m.test(text);
90
+ }
91
+ /** The path a target URL opens, as a route: the planning crawl never lists the page the run attached on. */
92
+ function routeOf(url) {
93
+ try {
94
+ const u = new URL(url);
95
+ return `${u.pathname}${u.search}`;
96
+ }
97
+ catch {
98
+ return "/";
99
+ }
100
+ }
101
+ /**
102
+ * Session names for the lanes: the lane's own name, unless another lane or the
103
+ * planner already has it. A session name is at most 40 characters.
104
+ */
105
+ export function laneSessions(names) {
106
+ const taken = new Set([PLANNER_SESSION]);
107
+ return names.map((name, i) => {
108
+ let session = name;
109
+ for (let k = i + 1; taken.has(session); k += 1)
110
+ session = `${name.slice(0, 40 - `-${k}`.length)}-${k}`;
111
+ taken.add(session);
112
+ return session;
113
+ });
114
+ }
115
+ /**
116
+ * Split what the planning crawl found between `count` lanes, by brief.ts: whole
117
+ * modules (a route's first path segment) per lane, balanced by route count.
118
+ * Fewer lanes come back when there are fewer modules. With fewer than two the
119
+ * run has nothing to split, and explores in one loop, saying why.
120
+ */
121
+ export function planCiLanes(o) {
122
+ const routes = [...new Set([routeOf(o.target), ...o.notes.keys()])];
123
+ if (o.count < 2)
124
+ return { lanes: [], oneLoop: "one lane was asked for" };
125
+ if (o.notes.size === 0 && o.planningFailed)
126
+ return { lanes: [], oneLoop: `${o.planningFailed}, so there was nothing to split` };
127
+ const briefs = planLanes(routes, o.count, { goal: o.focus, mode: o.mode });
128
+ if (briefs.length < 2) {
129
+ const where = briefs[0]?.modules[0];
130
+ return {
131
+ lanes: [],
132
+ oneLoop: `the planning crawl found ${routes.length} route(s), all in ${where ? `one module (${where})` : "no module"}, so there was nothing to split`,
133
+ };
134
+ }
135
+ const sessions = laneSessions(briefs.map((b) => b.lane));
136
+ return {
137
+ lanes: briefs.map((b, i) => ({
138
+ ...b,
139
+ session: sessions[i],
140
+ // On the target's origin whatever the route says: a path written `//host/x` must not become another host.
141
+ url: `${new URL(o.target).origin}${b.landing.startsWith("/") ? "" : "/"}${b.landing}`,
142
+ crawl: b.routes.flatMap((r) => o.notes.get(r) ?? []).slice(0, LIST_MAX),
143
+ })),
144
+ };
145
+ }
146
+ // ── what a lane is told ─────────────────────────────────────────────────────
147
+ /**
148
+ * The system prompt every lane is given: the method, then the rules of an
149
+ * unattended run as one lane of several. The same for every lane of a run, so
150
+ * the provider's prompt cache serves them all; what differs goes in the
151
+ * lane's first message (ciLaneKickoff).
152
+ */
153
+ export function ciLaneSystemPrompt(playbook, o) {
154
+ return (`${playbook}\n\n---\n\n` +
155
+ `# Running in CI, as one lane of several\n\n` +
156
+ `You are running unattended in a CI job, as one of several lanes exploring the same app at once, each in its own browser session and its own part of the app. There is no person to ask: never ask a question or wait for an answer; decide, and say in findings and notes what you assumed.\n\n` +
157
+ `- Your browser is already attached to the target in ${o.mode} mode, as your lane's own session. scout_attach, scout_close, scout_session, scout_playbook and scout_report are not available: the mode, the target and the session are fixed for this run, and the method is the text above. Skip the Setup steps that choose them, and never pass \`session\`.\n` +
158
+ `- The planner has crawled the routes it found, so route coverage may already read as complete. Your first message lists the routes your lane owns, from that crawl, and what it saw on them; the other lanes own the rest, at the same time. A route under your modules that the crawl did not list is yours too.\n` +
159
+ LANE_RULES.map((rule) => `- ${rule}\n`).join("") +
160
+ `- The level is ${o.level}. The run writes one report for every lane once all are done, and its contract counts what every lane did: do your share of it on your routes, the design audits included.\n` +
161
+ `- Tool results are text only; screenshots are not available. Long results are cut: prefer calls that return less, and snapshots only where you act.\n` +
162
+ `- When your routes are done, or your share of the budget is nearly spent, reply with a short summary and no tool call. That ends your lane; the others go on.\n`);
163
+ }
164
+ const n = (x) => x.toLocaleString("en-US");
165
+ /** A lane's first message: which lane it is, what it owns, where its browser is, what the crawl saw there, and its share of the budget. */
166
+ export function ciLaneKickoff(o) {
167
+ const { lane } = o;
168
+ const share = Math.max(1, Math.floor(o.caps.turns / o.laneCount));
169
+ return [
170
+ `Run an exploratory test session following the method, as lane "${lane.session}", one of ${o.laneCount} running at once.`,
171
+ `Target: ${o.url}`,
172
+ `Your lane: ${lane.objective}`,
173
+ `Your routes (${lane.routes.length}): ${lane.routes.slice(0, LIST_MAX).join(", ")}${lane.routes.length > LIST_MAX ? ` … and ${lane.routes.length - LIST_MAX} more` : ""}`,
174
+ `Your browser is on ${routeOf(lane.url)}.`,
175
+ lane.crawl.length > 0 ? `What the planning crawl saw on your routes:\n${lane.crawl.map((l) => ` ${l.trim()}`).join("\n")}` : "",
176
+ `Project directory (for scout_scan): ${o.projectDir}`,
177
+ `Level: ${o.level}`,
178
+ `Write mode: ${o.mode}`,
179
+ o.focus ? `Focus: ${o.focus}` : "",
180
+ `Budget: the run's ${o.caps.turns} model turns, ${n(o.caps.tokens)} tokens and ${Math.round(o.caps.wallMs / 60_000)} minutes are shared by the ${o.laneCount} lanes: plan on about ${share} turns. Several tool calls in one turn cost one turn. Every lane stops at the first cap the run reaches, and the report is written as it stands.`,
181
+ ]
182
+ .filter(Boolean)
183
+ .join("\n");
184
+ }
185
+ // ── how the lanes' endings become the run's ─────────────────────────────────
186
+ /** Caps in the order capReached checks them, so a run ended by two names the one checked first. */
187
+ const CAP_ORDER = ["time", "tokens", "turns"];
188
+ /**
189
+ * What ended a run whose exploration was split into lanes. A model API failure
190
+ * in any lane is the run's, since it is what the workflow must fix, and so is
191
+ * a lane that broke after attaching (could-not-start on an attached lane: the
192
+ * run could not finish, exit 2). Otherwise the run ended at a cap if any lane
193
+ * was stopped by one, a lane the time cap left no time to attach included, and
194
+ * the model finished it only when every lane that ran finished by itself. A
195
+ * lane whose browser could not attach does not change that, but is named in
196
+ * the detail; when none could, the run could not start.
197
+ */
198
+ export function mergeLaneStops(lanes) {
199
+ const failed = lanes.find((l) => l.attached && l.stop === "provider-error");
200
+ if (failed)
201
+ return { stop: "provider-error", stopDetail: `lane ${failed.session}${failed.stopDetail ? `: ${failed.stopDetail}` : ""}` };
202
+ const broke = lanes.find((l) => l.attached && l.stop === "could-not-start");
203
+ if (broke)
204
+ return { stop: "could-not-start", stopDetail: `lane ${broke.session}: ${broke.stopDetail ?? "it stopped for no reason it gave"}` };
205
+ const unattached = lanes.filter((l) => !l.attached && l.stop === "could-not-start").map((l) => l.session);
206
+ if (unattached.length === lanes.length) {
207
+ const first = lanes.find((l) => l.stopDetail);
208
+ return { stop: "could-not-start", stopDetail: `no lane could attach${first ? ` (${first.session}: ${first.stopDetail})` : ""}` };
209
+ }
210
+ const note = unattached.length > 0 ? { stopDetail: `${unattached.length} of ${lanes.length} lanes could not attach: ${unattached.join(", ")}` } : {};
211
+ for (const cap of CAP_ORDER)
212
+ if (lanes.some((l) => l.stop === cap))
213
+ return { stop: cap, ...note };
214
+ return { stop: "done", ...note };
215
+ }