scenescout 3.15.0 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +64 -15
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +190 -14
- package/dist/ci-run.js +268 -52
- package/dist/cli.js +104 -43
- package/dist/commands.js +3 -2
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1147 -286
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +182 -39
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +136 -18
- package/dist/engine/claims.js +159 -3
- package/dist/engine/collector.js +561 -30
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/design.js +281 -38
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +181 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +501 -47
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +41 -1
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/refresh.js +68 -3
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +241 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +4 -0
- package/dist/installer.js +267 -3
- package/dist/intake.js +104 -0
- package/dist/login-run.js +250 -36
- package/dist/mcp-server.js +567 -57
- package/dist/playbook.js +5 -0
- package/package.json +6 -3
- package/skills/scenescout/SKILL.md +49 -16
package/dist/engine/check.js
CHANGED
|
@@ -12,9 +12,11 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { createHash } from "node:crypto";
|
|
14
14
|
import { BROWSER_ENGINES } from "../browsers.js";
|
|
15
|
+
import { BASELINE_MODES, baselineEvidence, baselineFingerprint, countStatuses, describeChange, parseThreshold, VISUAL_RULE, } from "./baseline.js";
|
|
15
16
|
import { flowStepEvidence } from "./flow.js";
|
|
16
17
|
import { parseLimitFlag } from "./limits.js";
|
|
17
|
-
import { redactSecrets } from "./memory.js";
|
|
18
|
+
import { redactRoute, redactSecrets } from "./memory.js";
|
|
19
|
+
import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifFileFor, sarifLocation } from "./sarif.js";
|
|
18
20
|
import { httpErrorDetail, httpStatusOf } from "./oracles.js";
|
|
19
21
|
export const CHECK_SEVERITIES = ["high", "medium", "low"];
|
|
20
22
|
export const FAIL_ON = [...CHECK_SEVERITIES, "never"];
|
|
@@ -65,7 +67,11 @@ export const CHECK_RULES = {
|
|
|
65
67
|
title: "Control covered by pinned chrome",
|
|
66
68
|
help: "A fixed or sticky element sits on top of the control, so clicks land on it instead.",
|
|
67
69
|
},
|
|
68
|
-
"clipped-control": {
|
|
70
|
+
"clipped-control": {
|
|
71
|
+
severity: "medium",
|
|
72
|
+
title: "Control unreachable",
|
|
73
|
+
help: "The control is clipped inside a container that cannot scroll and has no control beside it to page it.",
|
|
74
|
+
},
|
|
69
75
|
"offpage-control": { severity: "medium", title: "Control outside the page", help: "The control is laid out where no scrolling can reach it." },
|
|
70
76
|
"overlapping-controls": { severity: "low", title: "Controls overlap", help: "Two controls in the same layer cover most of each other." },
|
|
71
77
|
"broken-image": { severity: "medium", title: "Broken image", help: "The browser could not render the image." },
|
|
@@ -78,7 +84,11 @@ export const CHECK_RULES = {
|
|
|
78
84
|
contrast: { severity: "low", title: "Text contrast below WCAG", help: "Text needs 4.5:1 (3:1 when large) against its background." },
|
|
79
85
|
"focus-indicator": { severity: "low", title: "No visible focus indicator", help: "Tabbing to the control changes nothing on screen." },
|
|
80
86
|
"horizontal-scroll": { severity: "medium", title: "Page scrolls sideways", help: "Content is wider than the viewport." },
|
|
81
|
-
"tiny-target": {
|
|
87
|
+
"tiny-target": {
|
|
88
|
+
severity: "low",
|
|
89
|
+
title: "Small click target",
|
|
90
|
+
help: "Below the 24px WCAG 2.2 target-size minimum, measured on what the user clicks (a native input's label or drop zone), with another target inside its 24px circle.",
|
|
91
|
+
},
|
|
82
92
|
"clipped-text": { severity: "low", title: "Text clipped", help: "Text is wider than its box and cut off without an ellipsis." },
|
|
83
93
|
"image-aspect": { severity: "low", title: "Image distorted", help: "The rendered box does not match the image's proportions." },
|
|
84
94
|
"flow-step-failed": {
|
|
@@ -86,6 +96,11 @@ export const CHECK_RULES = {
|
|
|
86
96
|
title: "Saved flow broke",
|
|
87
97
|
help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
|
|
88
98
|
},
|
|
99
|
+
[VISUAL_RULE]: {
|
|
100
|
+
severity: "high",
|
|
101
|
+
title: "Differs from its visual baseline",
|
|
102
|
+
help: "A page or element listed in the baselines' targets.json no longer looks like its baseline (--baseline compare): more of its pixels changed than --baseline-threshold allows, or its size changed. It is also filed when the page or element could not be captured at all, or its baseline cannot be used (half there, unreadable, or taken with other settings). An intended change is approved by running the check with --baseline update (and committing the new baseline, where the project keeps baselines in a folder it commits).",
|
|
103
|
+
},
|
|
89
104
|
};
|
|
90
105
|
/**
|
|
91
106
|
* Rules whose measurement is exact and whose meaning depends on a convention
|
|
@@ -189,16 +204,28 @@ export function geometryRule(line) {
|
|
|
189
204
|
*
|
|
190
205
|
* A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
|
|
191
206
|
* the same way; the two lists never share an entry.
|
|
207
|
+
*
|
|
208
|
+
* Visual baselines add one issue per target that is not met (baseline.ts
|
|
209
|
+
* decides which). Each target is its own fact already, so these skip the
|
|
210
|
+
* dedup, and their fingerprint comes from the target, not the evidence: the
|
|
211
|
+
* evidence carries this run's percentage, and the same target changing by a
|
|
212
|
+
* different amount is still the same alert.
|
|
192
213
|
*/
|
|
193
|
-
export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
214
|
+
export function checkFindings(routes, origin, ignore = [], flows = [], baselines = null) {
|
|
194
215
|
const byKey = new Map();
|
|
195
216
|
const looks = new Map();
|
|
217
|
+
/** Evidence as it is written: no origin, no secret, and bounded. */
|
|
218
|
+
const cleanEvidence = (evidence) => redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
|
|
196
219
|
const add = (rule, evidence, route, opts = {}) => {
|
|
197
220
|
if (ignore.includes(rule))
|
|
198
221
|
return;
|
|
199
|
-
const clean =
|
|
222
|
+
const clean = cleanEvidence(evidence);
|
|
200
223
|
const key = `${rule}\u0000${clean}`;
|
|
201
|
-
const
|
|
224
|
+
const issue = byKey.get(key);
|
|
225
|
+
// A fact a crawled page and a flow both show points at the flow: the flow is the file that reproduces it.
|
|
226
|
+
if (issue && opts.flow && !issue.flow)
|
|
227
|
+
issue.flow = opts.flow;
|
|
228
|
+
const found = issue ?? looks.get(key);
|
|
202
229
|
if (found) {
|
|
203
230
|
if (!found.routes.includes(route))
|
|
204
231
|
found.routes.push(route);
|
|
@@ -215,6 +242,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
215
242
|
evidence: clean,
|
|
216
243
|
routes: [route],
|
|
217
244
|
...(opts.embed ? { embed: opts.embed } : {}),
|
|
245
|
+
...(opts.flow ? { flow: opts.flow } : {}),
|
|
218
246
|
fingerprint,
|
|
219
247
|
});
|
|
220
248
|
};
|
|
@@ -251,13 +279,28 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
251
279
|
}
|
|
252
280
|
for (const f of flows) {
|
|
253
281
|
for (const { path, violation } of f.violations)
|
|
254
|
-
add(violationRule(violation), violation.detail, path, { embed: violation.embed });
|
|
282
|
+
add(violationRule(violation), violation.detail, path, { embed: violation.embed, flow: f.file });
|
|
255
283
|
// A refused step is not the app's defect: the check reports it as "could not run" (refusedFlowReason).
|
|
256
284
|
if (f.outcome.status === "failed")
|
|
257
|
-
add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
|
|
285
|
+
add("flow-step-failed", flowStepEvidence(f), f.outcome.path, { flow: f.file });
|
|
286
|
+
}
|
|
287
|
+
const visual = [];
|
|
288
|
+
if (baselines && !ignore.includes(VISUAL_RULE)) {
|
|
289
|
+
for (const r of baselines.results) {
|
|
290
|
+
const evidence = baselineEvidence(r, baselines);
|
|
291
|
+
if (evidence === null)
|
|
292
|
+
continue;
|
|
293
|
+
visual.push({
|
|
294
|
+
rule: VISUAL_RULE,
|
|
295
|
+
severity: CHECK_RULES[VISUAL_RULE].severity,
|
|
296
|
+
evidence: cleanEvidence(evidence),
|
|
297
|
+
routes: [r.path],
|
|
298
|
+
fingerprint: baselineFingerprint(baselines.engine, r),
|
|
299
|
+
});
|
|
300
|
+
}
|
|
258
301
|
}
|
|
259
302
|
return {
|
|
260
|
-
issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
303
|
+
issues: [...byKey.values(), ...visual].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
261
304
|
worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
|
|
262
305
|
};
|
|
263
306
|
}
|
|
@@ -299,15 +342,8 @@ export function withoutOwnResponse(r) {
|
|
|
299
342
|
const own = httpErrorDetail("GET", r.url.replace(/#.*$/, ""), r.status);
|
|
300
343
|
return { ...r, violations: r.violations.filter((v) => !(v.kind === "http_error" && v.detail === own)) };
|
|
301
344
|
}
|
|
302
|
-
/**
|
|
303
|
-
|
|
304
|
-
* (`?reset=…`, `?api_key=…`). Evidence is redacted where issues are built;
|
|
305
|
-
* this does the same for every route string before anything is written.
|
|
306
|
-
*/
|
|
307
|
-
export function redactRoute(route) {
|
|
308
|
-
// The trailing "[n secrets redacted]" note belongs to prose; in a route it would read as part of the path.
|
|
309
|
-
return redactSecrets(route).replace(/ \[\d+ secrets? redacted\]$/, "");
|
|
310
|
-
}
|
|
345
|
+
/** Lives beside redactSecrets so the baselines' file names (baseline.ts) use it too; exported here as before. */
|
|
346
|
+
export { redactRoute };
|
|
311
347
|
export function redactRoutes(routes) {
|
|
312
348
|
const clean = redactRoute;
|
|
313
349
|
return routes.map((r) => ({
|
|
@@ -317,6 +353,16 @@ export function redactRoutes(routes) {
|
|
|
317
353
|
violations: r.violations.map((v) => ({ ...v, url: clean(v.url) })),
|
|
318
354
|
}));
|
|
319
355
|
}
|
|
356
|
+
/**
|
|
357
|
+
* The same redaction for visual baselines: a target's path, and the reason a
|
|
358
|
+
* picture could not be taken, which can quote a URL the app redirected to.
|
|
359
|
+
*/
|
|
360
|
+
export function redactBaselineRun(run) {
|
|
361
|
+
return {
|
|
362
|
+
...run,
|
|
363
|
+
results: run.results.map((r) => ({ ...r, path: redactRoute(r.path), ...(r.detail !== undefined ? { detail: redactSecrets(r.detail) } : {}) })),
|
|
364
|
+
};
|
|
365
|
+
}
|
|
320
366
|
/** The same redaction for what flows measured: the page paths and the request URLs their violations quote. */
|
|
321
367
|
export function redactFlowRuns(runs) {
|
|
322
368
|
return runs.map((f) => ({
|
|
@@ -374,6 +420,10 @@ export const CHECK_OPTION_NAMES = [
|
|
|
374
420
|
"gate-retests",
|
|
375
421
|
"action-timeout-ms",
|
|
376
422
|
"nav-timeout-ms",
|
|
423
|
+
"baseline",
|
|
424
|
+
"baselines",
|
|
425
|
+
"baseline-threshold",
|
|
426
|
+
"sarif-file-anchor",
|
|
377
427
|
];
|
|
378
428
|
export const MAX_CHECK_ROUTES = 150;
|
|
379
429
|
/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
|
|
@@ -476,6 +526,22 @@ export function parseCheckArgs(args, cwd) {
|
|
|
476
526
|
const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
|
|
477
527
|
if (!gateRetests)
|
|
478
528
|
return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
|
|
529
|
+
const baseline = oneOf("baseline", BASELINE_MODES, "off");
|
|
530
|
+
if (!baseline)
|
|
531
|
+
return { ok: false, error: `--baseline must be one of ${BASELINE_MODES.join(", ")}` };
|
|
532
|
+
const threshold = parseThreshold(flags.get("baseline-threshold"));
|
|
533
|
+
if (!threshold.ok)
|
|
534
|
+
return threshold;
|
|
535
|
+
const baselinesDir = flags.get("baselines");
|
|
536
|
+
if (baselinesDir !== undefined && baselinesDir.trim() === "")
|
|
537
|
+
return { ok: false, error: "--baselines needs a directory" };
|
|
538
|
+
// A setting that would do nothing is a mistake to point out, not to ignore: the check would run without the baselines asked for.
|
|
539
|
+
if (baseline === "off" && (baselinesDir !== undefined || flags.has("baseline-threshold"))) {
|
|
540
|
+
return { ok: false, error: "--baselines and --baseline-threshold apply only with --baseline compare or --baseline update" };
|
|
541
|
+
}
|
|
542
|
+
const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
|
|
543
|
+
if (anchor && !anchor.ok)
|
|
544
|
+
return anchor;
|
|
479
545
|
const resolve = (p) => resolveArgPath(cwd, p);
|
|
480
546
|
return {
|
|
481
547
|
ok: true,
|
|
@@ -497,6 +563,10 @@ export function parseCheckArgs(args, cwd) {
|
|
|
497
563
|
flowWrites,
|
|
498
564
|
onRefusedStep,
|
|
499
565
|
gateRetests,
|
|
566
|
+
baseline,
|
|
567
|
+
...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
|
|
568
|
+
baselineThreshold: threshold.value,
|
|
569
|
+
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
500
570
|
},
|
|
501
571
|
};
|
|
502
572
|
}
|
|
@@ -550,10 +620,13 @@ export function describeSettings(result) {
|
|
|
550
620
|
}
|
|
551
621
|
const SARIF_LEVEL = { high: "error", medium: "warning", low: "note" };
|
|
552
622
|
/**
|
|
553
|
-
* SARIF 2.1.0, for code-scanning dashboards.
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
623
|
+
* SARIF 2.1.0, for code-scanning dashboards. Code scanning keeps a result
|
|
624
|
+
* only when its location is a repository file, so each result points at one
|
|
625
|
+
* (sarif.ts says which) and carries its routes in logical locations, in
|
|
626
|
+
* `properties.routes` and in its message. Fingerprints come from the
|
|
627
|
+
* evidence, not the location, so a preview deployment on a new URL does not
|
|
628
|
+
* reopen every alert; an unmet visual baseline's comes from its target and
|
|
629
|
+
* browser instead.
|
|
557
630
|
*/
|
|
558
631
|
const RETEST_SARIF_RULE = {
|
|
559
632
|
id: "open-finding-reproduces",
|
|
@@ -562,39 +635,39 @@ const RETEST_SARIF_RULE = {
|
|
|
562
635
|
help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
|
|
563
636
|
defaultConfiguration: { level: "error" },
|
|
564
637
|
};
|
|
565
|
-
function
|
|
566
|
-
return route === SHARED_CHROME_ROUTE
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
: { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
|
|
638
|
+
function routeLocation(file, route) {
|
|
639
|
+
return route === SHARED_CHROME_ROUTE ? sarifLocation(file, route, "the app's shared shell, on every page that renders it") : sarifLocation(file, route);
|
|
640
|
+
}
|
|
641
|
+
/** Where a result was seen, for its message: the first three routes and how many more. */
|
|
642
|
+
function onRoutes(routes) {
|
|
643
|
+
return ` — on ${routes.slice(0, 3).join(", ")}${routes.length > 3 ? ` and ${routes.length - 3} more` : ""}`;
|
|
572
644
|
}
|
|
573
|
-
|
|
645
|
+
/** `files` defaults to package.json as the anchor, with no flows directory; the CLI always passes what it resolved. */
|
|
646
|
+
export function toSarif(result, toolVersion, files = { anchor: SARIF_ANCHOR_FALLBACKS[0] }) {
|
|
574
647
|
const used = [...new Set(result.issues.map((i) => i.rule))];
|
|
575
648
|
const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
|
|
576
|
-
const base = new URL(result.url);
|
|
577
649
|
const reproducing = retestGateFailures(result);
|
|
578
650
|
const refused = result.flows.filter((f) => f.outcome.status === "refused");
|
|
579
651
|
const issueResults = result.issues.map((i) => ({
|
|
580
652
|
ruleId: i.rule,
|
|
581
653
|
level: SARIF_LEVEL[i.severity],
|
|
582
654
|
message: {
|
|
583
|
-
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.
|
|
655
|
+
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.embed ? ` (in an embed of ${i.embed})` : ""}${onRoutes(i.routes)}`,
|
|
584
656
|
},
|
|
585
|
-
locations: i.routes.slice(0, 10).map(
|
|
657
|
+
locations: i.routes.slice(0, 10).map((route) => routeLocation(sarifFileFor(files, i.flow), route)),
|
|
586
658
|
partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
|
|
659
|
+
properties: { routes: i.routes, ...(i.flow ? { flow: i.flow } : {}) },
|
|
587
660
|
}));
|
|
588
661
|
// Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
|
|
589
662
|
const lookResults = result.worthALook.map((o) => ({
|
|
590
663
|
ruleId: o.rule,
|
|
591
664
|
level: "note",
|
|
592
665
|
message: {
|
|
593
|
-
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${
|
|
666
|
+
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${onRoutes(o.routes)}. A defect only if your project uses ${o.convention}.`,
|
|
594
667
|
},
|
|
595
|
-
locations: o.routes.slice(0, 10).map(
|
|
668
|
+
locations: o.routes.slice(0, 10).map((route) => routeLocation(files.anchor, route)),
|
|
596
669
|
partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
|
|
597
|
-
properties: { tier: "worth-a-look", convention: o.convention },
|
|
670
|
+
properties: { tier: "worth-a-look", convention: o.convention, routes: o.routes },
|
|
598
671
|
}));
|
|
599
672
|
// Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
|
|
600
673
|
const retestResults = reproducing.map((r) => ({
|
|
@@ -602,9 +675,10 @@ export function toSarif(result, toolVersion) {
|
|
|
602
675
|
// The severity the finding was filed at, as the page rules map theirs; an unknown one reads as the worst.
|
|
603
676
|
level: SARIF_LEVEL[CHECK_SEVERITIES.includes(r.severity) ? r.severity : "high"],
|
|
604
677
|
message: {
|
|
605
|
-
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
678
|
+
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")}${onRoutes([r.path])} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
606
679
|
},
|
|
607
|
-
locations: [
|
|
680
|
+
locations: [routeLocation(files.anchor, r.path)],
|
|
681
|
+
properties: { routes: [r.path] },
|
|
608
682
|
partialFingerprints: { "scenescoutCheck/v1": createHash("sha256").update(`retest\u0000${r.id}`).digest("hex").slice(0, 32) },
|
|
609
683
|
}));
|
|
610
684
|
return {
|
|
@@ -648,7 +722,7 @@ export function toSarif(result, toolVersion) {
|
|
|
648
722
|
})),
|
|
649
723
|
},
|
|
650
724
|
],
|
|
651
|
-
|
|
725
|
+
properties: { app: new URL(result.url).origin },
|
|
652
726
|
results: [...issueResults, ...lookResults, ...retestResults],
|
|
653
727
|
},
|
|
654
728
|
],
|
|
@@ -679,6 +753,69 @@ function routeList(routes) {
|
|
|
679
753
|
const shown = routes.slice(0, 3).map(code);
|
|
680
754
|
return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
|
|
681
755
|
}
|
|
756
|
+
const BASELINE_MARK = {
|
|
757
|
+
matches: "✓",
|
|
758
|
+
changed: "✗",
|
|
759
|
+
"no-baseline": "○",
|
|
760
|
+
unusable: "✗",
|
|
761
|
+
"not-captured": "⊘",
|
|
762
|
+
updated: "↻",
|
|
763
|
+
};
|
|
764
|
+
/** One target's line in the report. */
|
|
765
|
+
function baselineLine(r, run) {
|
|
766
|
+
const what = `${BASELINE_MARK[r.status]} ${code(r.element)} on ${code(r.path)}`;
|
|
767
|
+
const notes = [r.platformNote, r.partial]
|
|
768
|
+
.filter(Boolean)
|
|
769
|
+
.map((n) => ` _(${cell(n)})_`)
|
|
770
|
+
.join("");
|
|
771
|
+
switch (r.status) {
|
|
772
|
+
case "matches":
|
|
773
|
+
return `- ${what}: ${run.mode === "update" ? `within the threshold of its baseline (${r.diff?.percent ?? 0}% changed), which was left as it was` : `matches (${r.diff?.percent ?? 0}% changed)`}${notes}`;
|
|
774
|
+
case "changed": {
|
|
775
|
+
const files = r.files ? ` — expected ${code(r.files.expected)}, now ${code(r.files.actual)}, diff ${code(r.files.diff)}` : "";
|
|
776
|
+
return `- ${what}: ${r.diff ? describeChange(r.diff, run.threshold) : "it no longer matches its baseline"}${files}${notes}`;
|
|
777
|
+
}
|
|
778
|
+
case "no-baseline":
|
|
779
|
+
return `- ${what}: no baseline yet${notes}`;
|
|
780
|
+
case "unusable":
|
|
781
|
+
return `- ${what}: its baseline cannot be used: ${cell(r.detail ?? "no reason given")}`;
|
|
782
|
+
case "not-captured":
|
|
783
|
+
return `- ${what}: could not be captured: ${cell(r.detail ?? "no reason given")}${run.mode === "update" ? "; its baseline was not written" : ""}`;
|
|
784
|
+
case "updated":
|
|
785
|
+
return `- ${what}: baseline written to ${code(r.baseline)} (${cell(r.detail ?? "updated")})${notes}`;
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
/** The report's section on visual baselines: every target and what became of it. */
|
|
789
|
+
function formatBaselines(run) {
|
|
790
|
+
const c = countStatuses(run.results);
|
|
791
|
+
const counts = [
|
|
792
|
+
["changed", "changed"],
|
|
793
|
+
["unusable", "unusable"],
|
|
794
|
+
["not-captured", "not captured"],
|
|
795
|
+
["no-baseline", "no baseline yet"],
|
|
796
|
+
["updated", "updated"],
|
|
797
|
+
["matches", run.mode === "update" ? "left as they were" : "match"],
|
|
798
|
+
]
|
|
799
|
+
.filter(([k]) => c[k] > 0)
|
|
800
|
+
.map(([k, label]) => `${c[k]} ${label}`)
|
|
801
|
+
.join(" · ");
|
|
802
|
+
const lines = [
|
|
803
|
+
`## Visual baselines (${run.results.length})`,
|
|
804
|
+
"",
|
|
805
|
+
`${run.mode === "update" ? "Updated" : "Compared"} in ${run.engine}, baselines in ${code(run.dir)} · ${run.threshold}% of a picture's pixels may change · ${counts}`,
|
|
806
|
+
"",
|
|
807
|
+
...run.results.map((r) => baselineLine(r, run)),
|
|
808
|
+
];
|
|
809
|
+
if (c.changed > 0)
|
|
810
|
+
lines.push("", "The pictures are beside this report, under `visual/` (in the job's artifact on CI): the baseline, the picture now, and the changed pixels in red.");
|
|
811
|
+
if (c["no-baseline"] > 0 || c.unusable > 0) {
|
|
812
|
+
lines.push("", "A target with no baseline yet is listed and never fails; one whose baseline cannot be used fails until it is taken again. Run the check with `--baseline update` to take them.");
|
|
813
|
+
}
|
|
814
|
+
if (run.results.some((r) => r.platformNote)) {
|
|
815
|
+
lines.push("", "_Some baselines were taken on another operating system. Each one draws text differently, so a change there may be the system's rather than the app's: take baselines where the check runs._");
|
|
816
|
+
}
|
|
817
|
+
return lines;
|
|
818
|
+
}
|
|
682
819
|
/** One markdown list item for an issue: its title, rule, evidence and routes. */
|
|
683
820
|
export function issueLine(i) {
|
|
684
821
|
return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
|
|
@@ -740,6 +877,8 @@ export function formatCheck(result) {
|
|
|
740
877
|
lines.push(`${result.url} · ${result.routes.length} route(s) · mode ${result.mode} · gate: ${result.failOn === "never" ? "report only" : `fail on ${result.failOn}${result.failOn === "low" ? "" : " or worse"}`}`);
|
|
741
878
|
lines.push("");
|
|
742
879
|
const unaudited = unauditedRoutes(result.routes).length;
|
|
880
|
+
// Nothing compared is not a match: a target with no baseline yet is named beside the verdict, as an unmeasured design is.
|
|
881
|
+
const uncompared = result.baselines?.mode === "compare" ? countStatuses(result.baselines.results)["no-baseline"] : 0;
|
|
743
882
|
const failingText = retestsFailing > 0
|
|
744
883
|
? `${failing} failing the gate (${failing - retestsFailing} issue(s), ${retestsFailing} re-tested finding(s))`
|
|
745
884
|
: `${failing} issue(s) at the gate's severity`;
|
|
@@ -748,6 +887,7 @@ export function formatCheck(result) {
|
|
|
748
887
|
? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
|
|
749
888
|
: `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
|
|
750
889
|
(unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
|
|
890
|
+
(uncompared > 0 ? ` · ${uncompared} visual target(s) not compared: no baseline yet` : "") +
|
|
751
891
|
(result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
|
|
752
892
|
// Right under the verdict: what a green check was allowed to do is part of what it means.
|
|
753
893
|
lines.push("", `Settings — ${describeSettings(result)}`);
|
|
@@ -772,6 +912,8 @@ export function formatCheck(result) {
|
|
|
772
912
|
for (const s of result.skippedFlows)
|
|
773
913
|
lines.push(`- skipped ${code(s.file)}: ${cell(s.reason)}`);
|
|
774
914
|
}
|
|
915
|
+
if (result.baselines)
|
|
916
|
+
lines.push("", ...formatBaselines(result.baselines));
|
|
775
917
|
if (result.retest && result.retest.open > 0) {
|
|
776
918
|
const { open, results } = result.retest;
|
|
777
919
|
lines.push("", `## Open findings re-tested (${results.length} of ${open})`, "");
|
|
@@ -837,6 +979,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
837
979
|
})),
|
|
838
980
|
skippedFlows: result.skippedFlows,
|
|
839
981
|
retest: result.retest,
|
|
982
|
+
baselines: result.baselines ?? null,
|
|
840
983
|
issues: result.issues,
|
|
841
984
|
// Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
|
|
842
985
|
worthALook: result.worthALook,
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lanes for `scenescout ci` (--lanes): the app split between several model
|
|
3
|
+
* loops that explore at once, each in its own browser session and its own
|
|
4
|
+
* part of the app, drawing on the run's one budget, with their findings folded
|
|
5
|
+
* into the one report. This file holds the rules, none of which needs a
|
|
6
|
+
* browser or a network: which routes the planning crawl found, how they are
|
|
7
|
+
* split (brief.ts, the split scout_lane_brief makes), what each lane is told,
|
|
8
|
+
* and how the lanes' endings become the run's. The loop that runs them is
|
|
9
|
+
* src/ci-run.ts; the shared budget is engine/ci.ts's. Why: ADR 20.
|
|
10
|
+
*/
|
|
11
|
+
import { LANE_RULES, planLanes } from "./brief.js";
|
|
12
|
+
import { CI_TOOLS } from "./ci.js";
|
|
13
|
+
/** The session the run attaches first. It plans, holds no lane, and writes the report once every lane is done. */
|
|
14
|
+
export const PLANNER_SESSION = "default";
|
|
15
|
+
/** Crawls the planning step makes at most: each visits the routes the pages of the one before linked to. */
|
|
16
|
+
export const PLAN_CRAWL_ROUNDS = 3;
|
|
17
|
+
/** The most routes a lane's first message lists, and the most crawl lines it is handed. */
|
|
18
|
+
const LIST_MAX = 40;
|
|
19
|
+
/**
|
|
20
|
+
* A lane's tools: an exploring run's, less scout_report. The run writes one
|
|
21
|
+
* report for every lane once all are done; a lane that wrote its own would
|
|
22
|
+
* report the others' work as unfinished.
|
|
23
|
+
*/
|
|
24
|
+
export const LANE_TOOLS = CI_TOOLS.filter((t) => t !== "scout_report");
|
|
25
|
+
// ── what the planning crawl found ───────────────────────────────────────────
|
|
26
|
+
/**
|
|
27
|
+
* A route line of a scout_crawl result: `<path> — <status> · <n> el …`, or
|
|
28
|
+
* `<path> — LOAD FAILED`. The path is what the crawl visited, as the browser
|
|
29
|
+
* opened it.
|
|
30
|
+
*/
|
|
31
|
+
const CRAWL_LINE = /^(\/\S*) — (.+)$/;
|
|
32
|
+
/** The heading of the crawl's problem list, whose entries name a route and say what is wrong on it. */
|
|
33
|
+
const PROBLEMS_HEADING = /^PROBLEM ROUTES \(\d+\):$/;
|
|
34
|
+
/** A problem entry's first line: the route, then ` → …`, `: <why it did not load>`, or nothing (its detail lines follow, indented). */
|
|
35
|
+
const PROBLEM_LINE = /^(\/\S*?)(?: →.*|: .*)?$/;
|
|
36
|
+
/**
|
|
37
|
+
* What a scout_crawl result says about each route it visited: the route's own
|
|
38
|
+
* line, then whatever its problem list says about it. Routes it skipped as
|
|
39
|
+
* off-origin are not the app's, and are left out.
|
|
40
|
+
*/
|
|
41
|
+
export function crawlNotes(text) {
|
|
42
|
+
const notes = new Map();
|
|
43
|
+
const add = (route, line) => {
|
|
44
|
+
const list = notes.get(route);
|
|
45
|
+
if (list)
|
|
46
|
+
list.push(line);
|
|
47
|
+
else
|
|
48
|
+
notes.set(route, [line]);
|
|
49
|
+
};
|
|
50
|
+
let inProblems = false;
|
|
51
|
+
let current = null;
|
|
52
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
53
|
+
const line = raw.trimEnd();
|
|
54
|
+
if (PROBLEMS_HEADING.test(line)) {
|
|
55
|
+
inProblems = true;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (!inProblems) {
|
|
59
|
+
const m = CRAWL_LINE.exec(line);
|
|
60
|
+
if (m && !/^SKIPPED/.test(m[2]))
|
|
61
|
+
add(m[1], line);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (line === "") {
|
|
65
|
+
// A blank line ends the entry it follows, not the list.
|
|
66
|
+
current = null;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (!/^(\/|\s)/.test(line)) {
|
|
70
|
+
// A line that is neither a route nor a detail (the coverage line) ends the list.
|
|
71
|
+
inProblems = false;
|
|
72
|
+
current = null;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (/^\s/.test(line)) {
|
|
76
|
+
if (current)
|
|
77
|
+
add(current, line);
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const p = PROBLEM_LINE.exec(line);
|
|
81
|
+
current = p && notes.has(p[1]) ? p[1] : null;
|
|
82
|
+
if (current)
|
|
83
|
+
add(current, line);
|
|
84
|
+
}
|
|
85
|
+
return notes;
|
|
86
|
+
}
|
|
87
|
+
/** Whether a crawl had nothing left to visit, so another round would find nothing either. */
|
|
88
|
+
export function crawlFoundNothing(text) {
|
|
89
|
+
return /^(?:\[session [^\]]*\]\s*)?(Nothing to crawl|Nothing new to crawl|No routes to crawl yet)/m.test(text);
|
|
90
|
+
}
|
|
91
|
+
/** The path a target URL opens, as a route: the planning crawl never lists the page the run attached on. */
|
|
92
|
+
function routeOf(url) {
|
|
93
|
+
try {
|
|
94
|
+
const u = new URL(url);
|
|
95
|
+
return `${u.pathname}${u.search}`;
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return "/";
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Session names for the lanes: the lane's own name, unless another lane or the
|
|
103
|
+
* planner already has it. A session name is at most 40 characters.
|
|
104
|
+
*/
|
|
105
|
+
export function laneSessions(names) {
|
|
106
|
+
const taken = new Set([PLANNER_SESSION]);
|
|
107
|
+
return names.map((name, i) => {
|
|
108
|
+
let session = name;
|
|
109
|
+
for (let k = i + 1; taken.has(session); k += 1)
|
|
110
|
+
session = `${name.slice(0, 40 - `-${k}`.length)}-${k}`;
|
|
111
|
+
taken.add(session);
|
|
112
|
+
return session;
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Split what the planning crawl found between `count` lanes, by brief.ts: whole
|
|
117
|
+
* modules (a route's first path segment) per lane, balanced by route count.
|
|
118
|
+
* Fewer lanes come back when there are fewer modules. With fewer than two the
|
|
119
|
+
* run has nothing to split, and explores in one loop, saying why.
|
|
120
|
+
*/
|
|
121
|
+
export function planCiLanes(o) {
|
|
122
|
+
const routes = [...new Set([routeOf(o.target), ...o.notes.keys()])];
|
|
123
|
+
if (o.count < 2)
|
|
124
|
+
return { lanes: [], oneLoop: "one lane was asked for" };
|
|
125
|
+
if (o.notes.size === 0 && o.planningFailed)
|
|
126
|
+
return { lanes: [], oneLoop: `${o.planningFailed}, so there was nothing to split` };
|
|
127
|
+
const briefs = planLanes(routes, o.count, { goal: o.focus, mode: o.mode });
|
|
128
|
+
if (briefs.length < 2) {
|
|
129
|
+
const where = briefs[0]?.modules[0];
|
|
130
|
+
return {
|
|
131
|
+
lanes: [],
|
|
132
|
+
oneLoop: `the planning crawl found ${routes.length} route(s), all in ${where ? `one module (${where})` : "no module"}, so there was nothing to split`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
const sessions = laneSessions(briefs.map((b) => b.lane));
|
|
136
|
+
return {
|
|
137
|
+
lanes: briefs.map((b, i) => ({
|
|
138
|
+
...b,
|
|
139
|
+
session: sessions[i],
|
|
140
|
+
// On the target's origin whatever the route says: a path written `//host/x` must not become another host.
|
|
141
|
+
url: `${new URL(o.target).origin}${b.landing.startsWith("/") ? "" : "/"}${b.landing}`,
|
|
142
|
+
crawl: b.routes.flatMap((r) => o.notes.get(r) ?? []).slice(0, LIST_MAX),
|
|
143
|
+
})),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
// ── what a lane is told ─────────────────────────────────────────────────────
|
|
147
|
+
/**
|
|
148
|
+
* The system prompt every lane is given: the method, then the rules of an
|
|
149
|
+
* unattended run as one lane of several. The same for every lane of a run, so
|
|
150
|
+
* the provider's prompt cache serves them all; what differs goes in the
|
|
151
|
+
* lane's first message (ciLaneKickoff).
|
|
152
|
+
*/
|
|
153
|
+
export function ciLaneSystemPrompt(playbook, o) {
|
|
154
|
+
return (`${playbook}\n\n---\n\n` +
|
|
155
|
+
`# Running in CI, as one lane of several\n\n` +
|
|
156
|
+
`You are running unattended in a CI job, as one of several lanes exploring the same app at once, each in its own browser session and its own part of the app. There is no person to ask: never ask a question or wait for an answer; decide, and say in findings and notes what you assumed.\n\n` +
|
|
157
|
+
`- Your browser is already attached to the target in ${o.mode} mode, as your lane's own session. scout_attach, scout_close, scout_session, scout_playbook and scout_report are not available: the mode, the target and the session are fixed for this run, and the method is the text above. Skip the Setup steps that choose them, and never pass \`session\`.\n` +
|
|
158
|
+
`- The planner has crawled the routes it found, so route coverage may already read as complete. Your first message lists the routes your lane owns, from that crawl, and what it saw on them; the other lanes own the rest, at the same time. A route under your modules that the crawl did not list is yours too.\n` +
|
|
159
|
+
LANE_RULES.map((rule) => `- ${rule}\n`).join("") +
|
|
160
|
+
`- The level is ${o.level}. The run writes one report for every lane once all are done, and its contract counts what every lane did: do your share of it on your routes, the design audits included.\n` +
|
|
161
|
+
`- Tool results are text only; screenshots are not available. Long results are cut: prefer calls that return less, and snapshots only where you act.\n` +
|
|
162
|
+
`- When your routes are done, or your share of the budget is nearly spent, reply with a short summary and no tool call. That ends your lane; the others go on.\n`);
|
|
163
|
+
}
|
|
164
|
+
const n = (x) => x.toLocaleString("en-US");
|
|
165
|
+
/** A lane's first message: which lane it is, what it owns, where its browser is, what the crawl saw there, and its share of the budget. */
|
|
166
|
+
export function ciLaneKickoff(o) {
|
|
167
|
+
const { lane } = o;
|
|
168
|
+
const share = Math.max(1, Math.floor(o.caps.turns / o.laneCount));
|
|
169
|
+
return [
|
|
170
|
+
`Run an exploratory test session following the method, as lane "${lane.session}", one of ${o.laneCount} running at once.`,
|
|
171
|
+
`Target: ${o.url}`,
|
|
172
|
+
`Your lane: ${lane.objective}`,
|
|
173
|
+
`Your routes (${lane.routes.length}): ${lane.routes.slice(0, LIST_MAX).join(", ")}${lane.routes.length > LIST_MAX ? ` … and ${lane.routes.length - LIST_MAX} more` : ""}`,
|
|
174
|
+
`Your browser is on ${routeOf(lane.url)}.`,
|
|
175
|
+
lane.crawl.length > 0 ? `What the planning crawl saw on your routes:\n${lane.crawl.map((l) => ` ${l.trim()}`).join("\n")}` : "",
|
|
176
|
+
`Project directory (for scout_scan): ${o.projectDir}`,
|
|
177
|
+
`Level: ${o.level}`,
|
|
178
|
+
`Write mode: ${o.mode}`,
|
|
179
|
+
o.focus ? `Focus: ${o.focus}` : "",
|
|
180
|
+
`Budget: the run's ${o.caps.turns} model turns, ${n(o.caps.tokens)} tokens and ${Math.round(o.caps.wallMs / 60_000)} minutes are shared by the ${o.laneCount} lanes: plan on about ${share} turns. Several tool calls in one turn cost one turn. Every lane stops at the first cap the run reaches, and the report is written as it stands.`,
|
|
181
|
+
]
|
|
182
|
+
.filter(Boolean)
|
|
183
|
+
.join("\n");
|
|
184
|
+
}
|
|
185
|
+
// ── how the lanes' endings become the run's ─────────────────────────────────
|
|
186
|
+
/** Caps in the order capReached checks them, so a run ended by two names the one checked first. */
|
|
187
|
+
const CAP_ORDER = ["time", "tokens", "turns"];
|
|
188
|
+
/**
|
|
189
|
+
* What ended a run whose exploration was split into lanes. A model API failure
|
|
190
|
+
* in any lane is the run's, since it is what the workflow must fix, and so is
|
|
191
|
+
* a lane that broke after attaching (could-not-start on an attached lane: the
|
|
192
|
+
* run could not finish, exit 2). Otherwise the run ended at a cap if any lane
|
|
193
|
+
* was stopped by one, a lane the time cap left no time to attach included, and
|
|
194
|
+
* the model finished it only when every lane that ran finished by itself. A
|
|
195
|
+
* lane whose browser could not attach does not change that, but is named in
|
|
196
|
+
* the detail; when none could, the run could not start.
|
|
197
|
+
*/
|
|
198
|
+
export function mergeLaneStops(lanes) {
|
|
199
|
+
const failed = lanes.find((l) => l.attached && l.stop === "provider-error");
|
|
200
|
+
if (failed)
|
|
201
|
+
return { stop: "provider-error", stopDetail: `lane ${failed.session}${failed.stopDetail ? `: ${failed.stopDetail}` : ""}` };
|
|
202
|
+
const broke = lanes.find((l) => l.attached && l.stop === "could-not-start");
|
|
203
|
+
if (broke)
|
|
204
|
+
return { stop: "could-not-start", stopDetail: `lane ${broke.session}: ${broke.stopDetail ?? "it stopped for no reason it gave"}` };
|
|
205
|
+
const unattached = lanes.filter((l) => !l.attached && l.stop === "could-not-start").map((l) => l.session);
|
|
206
|
+
if (unattached.length === lanes.length) {
|
|
207
|
+
const first = lanes.find((l) => l.stopDetail);
|
|
208
|
+
return { stop: "could-not-start", stopDetail: `no lane could attach${first ? ` (${first.session}: ${first.stopDetail})` : ""}` };
|
|
209
|
+
}
|
|
210
|
+
const note = unattached.length > 0 ? { stopDetail: `${unattached.length} of ${lanes.length} lanes could not attach: ${unattached.join(", ")}` } : {};
|
|
211
|
+
for (const cap of CAP_ORDER)
|
|
212
|
+
if (lanes.some((l) => l.stop === cap))
|
|
213
|
+
return { stop: cap, ...note };
|
|
214
|
+
return { stop: "done", ...note };
|
|
215
|
+
}
|