scenescout 3.15.0 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/README.md +70 -18
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +191 -14
- package/dist/ci-run.js +268 -52
- package/dist/cli.js +107 -47
- package/dist/commands.js +3 -2
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1147 -286
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +244 -42
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +136 -18
- package/dist/engine/claims.js +159 -3
- package/dist/engine/collector.js +561 -30
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/design.js +281 -38
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +181 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +501 -47
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +41 -1
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/refresh.js +68 -3
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +241 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/status-pane-page.js +441 -0
- package/dist/engine/status-pane.js +128 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +5 -0
- package/dist/installer.js +378 -8
- package/dist/intake.js +104 -0
- package/dist/login-run.js +250 -36
- package/dist/mcp-server.js +660 -65
- package/dist/playbook.js +5 -0
- package/dist/prompts.js +106 -0
- package/package.json +8 -5
- package/skills/scenescout/SKILL.md +49 -16
package/dist/engine/check.js
CHANGED
|
@@ -12,9 +12,11 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { createHash } from "node:crypto";
|
|
14
14
|
import { BROWSER_ENGINES } from "../browsers.js";
|
|
15
|
+
import { BASELINE_MODES, baselineEvidence, baselineFingerprint, countStatuses, describeChange, parseThreshold, VISUAL_RULE, } from "./baseline.js";
|
|
15
16
|
import { flowStepEvidence } from "./flow.js";
|
|
16
17
|
import { parseLimitFlag } from "./limits.js";
|
|
17
|
-
import { redactSecrets } from "./memory.js";
|
|
18
|
+
import { redactRoute, redactSecrets } from "./memory.js";
|
|
19
|
+
import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifFileFor, sarifLocation } from "./sarif.js";
|
|
18
20
|
import { httpErrorDetail, httpStatusOf } from "./oracles.js";
|
|
19
21
|
export const CHECK_SEVERITIES = ["high", "medium", "low"];
|
|
20
22
|
export const FAIL_ON = [...CHECK_SEVERITIES, "never"];
|
|
@@ -65,7 +67,11 @@ export const CHECK_RULES = {
|
|
|
65
67
|
title: "Control covered by pinned chrome",
|
|
66
68
|
help: "A fixed or sticky element sits on top of the control, so clicks land on it instead.",
|
|
67
69
|
},
|
|
68
|
-
"clipped-control": {
|
|
70
|
+
"clipped-control": {
|
|
71
|
+
severity: "medium",
|
|
72
|
+
title: "Control unreachable",
|
|
73
|
+
help: "The control is clipped inside a container that cannot scroll and has no control beside it to page it.",
|
|
74
|
+
},
|
|
69
75
|
"offpage-control": { severity: "medium", title: "Control outside the page", help: "The control is laid out where no scrolling can reach it." },
|
|
70
76
|
"overlapping-controls": { severity: "low", title: "Controls overlap", help: "Two controls in the same layer cover most of each other." },
|
|
71
77
|
"broken-image": { severity: "medium", title: "Broken image", help: "The browser could not render the image." },
|
|
@@ -78,7 +84,11 @@ export const CHECK_RULES = {
|
|
|
78
84
|
contrast: { severity: "low", title: "Text contrast below WCAG", help: "Text needs 4.5:1 (3:1 when large) against its background." },
|
|
79
85
|
"focus-indicator": { severity: "low", title: "No visible focus indicator", help: "Tabbing to the control changes nothing on screen." },
|
|
80
86
|
"horizontal-scroll": { severity: "medium", title: "Page scrolls sideways", help: "Content is wider than the viewport." },
|
|
81
|
-
"tiny-target": {
|
|
87
|
+
"tiny-target": {
|
|
88
|
+
severity: "low",
|
|
89
|
+
title: "Small click target",
|
|
90
|
+
help: "Below the 24px WCAG 2.2 target-size minimum, measured on what the user clicks (a native input's label or drop zone), with another target inside its 24px circle.",
|
|
91
|
+
},
|
|
82
92
|
"clipped-text": { severity: "low", title: "Text clipped", help: "Text is wider than its box and cut off without an ellipsis." },
|
|
83
93
|
"image-aspect": { severity: "low", title: "Image distorted", help: "The rendered box does not match the image's proportions." },
|
|
84
94
|
"flow-step-failed": {
|
|
@@ -86,6 +96,11 @@ export const CHECK_RULES = {
|
|
|
86
96
|
title: "Saved flow broke",
|
|
87
97
|
help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
|
|
88
98
|
},
|
|
99
|
+
[VISUAL_RULE]: {
|
|
100
|
+
severity: "high",
|
|
101
|
+
title: "Differs from its visual baseline",
|
|
102
|
+
help: "A page or element listed in the baselines' targets.json no longer looks like its baseline (--baseline compare): more of its pixels changed than --baseline-threshold allows, or its size changed. It is also filed when the page or element could not be captured at all, or its baseline cannot be used (half there, unreadable, or taken with other settings). An intended change is approved by running the check with --baseline update (and committing the new baseline, where the project keeps baselines in a folder it commits).",
|
|
103
|
+
},
|
|
89
104
|
};
|
|
90
105
|
/**
|
|
91
106
|
* Rules whose measurement is exact and whose meaning depends on a convention
|
|
@@ -189,16 +204,34 @@ export function geometryRule(line) {
|
|
|
189
204
|
*
|
|
190
205
|
* A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
|
|
191
206
|
* the same way; the two lists never share an entry.
|
|
207
|
+
*
|
|
208
|
+
* Visual baselines add one issue per target that is not met (baseline.ts
|
|
209
|
+
* decides which). Each target is its own fact already, so these skip the
|
|
210
|
+
* dedup, and their fingerprint comes from the target, not the evidence: the
|
|
211
|
+
* evidence carries this run's percentage, and the same target changing by a
|
|
212
|
+
* different amount is still the same alert.
|
|
213
|
+
*
|
|
214
|
+
* `--ignore-path` drops a fact on the route it names, or one rule there,
|
|
215
|
+
* before the fact is filed. A fact also seen on a route that is not exempted
|
|
216
|
+
* stays, on that route alone.
|
|
192
217
|
*/
|
|
193
|
-
export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
218
|
+
export function checkFindings(routes, origin, ignore = [], flows = [], baselines = null, ignorePaths = []) {
|
|
194
219
|
const byKey = new Map();
|
|
195
220
|
const looks = new Map();
|
|
221
|
+
/** A route (and, when named, one rule on it) that --ignore-path exempts. */
|
|
222
|
+
const exempt = (rule, route) => ignorePaths.some((e) => e.path === route && (e.rule === undefined || e.rule === rule));
|
|
223
|
+
/** Evidence as it is written: no origin, no secret, and bounded. */
|
|
224
|
+
const cleanEvidence = (evidence) => redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
|
|
196
225
|
const add = (rule, evidence, route, opts = {}) => {
|
|
197
|
-
if (ignore.includes(rule))
|
|
226
|
+
if (ignore.includes(rule) || exempt(rule, route))
|
|
198
227
|
return;
|
|
199
|
-
const clean =
|
|
228
|
+
const clean = cleanEvidence(evidence);
|
|
200
229
|
const key = `${rule}\u0000${clean}`;
|
|
201
|
-
const
|
|
230
|
+
const issue = byKey.get(key);
|
|
231
|
+
// A fact a crawled page and a flow both show points at the flow: the flow is the file that reproduces it.
|
|
232
|
+
if (issue && opts.flow && !issue.flow)
|
|
233
|
+
issue.flow = opts.flow;
|
|
234
|
+
const found = issue ?? looks.get(key);
|
|
202
235
|
if (found) {
|
|
203
236
|
if (!found.routes.includes(route))
|
|
204
237
|
found.routes.push(route);
|
|
@@ -215,6 +248,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
215
248
|
evidence: clean,
|
|
216
249
|
routes: [route],
|
|
217
250
|
...(opts.embed ? { embed: opts.embed } : {}),
|
|
251
|
+
...(opts.flow ? { flow: opts.flow } : {}),
|
|
218
252
|
fingerprint,
|
|
219
253
|
});
|
|
220
254
|
};
|
|
@@ -251,19 +285,36 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
251
285
|
}
|
|
252
286
|
for (const f of flows) {
|
|
253
287
|
for (const { path, violation } of f.violations)
|
|
254
|
-
add(violationRule(violation), violation.detail, path, { embed: violation.embed });
|
|
288
|
+
add(violationRule(violation), violation.detail, path, { embed: violation.embed, flow: f.file });
|
|
255
289
|
// A refused step is not the app's defect: the check reports it as "could not run" (refusedFlowReason).
|
|
256
290
|
if (f.outcome.status === "failed")
|
|
257
|
-
add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
|
|
291
|
+
add("flow-step-failed", flowStepEvidence(f), f.outcome.path, { flow: f.file });
|
|
292
|
+
}
|
|
293
|
+
const visual = [];
|
|
294
|
+
if (baselines && !ignore.includes(VISUAL_RULE)) {
|
|
295
|
+
for (const r of baselines.results) {
|
|
296
|
+
if (exempt(VISUAL_RULE, r.path))
|
|
297
|
+
continue;
|
|
298
|
+
const evidence = baselineEvidence(r, baselines);
|
|
299
|
+
if (evidence === null)
|
|
300
|
+
continue;
|
|
301
|
+
visual.push({
|
|
302
|
+
rule: VISUAL_RULE,
|
|
303
|
+
severity: CHECK_RULES[VISUAL_RULE].severity,
|
|
304
|
+
evidence: cleanEvidence(evidence),
|
|
305
|
+
routes: [r.path],
|
|
306
|
+
fingerprint: baselineFingerprint(baselines.engine, r),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
258
309
|
}
|
|
259
310
|
return {
|
|
260
|
-
issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
311
|
+
issues: [...byKey.values(), ...visual].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
261
312
|
worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
|
|
262
313
|
};
|
|
263
314
|
}
|
|
264
315
|
/** The defect tier of `checkFindings`: what counts, and what the gate reads. */
|
|
265
|
-
export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
|
|
266
|
-
return checkFindings(routes, origin, ignore, flows).issues;
|
|
316
|
+
export function issuesFromRoutes(routes, origin, ignore = [], flows = [], ignorePaths = []) {
|
|
317
|
+
return checkFindings(routes, origin, ignore, flows, null, ignorePaths).issues;
|
|
267
318
|
}
|
|
268
319
|
/**
|
|
269
320
|
* Why a check has nothing to give a verdict on, or null when it measured the
|
|
@@ -299,15 +350,8 @@ export function withoutOwnResponse(r) {
|
|
|
299
350
|
const own = httpErrorDetail("GET", r.url.replace(/#.*$/, ""), r.status);
|
|
300
351
|
return { ...r, violations: r.violations.filter((v) => !(v.kind === "http_error" && v.detail === own)) };
|
|
301
352
|
}
|
|
302
|
-
/**
|
|
303
|
-
|
|
304
|
-
* (`?reset=…`, `?api_key=…`). Evidence is redacted where issues are built;
|
|
305
|
-
* this does the same for every route string before anything is written.
|
|
306
|
-
*/
|
|
307
|
-
export function redactRoute(route) {
|
|
308
|
-
// The trailing "[n secrets redacted]" note belongs to prose; in a route it would read as part of the path.
|
|
309
|
-
return redactSecrets(route).replace(/ \[\d+ secrets? redacted\]$/, "");
|
|
310
|
-
}
|
|
353
|
+
/** Lives beside redactSecrets so the baselines' file names (baseline.ts) use it too; exported here as before. */
|
|
354
|
+
export { redactRoute };
|
|
311
355
|
export function redactRoutes(routes) {
|
|
312
356
|
const clean = redactRoute;
|
|
313
357
|
return routes.map((r) => ({
|
|
@@ -317,6 +361,16 @@ export function redactRoutes(routes) {
|
|
|
317
361
|
violations: r.violations.map((v) => ({ ...v, url: clean(v.url) })),
|
|
318
362
|
}));
|
|
319
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* The same redaction for visual baselines: a target's path, and the reason a
|
|
366
|
+
* picture could not be taken, which can quote a URL the app redirected to.
|
|
367
|
+
*/
|
|
368
|
+
export function redactBaselineRun(run) {
|
|
369
|
+
return {
|
|
370
|
+
...run,
|
|
371
|
+
results: run.results.map((r) => ({ ...r, path: redactRoute(r.path), ...(r.detail !== undefined ? { detail: redactSecrets(r.detail) } : {}) })),
|
|
372
|
+
};
|
|
373
|
+
}
|
|
320
374
|
/** The same redaction for what flows measured: the page paths and the request URLs their violations quote. */
|
|
321
375
|
export function redactFlowRuns(runs) {
|
|
322
376
|
return runs.map((f) => ({
|
|
@@ -367,6 +421,7 @@ export const CHECK_OPTION_NAMES = [
|
|
|
367
421
|
"max-routes",
|
|
368
422
|
"paths",
|
|
369
423
|
"ignore",
|
|
424
|
+
"ignore-path",
|
|
370
425
|
"flows",
|
|
371
426
|
"retest",
|
|
372
427
|
"flow-writes",
|
|
@@ -374,6 +429,10 @@ export const CHECK_OPTION_NAMES = [
|
|
|
374
429
|
"gate-retests",
|
|
375
430
|
"action-timeout-ms",
|
|
376
431
|
"nav-timeout-ms",
|
|
432
|
+
"baseline",
|
|
433
|
+
"baselines",
|
|
434
|
+
"baseline-threshold",
|
|
435
|
+
"sarif-file-anchor",
|
|
377
436
|
];
|
|
378
437
|
export const MAX_CHECK_ROUTES = 150;
|
|
379
438
|
/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
|
|
@@ -457,6 +516,50 @@ export function parseCheckArgs(args, cwd) {
|
|
|
457
516
|
const unknownRules = ignore.filter((r) => !CHECK_RULE_IDS.includes(r));
|
|
458
517
|
if (unknownRules.length > 0)
|
|
459
518
|
return { ok: false, error: `unknown rule(s) in --ignore: ${unknownRules.join(", ")}. Rules: ${CHECK_RULE_IDS.join(", ")}` };
|
|
519
|
+
const ignorePathList = list(flags.get("ignore-path"));
|
|
520
|
+
if (ignorePathList && ignorePathList.length === 0)
|
|
521
|
+
return { ok: false, error: "--ignore-path is empty" };
|
|
522
|
+
const ignorePaths = [];
|
|
523
|
+
const unknownPathRules = [];
|
|
524
|
+
const badPathEntries = [];
|
|
525
|
+
const ruleAfterPath = [];
|
|
526
|
+
for (const entry of ignorePathList ?? []) {
|
|
527
|
+
if (entry.startsWith("/")) {
|
|
528
|
+
// `/path:rule` would otherwise be a path that never matches. A colon is
|
|
529
|
+
// still allowed in a path when what follows it is not a rule id.
|
|
530
|
+
const suffix = entry.slice(entry.lastIndexOf(":") + 1);
|
|
531
|
+
if (entry.includes(":") && CHECK_RULE_IDS.includes(suffix)) {
|
|
532
|
+
ruleAfterPath.push(entry);
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
ignorePaths.push({ path: entry });
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
const colon = entry.indexOf(":");
|
|
539
|
+
const rule = colon > 0 ? entry.slice(0, colon) : "";
|
|
540
|
+
const path = colon > 0 ? entry.slice(colon + 1) : "";
|
|
541
|
+
if (path.startsWith("/") && CHECK_RULE_IDS.includes(rule)) {
|
|
542
|
+
ignorePaths.push({ path, rule: rule });
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
if (path.startsWith("/") && rule) {
|
|
546
|
+
unknownPathRules.push(rule);
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
badPathEntries.push(entry);
|
|
550
|
+
}
|
|
551
|
+
if (unknownPathRules.length > 0) {
|
|
552
|
+
return { ok: false, error: `unknown rule(s) in --ignore-path: ${unknownPathRules.join(", ")}. Rules: ${CHECK_RULE_IDS.join(", ")}` };
|
|
553
|
+
}
|
|
554
|
+
if (ruleAfterPath.length > 0) {
|
|
555
|
+
return { ok: false, error: `--ignore-path names a rule as rule:/path (got ${ruleAfterPath.join(", ")})` };
|
|
556
|
+
}
|
|
557
|
+
if (badPathEntries.length > 0) {
|
|
558
|
+
return {
|
|
559
|
+
ok: false,
|
|
560
|
+
error: `--ignore-path entries are paths starting with /, or one rule on a path as rule:/path (got ${badPathEntries.join(", ")})`,
|
|
561
|
+
};
|
|
562
|
+
}
|
|
460
563
|
const retest = flags.get("retest") ?? "on";
|
|
461
564
|
if (retest !== "on" && retest !== "off")
|
|
462
565
|
return { ok: false, error: "--retest must be on or off" };
|
|
@@ -476,6 +579,22 @@ export function parseCheckArgs(args, cwd) {
|
|
|
476
579
|
const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
|
|
477
580
|
if (!gateRetests)
|
|
478
581
|
return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
|
|
582
|
+
const baseline = oneOf("baseline", BASELINE_MODES, "off");
|
|
583
|
+
if (!baseline)
|
|
584
|
+
return { ok: false, error: `--baseline must be one of ${BASELINE_MODES.join(", ")}` };
|
|
585
|
+
const threshold = parseThreshold(flags.get("baseline-threshold"));
|
|
586
|
+
if (!threshold.ok)
|
|
587
|
+
return threshold;
|
|
588
|
+
const baselinesDir = flags.get("baselines");
|
|
589
|
+
if (baselinesDir !== undefined && baselinesDir.trim() === "")
|
|
590
|
+
return { ok: false, error: "--baselines needs a directory" };
|
|
591
|
+
// A setting that would do nothing is a mistake to point out, not to ignore: the check would run without the baselines asked for.
|
|
592
|
+
if (baseline === "off" && (baselinesDir !== undefined || flags.has("baseline-threshold"))) {
|
|
593
|
+
return { ok: false, error: "--baselines and --baseline-threshold apply only with --baseline compare or --baseline update" };
|
|
594
|
+
}
|
|
595
|
+
const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
|
|
596
|
+
if (anchor && !anchor.ok)
|
|
597
|
+
return anchor;
|
|
479
598
|
const resolve = (p) => resolveArgPath(cwd, p);
|
|
480
599
|
return {
|
|
481
600
|
ok: true,
|
|
@@ -492,11 +611,16 @@ export function parseCheckArgs(args, cwd) {
|
|
|
492
611
|
maxRoutes,
|
|
493
612
|
...(paths ? { paths } : {}),
|
|
494
613
|
ignore: ignore,
|
|
614
|
+
ignorePaths,
|
|
495
615
|
...(flows !== undefined ? { flows: flows === "off" ? "off" : resolve(flows) } : {}),
|
|
496
616
|
retest: retest === "on",
|
|
497
617
|
flowWrites,
|
|
498
618
|
onRefusedStep,
|
|
499
619
|
gateRetests,
|
|
620
|
+
baseline,
|
|
621
|
+
...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
|
|
622
|
+
baselineThreshold: threshold.value,
|
|
623
|
+
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
500
624
|
},
|
|
501
625
|
};
|
|
502
626
|
}
|
|
@@ -550,10 +674,13 @@ export function describeSettings(result) {
|
|
|
550
674
|
}
|
|
551
675
|
const SARIF_LEVEL = { high: "error", medium: "warning", low: "note" };
|
|
552
676
|
/**
|
|
553
|
-
* SARIF 2.1.0, for code-scanning dashboards.
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
677
|
+
* SARIF 2.1.0, for code-scanning dashboards. Code scanning keeps a result
|
|
678
|
+
* only when its location is a repository file, so each result points at one
|
|
679
|
+
* (sarif.ts says which) and carries its routes in logical locations, in
|
|
680
|
+
* `properties.routes` and in its message. Fingerprints come from the
|
|
681
|
+
* evidence, not the location, so a preview deployment on a new URL does not
|
|
682
|
+
* reopen every alert; an unmet visual baseline's comes from its target and
|
|
683
|
+
* browser instead.
|
|
557
684
|
*/
|
|
558
685
|
const RETEST_SARIF_RULE = {
|
|
559
686
|
id: "open-finding-reproduces",
|
|
@@ -562,39 +689,39 @@ const RETEST_SARIF_RULE = {
|
|
|
562
689
|
help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
|
|
563
690
|
defaultConfiguration: { level: "error" },
|
|
564
691
|
};
|
|
565
|
-
function
|
|
566
|
-
return route === SHARED_CHROME_ROUTE
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
: { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
|
|
692
|
+
function routeLocation(file, route) {
|
|
693
|
+
return route === SHARED_CHROME_ROUTE ? sarifLocation(file, route, "the app's shared shell, on every page that renders it") : sarifLocation(file, route);
|
|
694
|
+
}
|
|
695
|
+
/** Where a result was seen, for its message: the first three routes and how many more. */
|
|
696
|
+
function onRoutes(routes) {
|
|
697
|
+
return ` — on ${routes.slice(0, 3).join(", ")}${routes.length > 3 ? ` and ${routes.length - 3} more` : ""}`;
|
|
572
698
|
}
|
|
573
|
-
|
|
699
|
+
/** `files` defaults to package.json as the anchor, with no flows directory; the CLI always passes what it resolved. */
|
|
700
|
+
export function toSarif(result, toolVersion, files = { anchor: SARIF_ANCHOR_FALLBACKS[0] }) {
|
|
574
701
|
const used = [...new Set(result.issues.map((i) => i.rule))];
|
|
575
702
|
const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
|
|
576
|
-
const base = new URL(result.url);
|
|
577
703
|
const reproducing = retestGateFailures(result);
|
|
578
704
|
const refused = result.flows.filter((f) => f.outcome.status === "refused");
|
|
579
705
|
const issueResults = result.issues.map((i) => ({
|
|
580
706
|
ruleId: i.rule,
|
|
581
707
|
level: SARIF_LEVEL[i.severity],
|
|
582
708
|
message: {
|
|
583
|
-
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.
|
|
709
|
+
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.embed ? ` (in an embed of ${i.embed})` : ""}${onRoutes(i.routes)}`,
|
|
584
710
|
},
|
|
585
|
-
locations: i.routes.slice(0, 10).map(
|
|
711
|
+
locations: i.routes.slice(0, 10).map((route) => routeLocation(sarifFileFor(files, i.flow), route)),
|
|
586
712
|
partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
|
|
713
|
+
properties: { routes: i.routes, ...(i.flow ? { flow: i.flow } : {}) },
|
|
587
714
|
}));
|
|
588
715
|
// Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
|
|
589
716
|
const lookResults = result.worthALook.map((o) => ({
|
|
590
717
|
ruleId: o.rule,
|
|
591
718
|
level: "note",
|
|
592
719
|
message: {
|
|
593
|
-
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${
|
|
720
|
+
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${onRoutes(o.routes)}. A defect only if your project uses ${o.convention}.`,
|
|
594
721
|
},
|
|
595
|
-
locations: o.routes.slice(0, 10).map(
|
|
722
|
+
locations: o.routes.slice(0, 10).map((route) => routeLocation(files.anchor, route)),
|
|
596
723
|
partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
|
|
597
|
-
properties: { tier: "worth-a-look", convention: o.convention },
|
|
724
|
+
properties: { tier: "worth-a-look", convention: o.convention, routes: o.routes },
|
|
598
725
|
}));
|
|
599
726
|
// Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
|
|
600
727
|
const retestResults = reproducing.map((r) => ({
|
|
@@ -602,9 +729,10 @@ export function toSarif(result, toolVersion) {
|
|
|
602
729
|
// The severity the finding was filed at, as the page rules map theirs; an unknown one reads as the worst.
|
|
603
730
|
level: SARIF_LEVEL[CHECK_SEVERITIES.includes(r.severity) ? r.severity : "high"],
|
|
604
731
|
message: {
|
|
605
|
-
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
732
|
+
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")}${onRoutes([r.path])} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
606
733
|
},
|
|
607
|
-
locations: [
|
|
734
|
+
locations: [routeLocation(files.anchor, r.path)],
|
|
735
|
+
properties: { routes: [r.path] },
|
|
608
736
|
partialFingerprints: { "scenescoutCheck/v1": createHash("sha256").update(`retest\u0000${r.id}`).digest("hex").slice(0, 32) },
|
|
609
737
|
}));
|
|
610
738
|
return {
|
|
@@ -648,7 +776,7 @@ export function toSarif(result, toolVersion) {
|
|
|
648
776
|
})),
|
|
649
777
|
},
|
|
650
778
|
],
|
|
651
|
-
|
|
779
|
+
properties: { app: new URL(result.url).origin },
|
|
652
780
|
results: [...issueResults, ...lookResults, ...retestResults],
|
|
653
781
|
},
|
|
654
782
|
],
|
|
@@ -679,6 +807,69 @@ function routeList(routes) {
|
|
|
679
807
|
const shown = routes.slice(0, 3).map(code);
|
|
680
808
|
return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
|
|
681
809
|
}
|
|
810
|
+
const BASELINE_MARK = {
|
|
811
|
+
matches: "✓",
|
|
812
|
+
changed: "✗",
|
|
813
|
+
"no-baseline": "○",
|
|
814
|
+
unusable: "✗",
|
|
815
|
+
"not-captured": "⊘",
|
|
816
|
+
updated: "↻",
|
|
817
|
+
};
|
|
818
|
+
/** One target's line in the report. */
|
|
819
|
+
function baselineLine(r, run) {
|
|
820
|
+
const what = `${BASELINE_MARK[r.status]} ${code(r.element)} on ${code(r.path)}`;
|
|
821
|
+
const notes = [r.platformNote, r.partial]
|
|
822
|
+
.filter(Boolean)
|
|
823
|
+
.map((n) => ` _(${cell(n)})_`)
|
|
824
|
+
.join("");
|
|
825
|
+
switch (r.status) {
|
|
826
|
+
case "matches":
|
|
827
|
+
return `- ${what}: ${run.mode === "update" ? `within the threshold of its baseline (${r.diff?.percent ?? 0}% changed), which was left as it was` : `matches (${r.diff?.percent ?? 0}% changed)`}${notes}`;
|
|
828
|
+
case "changed": {
|
|
829
|
+
const files = r.files ? ` — expected ${code(r.files.expected)}, now ${code(r.files.actual)}, diff ${code(r.files.diff)}` : "";
|
|
830
|
+
return `- ${what}: ${r.diff ? describeChange(r.diff, run.threshold) : "it no longer matches its baseline"}${files}${notes}`;
|
|
831
|
+
}
|
|
832
|
+
case "no-baseline":
|
|
833
|
+
return `- ${what}: no baseline yet${notes}`;
|
|
834
|
+
case "unusable":
|
|
835
|
+
return `- ${what}: its baseline cannot be used: ${cell(r.detail ?? "no reason given")}`;
|
|
836
|
+
case "not-captured":
|
|
837
|
+
return `- ${what}: could not be captured: ${cell(r.detail ?? "no reason given")}${run.mode === "update" ? "; its baseline was not written" : ""}`;
|
|
838
|
+
case "updated":
|
|
839
|
+
return `- ${what}: baseline written to ${code(r.baseline)} (${cell(r.detail ?? "updated")})${notes}`;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
/** The report's section on visual baselines: every target and what became of it. */
|
|
843
|
+
function formatBaselines(run) {
|
|
844
|
+
const c = countStatuses(run.results);
|
|
845
|
+
const counts = [
|
|
846
|
+
["changed", "changed"],
|
|
847
|
+
["unusable", "unusable"],
|
|
848
|
+
["not-captured", "not captured"],
|
|
849
|
+
["no-baseline", "no baseline yet"],
|
|
850
|
+
["updated", "updated"],
|
|
851
|
+
["matches", run.mode === "update" ? "left as they were" : "match"],
|
|
852
|
+
]
|
|
853
|
+
.filter(([k]) => c[k] > 0)
|
|
854
|
+
.map(([k, label]) => `${c[k]} ${label}`)
|
|
855
|
+
.join(" · ");
|
|
856
|
+
const lines = [
|
|
857
|
+
`## Visual baselines (${run.results.length})`,
|
|
858
|
+
"",
|
|
859
|
+
`${run.mode === "update" ? "Updated" : "Compared"} in ${run.engine}, baselines in ${code(run.dir)} · ${run.threshold}% of a picture's pixels may change · ${counts}`,
|
|
860
|
+
"",
|
|
861
|
+
...run.results.map((r) => baselineLine(r, run)),
|
|
862
|
+
];
|
|
863
|
+
if (c.changed > 0)
|
|
864
|
+
lines.push("", "The pictures are beside this report, under `visual/` (in the job's artifact on CI): the baseline, the picture now, and the changed pixels in red.");
|
|
865
|
+
if (c["no-baseline"] > 0 || c.unusable > 0) {
|
|
866
|
+
lines.push("", "A target with no baseline yet is listed and never fails; one whose baseline cannot be used fails until it is taken again. Run the check with `--baseline update` to take them.");
|
|
867
|
+
}
|
|
868
|
+
if (run.results.some((r) => r.platformNote)) {
|
|
869
|
+
lines.push("", "_Some baselines were taken on another operating system. Each one draws text differently, so a change there may be the system's rather than the app's: take baselines where the check runs._");
|
|
870
|
+
}
|
|
871
|
+
return lines;
|
|
872
|
+
}
|
|
682
873
|
/** One markdown list item for an issue: its title, rule, evidence and routes. */
|
|
683
874
|
export function issueLine(i) {
|
|
684
875
|
return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
|
|
@@ -740,6 +931,8 @@ export function formatCheck(result) {
|
|
|
740
931
|
lines.push(`${result.url} · ${result.routes.length} route(s) · mode ${result.mode} · gate: ${result.failOn === "never" ? "report only" : `fail on ${result.failOn}${result.failOn === "low" ? "" : " or worse"}`}`);
|
|
741
932
|
lines.push("");
|
|
742
933
|
const unaudited = unauditedRoutes(result.routes).length;
|
|
934
|
+
// Nothing compared is not a match: a target with no baseline yet is named beside the verdict, as an unmeasured design is.
|
|
935
|
+
const uncompared = result.baselines?.mode === "compare" ? countStatuses(result.baselines.results)["no-baseline"] : 0;
|
|
743
936
|
const failingText = retestsFailing > 0
|
|
744
937
|
? `${failing} failing the gate (${failing - retestsFailing} issue(s), ${retestsFailing} re-tested finding(s))`
|
|
745
938
|
: `${failing} issue(s) at the gate's severity`;
|
|
@@ -748,6 +941,7 @@ export function formatCheck(result) {
|
|
|
748
941
|
? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
|
|
749
942
|
: `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
|
|
750
943
|
(unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
|
|
944
|
+
(uncompared > 0 ? ` · ${uncompared} visual target(s) not compared: no baseline yet` : "") +
|
|
751
945
|
(result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
|
|
752
946
|
// Right under the verdict: what a green check was allowed to do is part of what it means.
|
|
753
947
|
lines.push("", `Settings — ${describeSettings(result)}`);
|
|
@@ -772,6 +966,8 @@ export function formatCheck(result) {
|
|
|
772
966
|
for (const s of result.skippedFlows)
|
|
773
967
|
lines.push(`- skipped ${code(s.file)}: ${cell(s.reason)}`);
|
|
774
968
|
}
|
|
969
|
+
if (result.baselines)
|
|
970
|
+
lines.push("", ...formatBaselines(result.baselines));
|
|
775
971
|
if (result.retest && result.retest.open > 0) {
|
|
776
972
|
const { open, results } = result.retest;
|
|
777
973
|
lines.push("", `## Open findings re-tested (${results.length} of ${open})`, "");
|
|
@@ -798,6 +994,10 @@ export function formatCheck(result) {
|
|
|
798
994
|
}
|
|
799
995
|
if (result.ignored.length > 0)
|
|
800
996
|
lines.push("", `Rules ignored by --ignore: ${result.ignored.join(", ")}`);
|
|
997
|
+
if (result.ignoredPaths.length > 0) {
|
|
998
|
+
const shown = result.ignoredPaths.map((e) => (e.rule ? `${e.rule}:${e.path}` : e.path));
|
|
999
|
+
lines.push("", `Paths exempted by --ignore-path: ${shown.join(", ")}`);
|
|
1000
|
+
}
|
|
801
1001
|
lines.push("", result.flows.length > 0
|
|
802
1002
|
? "_A check visits pages, measures what loads and replays the flows saved for it. It does not explore, fill forms of its own accord or compare roles; an exploratory run does that._"
|
|
803
1003
|
: "_A check visits pages and measures what loads. It does not fill forms, click through flows or compare roles; an exploratory run does that. Flows saved in .scenescout/flows are replayed._");
|
|
@@ -826,6 +1026,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
826
1026
|
unvisited: result.unvisited,
|
|
827
1027
|
...(result.timeBudget ? { timeBudget: result.timeBudget } : {}),
|
|
828
1028
|
ignored: result.ignored,
|
|
1029
|
+
ignoredPaths: result.ignoredPaths,
|
|
829
1030
|
flows: result.flows.map((f) => ({
|
|
830
1031
|
name: f.name,
|
|
831
1032
|
file: f.file,
|
|
@@ -837,6 +1038,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
837
1038
|
})),
|
|
838
1039
|
skippedFlows: result.skippedFlows,
|
|
839
1040
|
retest: result.retest,
|
|
1041
|
+
baselines: result.baselines ?? null,
|
|
840
1042
|
issues: result.issues,
|
|
841
1043
|
// Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
|
|
842
1044
|
worthALook: result.worthALook,
|