scenescout 3.14.1 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +90 -18
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +211 -24
- package/dist/ci-run.js +374 -62
- package/dist/cli.js +221 -47
- package/dist/commands.js +42 -4
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1660 -386
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +258 -68
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +233 -18
- package/dist/engine/claims.js +234 -12
- package/dist/engine/collector.js +851 -78
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/dedup.js +329 -9
- package/dist/engine/design.js +355 -43
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +198 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +668 -70
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +167 -2
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/provider.js +4 -3
- package/dist/engine/refresh.js +338 -25
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +248 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/scripted-login.js +342 -62
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +581 -0
- package/dist/installer.js +267 -3
- package/dist/intake.js +104 -0
- package/dist/login-run.js +386 -71
- package/dist/mcp-server.js +637 -68
- package/dist/playbook.js +5 -0
- package/package.json +6 -3
- package/skills/scenescout/SKILL.md +51 -18
package/dist/engine/check.js
CHANGED
|
@@ -12,9 +12,11 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { createHash } from "node:crypto";
|
|
14
14
|
import { BROWSER_ENGINES } from "../browsers.js";
|
|
15
|
+
import { BASELINE_MODES, baselineEvidence, baselineFingerprint, countStatuses, describeChange, parseThreshold, VISUAL_RULE, } from "./baseline.js";
|
|
15
16
|
import { flowStepEvidence } from "./flow.js";
|
|
16
17
|
import { parseLimitFlag } from "./limits.js";
|
|
17
|
-
import { redactSecrets } from "./memory.js";
|
|
18
|
+
import { redactRoute, redactSecrets } from "./memory.js";
|
|
19
|
+
import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifFileFor, sarifLocation } from "./sarif.js";
|
|
18
20
|
import { httpErrorDetail, httpStatusOf } from "./oracles.js";
|
|
19
21
|
export const CHECK_SEVERITIES = ["high", "medium", "low"];
|
|
20
22
|
export const FAIL_ON = [...CHECK_SEVERITIES, "never"];
|
|
@@ -65,7 +67,11 @@ export const CHECK_RULES = {
|
|
|
65
67
|
title: "Control covered by pinned chrome",
|
|
66
68
|
help: "A fixed or sticky element sits on top of the control, so clicks land on it instead.",
|
|
67
69
|
},
|
|
68
|
-
"clipped-control": {
|
|
70
|
+
"clipped-control": {
|
|
71
|
+
severity: "medium",
|
|
72
|
+
title: "Control unreachable",
|
|
73
|
+
help: "The control is clipped inside a container that cannot scroll and has no control beside it to page it.",
|
|
74
|
+
},
|
|
69
75
|
"offpage-control": { severity: "medium", title: "Control outside the page", help: "The control is laid out where no scrolling can reach it." },
|
|
70
76
|
"overlapping-controls": { severity: "low", title: "Controls overlap", help: "Two controls in the same layer cover most of each other." },
|
|
71
77
|
"broken-image": { severity: "medium", title: "Broken image", help: "The browser could not render the image." },
|
|
@@ -78,7 +84,11 @@ export const CHECK_RULES = {
|
|
|
78
84
|
contrast: { severity: "low", title: "Text contrast below WCAG", help: "Text needs 4.5:1 (3:1 when large) against its background." },
|
|
79
85
|
"focus-indicator": { severity: "low", title: "No visible focus indicator", help: "Tabbing to the control changes nothing on screen." },
|
|
80
86
|
"horizontal-scroll": { severity: "medium", title: "Page scrolls sideways", help: "Content is wider than the viewport." },
|
|
81
|
-
"tiny-target": {
|
|
87
|
+
"tiny-target": {
|
|
88
|
+
severity: "low",
|
|
89
|
+
title: "Small click target",
|
|
90
|
+
help: "Below the 24px WCAG 2.2 target-size minimum, measured on what the user clicks (a native input's label or drop zone), with another target inside its 24px circle.",
|
|
91
|
+
},
|
|
82
92
|
"clipped-text": { severity: "low", title: "Text clipped", help: "Text is wider than its box and cut off without an ellipsis." },
|
|
83
93
|
"image-aspect": { severity: "low", title: "Image distorted", help: "The rendered box does not match the image's proportions." },
|
|
84
94
|
"flow-step-failed": {
|
|
@@ -86,6 +96,11 @@ export const CHECK_RULES = {
|
|
|
86
96
|
title: "Saved flow broke",
|
|
87
97
|
help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
|
|
88
98
|
},
|
|
99
|
+
[VISUAL_RULE]: {
|
|
100
|
+
severity: "high",
|
|
101
|
+
title: "Differs from its visual baseline",
|
|
102
|
+
help: "A page or element listed in the baselines' targets.json no longer looks like its baseline (--baseline compare): more of its pixels changed than --baseline-threshold allows, or its size changed. It is also filed when the page or element could not be captured at all, or its baseline cannot be used (half there, unreadable, or taken with other settings). An intended change is approved by running the check with --baseline update (and committing the new baseline, where the project keeps baselines in a folder it commits).",
|
|
103
|
+
},
|
|
89
104
|
};
|
|
90
105
|
/**
|
|
91
106
|
* Rules whose measurement is exact and whose meaning depends on a convention
|
|
@@ -107,12 +122,19 @@ export const WORTH_A_LOOK_RULES = {
|
|
|
107
122
|
help: "Links with no underline, in the same colour as the page's body text. In running text a reader cannot tell them from the text around them; in navigation this styling is common, and the check cannot tell the two apart.",
|
|
108
123
|
convention: "a visible link style (an underline or a distinct colour) wherever links appear, navigation included",
|
|
109
124
|
},
|
|
125
|
+
"scrolled-out-controls": {
|
|
126
|
+
title: "Controls scrolled out of view sideways",
|
|
127
|
+
help: "Controls inside a horizontally scrolling container (a wide table's last column, say) lie outside its visible width at this viewport, so only a sideways scroll of the container shows them. A wide table that scrolls is a common, deliberate layout; it matters where a project keeps every row's actions in view.",
|
|
128
|
+
convention: "row actions and other controls that stay in view without a sideways scroll of their container at this viewport width",
|
|
129
|
+
},
|
|
110
130
|
};
|
|
111
131
|
export const CHECK_RULE_IDS = [...Object.keys(CHECK_RULES), ...Object.keys(WORTH_A_LOOK_RULES)];
|
|
112
132
|
export function isWorthALookRule(rule) {
|
|
113
133
|
return Object.prototype.hasOwnProperty.call(WORTH_A_LOOK_RULES, rule);
|
|
114
134
|
}
|
|
115
|
-
const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
|
|
135
|
+
export const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
|
|
136
|
+
/** The route a defect of the app's shared shell is charged to: one fix, however many pages show it. */
|
|
137
|
+
export const SHARED_CHROME_ROUTE = "(shared chrome)";
|
|
116
138
|
/** Snapshot refs (`e12`) are numbered per run; evidence carrying them would never match itself twice. */
|
|
117
139
|
function stripRefs(line) {
|
|
118
140
|
return line.replace(/\be\d+\s+/g, "").trim();
|
|
@@ -166,6 +188,8 @@ export function geometryRule(line) {
|
|
|
166
188
|
return "offpage-control";
|
|
167
189
|
if (/ overlaps /.test(line))
|
|
168
190
|
return "overlapping-controls";
|
|
191
|
+
if (/ scrolled out of view inside a horizontally scrolling container /.test(line))
|
|
192
|
+
return "scrolled-out-controls";
|
|
169
193
|
return "layout-issue";
|
|
170
194
|
}
|
|
171
195
|
/**
|
|
@@ -180,16 +204,28 @@ export function geometryRule(line) {
|
|
|
180
204
|
*
|
|
181
205
|
* A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
|
|
182
206
|
* the same way; the two lists never share an entry.
|
|
207
|
+
*
|
|
208
|
+
* Visual baselines add one issue per target that is not met (baseline.ts
|
|
209
|
+
* decides which). Each target is its own fact already, so these skip the
|
|
210
|
+
* dedup, and their fingerprint comes from the target, not the evidence: the
|
|
211
|
+
* evidence carries this run's percentage, and the same target changing by a
|
|
212
|
+
* different amount is still the same alert.
|
|
183
213
|
*/
|
|
184
|
-
export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
214
|
+
export function checkFindings(routes, origin, ignore = [], flows = [], baselines = null) {
|
|
185
215
|
const byKey = new Map();
|
|
186
216
|
const looks = new Map();
|
|
217
|
+
/** Evidence as it is written: no origin, no secret, and bounded. */
|
|
218
|
+
const cleanEvidence = (evidence) => redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
|
|
187
219
|
const add = (rule, evidence, route, opts = {}) => {
|
|
188
220
|
if (ignore.includes(rule))
|
|
189
221
|
return;
|
|
190
|
-
const clean =
|
|
222
|
+
const clean = cleanEvidence(evidence);
|
|
191
223
|
const key = `${rule}\u0000${clean}`;
|
|
192
|
-
const
|
|
224
|
+
const issue = byKey.get(key);
|
|
225
|
+
// A fact a crawled page and a flow both show points at the flow: the flow is the file that reproduces it.
|
|
226
|
+
if (issue && opts.flow && !issue.flow)
|
|
227
|
+
issue.flow = opts.flow;
|
|
228
|
+
const found = issue ?? looks.get(key);
|
|
193
229
|
if (found) {
|
|
194
230
|
if (!found.routes.includes(route))
|
|
195
231
|
found.routes.push(route);
|
|
@@ -206,6 +242,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
206
242
|
evidence: clean,
|
|
207
243
|
routes: [route],
|
|
208
244
|
...(opts.embed ? { embed: opts.embed } : {}),
|
|
245
|
+
...(opts.flow ? { flow: opts.flow } : {}),
|
|
209
246
|
fingerprint,
|
|
210
247
|
});
|
|
211
248
|
};
|
|
@@ -238,17 +275,32 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
238
275
|
for (const p of r.placeholderOnly)
|
|
239
276
|
add("placeholder-only-label", p, route);
|
|
240
277
|
for (const d of r.design)
|
|
241
|
-
add(d.rule, d.detail, d.chrome ?
|
|
278
|
+
add(d.rule, d.detail, d.chrome ? SHARED_CHROME_ROUTE : route);
|
|
242
279
|
}
|
|
243
280
|
for (const f of flows) {
|
|
244
281
|
for (const { path, violation } of f.violations)
|
|
245
|
-
add(violationRule(violation), violation.detail, path, { embed: violation.embed });
|
|
282
|
+
add(violationRule(violation), violation.detail, path, { embed: violation.embed, flow: f.file });
|
|
246
283
|
// A refused step is not the app's defect: the check reports it as "could not run" (refusedFlowReason).
|
|
247
284
|
if (f.outcome.status === "failed")
|
|
248
|
-
add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
|
|
285
|
+
add("flow-step-failed", flowStepEvidence(f), f.outcome.path, { flow: f.file });
|
|
286
|
+
}
|
|
287
|
+
const visual = [];
|
|
288
|
+
if (baselines && !ignore.includes(VISUAL_RULE)) {
|
|
289
|
+
for (const r of baselines.results) {
|
|
290
|
+
const evidence = baselineEvidence(r, baselines);
|
|
291
|
+
if (evidence === null)
|
|
292
|
+
continue;
|
|
293
|
+
visual.push({
|
|
294
|
+
rule: VISUAL_RULE,
|
|
295
|
+
severity: CHECK_RULES[VISUAL_RULE].severity,
|
|
296
|
+
evidence: cleanEvidence(evidence),
|
|
297
|
+
routes: [r.path],
|
|
298
|
+
fingerprint: baselineFingerprint(baselines.engine, r),
|
|
299
|
+
});
|
|
300
|
+
}
|
|
249
301
|
}
|
|
250
302
|
return {
|
|
251
|
-
issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
303
|
+
issues: [...byKey.values(), ...visual].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
252
304
|
worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
|
|
253
305
|
};
|
|
254
306
|
}
|
|
@@ -290,15 +342,8 @@ export function withoutOwnResponse(r) {
|
|
|
290
342
|
const own = httpErrorDetail("GET", r.url.replace(/#.*$/, ""), r.status);
|
|
291
343
|
return { ...r, violations: r.violations.filter((v) => !(v.kind === "http_error" && v.detail === own)) };
|
|
292
344
|
}
|
|
293
|
-
/**
|
|
294
|
-
|
|
295
|
-
* (`?reset=…`, `?api_key=…`). Evidence is redacted where issues are built;
|
|
296
|
-
* this does the same for every route string before anything is written.
|
|
297
|
-
*/
|
|
298
|
-
export function redactRoute(route) {
|
|
299
|
-
// The trailing "[n secrets redacted]" note belongs to prose; in a route it would read as part of the path.
|
|
300
|
-
return redactSecrets(route).replace(/ \[\d+ secrets? redacted\]$/, "");
|
|
301
|
-
}
|
|
345
|
+
/** Lives beside redactSecrets so the baselines' file names (baseline.ts) use it too; exported here as before. */
|
|
346
|
+
export { redactRoute };
|
|
302
347
|
export function redactRoutes(routes) {
|
|
303
348
|
const clean = redactRoute;
|
|
304
349
|
return routes.map((r) => ({
|
|
@@ -308,6 +353,16 @@ export function redactRoutes(routes) {
|
|
|
308
353
|
violations: r.violations.map((v) => ({ ...v, url: clean(v.url) })),
|
|
309
354
|
}));
|
|
310
355
|
}
|
|
356
|
+
/**
|
|
357
|
+
* The same redaction for visual baselines: a target's path, and the reason a
|
|
358
|
+
* picture could not be taken, which can quote a URL the app redirected to.
|
|
359
|
+
*/
|
|
360
|
+
export function redactBaselineRun(run) {
|
|
361
|
+
return {
|
|
362
|
+
...run,
|
|
363
|
+
results: run.results.map((r) => ({ ...r, path: redactRoute(r.path), ...(r.detail !== undefined ? { detail: redactSecrets(r.detail) } : {}) })),
|
|
364
|
+
};
|
|
365
|
+
}
|
|
311
366
|
/** The same redaction for what flows measured: the page paths and the request URLs their violations quote. */
|
|
312
367
|
export function redactFlowRuns(runs) {
|
|
313
368
|
return runs.map((f) => ({
|
|
@@ -365,9 +420,19 @@ export const CHECK_OPTION_NAMES = [
|
|
|
365
420
|
"gate-retests",
|
|
366
421
|
"action-timeout-ms",
|
|
367
422
|
"nav-timeout-ms",
|
|
423
|
+
"baseline",
|
|
424
|
+
"baselines",
|
|
425
|
+
"baseline-threshold",
|
|
426
|
+
"sarif-file-anchor",
|
|
368
427
|
];
|
|
369
428
|
export const MAX_CHECK_ROUTES = 150;
|
|
429
|
+
/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
|
|
430
|
+
export const MAX_DISCOVERY_ROUNDS = 6;
|
|
370
431
|
export const DEFAULT_CHECK_ROUTES = 50;
|
|
432
|
+
/** A path given on the command line, resolved from the directory the command runs in; the same on every platform. */
|
|
433
|
+
export function resolveArgPath(cwd, p) {
|
|
434
|
+
return p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`;
|
|
435
|
+
}
|
|
371
436
|
/** Parse `scenescout check` arguments. Every mistake is a sentence, never a half-configured run. */
|
|
372
437
|
export function parseCheckArgs(args, cwd) {
|
|
373
438
|
const positional = [];
|
|
@@ -461,7 +526,23 @@ export function parseCheckArgs(args, cwd) {
|
|
|
461
526
|
const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
|
|
462
527
|
if (!gateRetests)
|
|
463
528
|
return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
|
|
464
|
-
const
|
|
529
|
+
const baseline = oneOf("baseline", BASELINE_MODES, "off");
|
|
530
|
+
if (!baseline)
|
|
531
|
+
return { ok: false, error: `--baseline must be one of ${BASELINE_MODES.join(", ")}` };
|
|
532
|
+
const threshold = parseThreshold(flags.get("baseline-threshold"));
|
|
533
|
+
if (!threshold.ok)
|
|
534
|
+
return threshold;
|
|
535
|
+
const baselinesDir = flags.get("baselines");
|
|
536
|
+
if (baselinesDir !== undefined && baselinesDir.trim() === "")
|
|
537
|
+
return { ok: false, error: "--baselines needs a directory" };
|
|
538
|
+
// A setting that would do nothing is a mistake to point out, not to ignore: the check would run without the baselines asked for.
|
|
539
|
+
if (baseline === "off" && (baselinesDir !== undefined || flags.has("baseline-threshold"))) {
|
|
540
|
+
return { ok: false, error: "--baselines and --baseline-threshold apply only with --baseline compare or --baseline update" };
|
|
541
|
+
}
|
|
542
|
+
const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
|
|
543
|
+
if (anchor && !anchor.ok)
|
|
544
|
+
return anchor;
|
|
545
|
+
const resolve = (p) => resolveArgPath(cwd, p);
|
|
465
546
|
return {
|
|
466
547
|
ok: true,
|
|
467
548
|
options: {
|
|
@@ -482,6 +563,10 @@ export function parseCheckArgs(args, cwd) {
|
|
|
482
563
|
flowWrites,
|
|
483
564
|
onRefusedStep,
|
|
484
565
|
gateRetests,
|
|
566
|
+
baseline,
|
|
567
|
+
...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
|
|
568
|
+
baselineThreshold: threshold.value,
|
|
569
|
+
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
485
570
|
},
|
|
486
571
|
};
|
|
487
572
|
}
|
|
@@ -535,10 +620,13 @@ export function describeSettings(result) {
|
|
|
535
620
|
}
|
|
536
621
|
const SARIF_LEVEL = { high: "error", medium: "warning", low: "note" };
|
|
537
622
|
/**
|
|
538
|
-
* SARIF 2.1.0, for code-scanning dashboards.
|
|
539
|
-
*
|
|
540
|
-
*
|
|
541
|
-
*
|
|
623
|
+
* SARIF 2.1.0, for code-scanning dashboards. Code scanning keeps a result
|
|
624
|
+
* only when its location is a repository file, so each result points at one
|
|
625
|
+
* (sarif.ts says which) and carries its routes in logical locations, in
|
|
626
|
+
* `properties.routes` and in its message. Fingerprints come from the
|
|
627
|
+
* evidence, not the location, so a preview deployment on a new URL does not
|
|
628
|
+
* reopen every alert; an unmet visual baseline's comes from its target and
|
|
629
|
+
* browser instead.
|
|
542
630
|
*/
|
|
543
631
|
const RETEST_SARIF_RULE = {
|
|
544
632
|
id: "open-finding-reproduces",
|
|
@@ -547,39 +635,39 @@ const RETEST_SARIF_RULE = {
|
|
|
547
635
|
help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
|
|
548
636
|
defaultConfiguration: { level: "error" },
|
|
549
637
|
};
|
|
550
|
-
function
|
|
551
|
-
return route === "
|
|
552
|
-
? {
|
|
553
|
-
physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
|
|
554
|
-
message: { text: "the app's shared shell, on every page that renders it" },
|
|
555
|
-
}
|
|
556
|
-
: { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
|
|
638
|
+
function routeLocation(file, route) {
|
|
639
|
+
return route === SHARED_CHROME_ROUTE ? sarifLocation(file, route, "the app's shared shell, on every page that renders it") : sarifLocation(file, route);
|
|
557
640
|
}
|
|
558
|
-
|
|
641
|
+
/** Where a result was seen, for its message: the first three routes and how many more. */
|
|
642
|
+
function onRoutes(routes) {
|
|
643
|
+
return ` — on ${routes.slice(0, 3).join(", ")}${routes.length > 3 ? ` and ${routes.length - 3} more` : ""}`;
|
|
644
|
+
}
|
|
645
|
+
/** `files` defaults to package.json as the anchor, with no flows directory; the CLI always passes what it resolved. */
|
|
646
|
+
export function toSarif(result, toolVersion, files = { anchor: SARIF_ANCHOR_FALLBACKS[0] }) {
|
|
559
647
|
const used = [...new Set(result.issues.map((i) => i.rule))];
|
|
560
648
|
const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
|
|
561
|
-
const base = new URL(result.url);
|
|
562
649
|
const reproducing = retestGateFailures(result);
|
|
563
650
|
const refused = result.flows.filter((f) => f.outcome.status === "refused");
|
|
564
651
|
const issueResults = result.issues.map((i) => ({
|
|
565
652
|
ruleId: i.rule,
|
|
566
653
|
level: SARIF_LEVEL[i.severity],
|
|
567
654
|
message: {
|
|
568
|
-
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.
|
|
655
|
+
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.embed ? ` (in an embed of ${i.embed})` : ""}${onRoutes(i.routes)}`,
|
|
569
656
|
},
|
|
570
|
-
locations: i.routes.slice(0, 10).map(
|
|
657
|
+
locations: i.routes.slice(0, 10).map((route) => routeLocation(sarifFileFor(files, i.flow), route)),
|
|
571
658
|
partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
|
|
659
|
+
properties: { routes: i.routes, ...(i.flow ? { flow: i.flow } : {}) },
|
|
572
660
|
}));
|
|
573
661
|
// Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
|
|
574
662
|
const lookResults = result.worthALook.map((o) => ({
|
|
575
663
|
ruleId: o.rule,
|
|
576
664
|
level: "note",
|
|
577
665
|
message: {
|
|
578
|
-
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${
|
|
666
|
+
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${onRoutes(o.routes)}. A defect only if your project uses ${o.convention}.`,
|
|
579
667
|
},
|
|
580
|
-
locations: o.routes.slice(0, 10).map(
|
|
668
|
+
locations: o.routes.slice(0, 10).map((route) => routeLocation(files.anchor, route)),
|
|
581
669
|
partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
|
|
582
|
-
properties: { tier: "worth-a-look", convention: o.convention },
|
|
670
|
+
properties: { tier: "worth-a-look", convention: o.convention, routes: o.routes },
|
|
583
671
|
}));
|
|
584
672
|
// Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
|
|
585
673
|
const retestResults = reproducing.map((r) => ({
|
|
@@ -587,9 +675,10 @@ export function toSarif(result, toolVersion) {
|
|
|
587
675
|
// The severity the finding was filed at, as the page rules map theirs; an unknown one reads as the worst.
|
|
588
676
|
level: SARIF_LEVEL[CHECK_SEVERITIES.includes(r.severity) ? r.severity : "high"],
|
|
589
677
|
message: {
|
|
590
|
-
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
678
|
+
text: `Open finding still reproduces: [${r.severity}] ${r.title} (${r.id}) — ${r.signatures.join(", ")}${onRoutes([r.path])} (gated by --gate-retests ${result.settings.gateRetests})`,
|
|
591
679
|
},
|
|
592
|
-
locations: [
|
|
680
|
+
locations: [routeLocation(files.anchor, r.path)],
|
|
681
|
+
properties: { routes: [r.path] },
|
|
593
682
|
partialFingerprints: { "scenescoutCheck/v1": createHash("sha256").update(`retest\u0000${r.id}`).digest("hex").slice(0, 32) },
|
|
594
683
|
}));
|
|
595
684
|
return {
|
|
@@ -633,7 +722,7 @@ export function toSarif(result, toolVersion) {
|
|
|
633
722
|
})),
|
|
634
723
|
},
|
|
635
724
|
],
|
|
636
|
-
|
|
725
|
+
properties: { app: new URL(result.url).origin },
|
|
637
726
|
results: [...issueResults, ...lookResults, ...retestResults],
|
|
638
727
|
},
|
|
639
728
|
],
|
|
@@ -659,10 +748,126 @@ function code(text) {
|
|
|
659
748
|
const pad = longest > 0 ? " " : "";
|
|
660
749
|
return `${fence}${pad}${text}${pad}${fence}`;
|
|
661
750
|
}
|
|
751
|
+
/** The first three routes as code spans, and how many more. */
|
|
662
752
|
function routeList(routes) {
|
|
663
753
|
const shown = routes.slice(0, 3).map(code);
|
|
664
754
|
return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
|
|
665
755
|
}
|
|
756
|
+
const BASELINE_MARK = {
|
|
757
|
+
matches: "✓",
|
|
758
|
+
changed: "✗",
|
|
759
|
+
"no-baseline": "○",
|
|
760
|
+
unusable: "✗",
|
|
761
|
+
"not-captured": "⊘",
|
|
762
|
+
updated: "↻",
|
|
763
|
+
};
|
|
764
|
+
/** One target's line in the report. */
|
|
765
|
+
function baselineLine(r, run) {
|
|
766
|
+
const what = `${BASELINE_MARK[r.status]} ${code(r.element)} on ${code(r.path)}`;
|
|
767
|
+
const notes = [r.platformNote, r.partial]
|
|
768
|
+
.filter(Boolean)
|
|
769
|
+
.map((n) => ` _(${cell(n)})_`)
|
|
770
|
+
.join("");
|
|
771
|
+
switch (r.status) {
|
|
772
|
+
case "matches":
|
|
773
|
+
return `- ${what}: ${run.mode === "update" ? `within the threshold of its baseline (${r.diff?.percent ?? 0}% changed), which was left as it was` : `matches (${r.diff?.percent ?? 0}% changed)`}${notes}`;
|
|
774
|
+
case "changed": {
|
|
775
|
+
const files = r.files ? ` — expected ${code(r.files.expected)}, now ${code(r.files.actual)}, diff ${code(r.files.diff)}` : "";
|
|
776
|
+
return `- ${what}: ${r.diff ? describeChange(r.diff, run.threshold) : "it no longer matches its baseline"}${files}${notes}`;
|
|
777
|
+
}
|
|
778
|
+
case "no-baseline":
|
|
779
|
+
return `- ${what}: no baseline yet${notes}`;
|
|
780
|
+
case "unusable":
|
|
781
|
+
return `- ${what}: its baseline cannot be used: ${cell(r.detail ?? "no reason given")}`;
|
|
782
|
+
case "not-captured":
|
|
783
|
+
return `- ${what}: could not be captured: ${cell(r.detail ?? "no reason given")}${run.mode === "update" ? "; its baseline was not written" : ""}`;
|
|
784
|
+
case "updated":
|
|
785
|
+
return `- ${what}: baseline written to ${code(r.baseline)} (${cell(r.detail ?? "updated")})${notes}`;
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
/** The report's section on visual baselines: every target and what became of it. */
|
|
789
|
+
function formatBaselines(run) {
|
|
790
|
+
const c = countStatuses(run.results);
|
|
791
|
+
const counts = [
|
|
792
|
+
["changed", "changed"],
|
|
793
|
+
["unusable", "unusable"],
|
|
794
|
+
["not-captured", "not captured"],
|
|
795
|
+
["no-baseline", "no baseline yet"],
|
|
796
|
+
["updated", "updated"],
|
|
797
|
+
["matches", run.mode === "update" ? "left as they were" : "match"],
|
|
798
|
+
]
|
|
799
|
+
.filter(([k]) => c[k] > 0)
|
|
800
|
+
.map(([k, label]) => `${c[k]} ${label}`)
|
|
801
|
+
.join(" · ");
|
|
802
|
+
const lines = [
|
|
803
|
+
`## Visual baselines (${run.results.length})`,
|
|
804
|
+
"",
|
|
805
|
+
`${run.mode === "update" ? "Updated" : "Compared"} in ${run.engine}, baselines in ${code(run.dir)} · ${run.threshold}% of a picture's pixels may change · ${counts}`,
|
|
806
|
+
"",
|
|
807
|
+
...run.results.map((r) => baselineLine(r, run)),
|
|
808
|
+
];
|
|
809
|
+
if (c.changed > 0)
|
|
810
|
+
lines.push("", "The pictures are beside this report, under `visual/` (in the job's artifact on CI): the baseline, the picture now, and the changed pixels in red.");
|
|
811
|
+
if (c["no-baseline"] > 0 || c.unusable > 0) {
|
|
812
|
+
lines.push("", "A target with no baseline yet is listed and never fails; one whose baseline cannot be used fails until it is taken again. Run the check with `--baseline update` to take them.");
|
|
813
|
+
}
|
|
814
|
+
if (run.results.some((r) => r.platformNote)) {
|
|
815
|
+
lines.push("", "_Some baselines were taken on another operating system. Each one draws text differently, so a change there may be the system's rather than the app's: take baselines where the check runs._");
|
|
816
|
+
}
|
|
817
|
+
return lines;
|
|
818
|
+
}
|
|
819
|
+
/** One markdown list item for an issue: its title, rule, evidence and routes. */
|
|
820
|
+
export function issueLine(i) {
|
|
821
|
+
return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
|
|
822
|
+
}
|
|
823
|
+
/** The issues, one section per severity, worst first. Shared by the check's report and the first run's. */
|
|
824
|
+
export function issueSections(issues) {
|
|
825
|
+
const lines = [];
|
|
826
|
+
for (const sev of CHECK_SEVERITIES) {
|
|
827
|
+
const of = issues.filter((i) => i.severity === sev);
|
|
828
|
+
if (of.length === 0)
|
|
829
|
+
continue;
|
|
830
|
+
lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
|
|
831
|
+
for (const i of of)
|
|
832
|
+
lines.push(`- ${issueLine(i)}`);
|
|
833
|
+
}
|
|
834
|
+
return lines;
|
|
835
|
+
}
|
|
836
|
+
const WORTH_A_LOOK_INTRO = "Measured exactly, and defects only under a convention of your project that the check cannot see.";
|
|
837
|
+
/** The worth-a-look section, or nothing when there is none. `gated`: the report is a gate's, so say these never fail it. */
|
|
838
|
+
export function worthALookSection(observations, gated = true) {
|
|
839
|
+
if (observations.length === 0)
|
|
840
|
+
return [];
|
|
841
|
+
const lines = [
|
|
842
|
+
"",
|
|
843
|
+
`## Worth a look (${observations.length})`,
|
|
844
|
+
"",
|
|
845
|
+
`${WORTH_A_LOOK_INTRO} They are not counted above${gated ? " and never fail the gate, at any --fail-on" : ""}.`,
|
|
846
|
+
"",
|
|
847
|
+
];
|
|
848
|
+
for (const o of observations) {
|
|
849
|
+
lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
|
|
850
|
+
}
|
|
851
|
+
return lines;
|
|
852
|
+
}
|
|
853
|
+
/** The table of routes measured, with each one's status, controls and issue count. */
|
|
854
|
+
export function routesTable(result) {
|
|
855
|
+
const lines = ["", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|"];
|
|
856
|
+
for (const r of result.routes) {
|
|
857
|
+
const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
|
|
858
|
+
const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
|
|
859
|
+
(r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
|
|
860
|
+
// Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
|
|
861
|
+
lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
|
|
862
|
+
}
|
|
863
|
+
return lines;
|
|
864
|
+
}
|
|
865
|
+
/** The routes known and not visited, as one line; nothing when every known route was visited. */
|
|
866
|
+
export function unvisitedLine(unvisited, why) {
|
|
867
|
+
if (unvisited.length === 0)
|
|
868
|
+
return [];
|
|
869
|
+
return ["", `Not visited (${why}): ${unvisited.slice(0, 20).map(code).join(", ")}${unvisited.length > 20 ? " …" : ""}`];
|
|
870
|
+
}
|
|
666
871
|
/** The human report: the verdict first, then what failed it, then everything else. */
|
|
667
872
|
export function formatCheck(result) {
|
|
668
873
|
const { passed, counts, failing, retestsFailing, couldNotRun } = summarise(result);
|
|
@@ -672,6 +877,8 @@ export function formatCheck(result) {
|
|
|
672
877
|
lines.push(`${result.url} · ${result.routes.length} route(s) · mode ${result.mode} · gate: ${result.failOn === "never" ? "report only" : `fail on ${result.failOn}${result.failOn === "low" ? "" : " or worse"}`}`);
|
|
673
878
|
lines.push("");
|
|
674
879
|
const unaudited = unauditedRoutes(result.routes).length;
|
|
880
|
+
// Nothing compared is not a match: a target with no baseline yet is named beside the verdict, as an unmeasured design is.
|
|
881
|
+
const uncompared = result.baselines?.mode === "compare" ? countStatuses(result.baselines.results)["no-baseline"] : 0;
|
|
675
882
|
const failingText = retestsFailing > 0
|
|
676
883
|
? `${failing} failing the gate (${failing - retestsFailing} issue(s), ${retestsFailing} re-tested finding(s))`
|
|
677
884
|
: `${failing} issue(s) at the gate's severity`;
|
|
@@ -680,35 +887,14 @@ export function formatCheck(result) {
|
|
|
680
887
|
? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
|
|
681
888
|
: `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
|
|
682
889
|
(unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
|
|
890
|
+
(uncompared > 0 ? ` · ${uncompared} visual target(s) not compared: no baseline yet` : "") +
|
|
683
891
|
(result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
|
|
684
892
|
// Right under the verdict: what a green check was allowed to do is part of what it means.
|
|
685
893
|
lines.push("", `Settings — ${describeSettings(result)}`);
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
|
|
691
|
-
for (const i of of) {
|
|
692
|
-
lines.push(`- **${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`);
|
|
693
|
-
}
|
|
694
|
-
}
|
|
695
|
-
if (result.worthALook.length > 0) {
|
|
696
|
-
lines.push("", `## Worth a look (${result.worthALook.length})`, "", "Measured exactly, and defects only under a convention of your project that the check cannot see. They are not counted above and never fail the gate, at any --fail-on.", "");
|
|
697
|
-
for (const o of result.worthALook) {
|
|
698
|
-
lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
|
|
699
|
-
}
|
|
700
|
-
}
|
|
701
|
-
lines.push("", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|");
|
|
702
|
-
for (const r of result.routes) {
|
|
703
|
-
const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
|
|
704
|
-
const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
|
|
705
|
-
(r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
|
|
706
|
-
// Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
|
|
707
|
-
lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
|
|
708
|
-
}
|
|
709
|
-
if (result.unvisited.length > 0) {
|
|
710
|
-
lines.push("", `Not visited (over --max-routes): ${result.unvisited.slice(0, 20).map(code).join(", ")}${result.unvisited.length > 20 ? " …" : ""}`);
|
|
711
|
-
}
|
|
894
|
+
lines.push(...issueSections(result.issues));
|
|
895
|
+
lines.push(...worthALookSection(result.worthALook));
|
|
896
|
+
lines.push(...routesTable(result));
|
|
897
|
+
lines.push(...unvisitedLine(result.unvisited, "over --max-routes"));
|
|
712
898
|
if (result.flows.length > 0 || result.skippedFlows.length > 0) {
|
|
713
899
|
lines.push("", `## Flows (${result.flows.length})`, "");
|
|
714
900
|
for (const f of result.flows) {
|
|
@@ -726,6 +912,8 @@ export function formatCheck(result) {
|
|
|
726
912
|
for (const s of result.skippedFlows)
|
|
727
913
|
lines.push(`- skipped ${code(s.file)}: ${cell(s.reason)}`);
|
|
728
914
|
}
|
|
915
|
+
if (result.baselines)
|
|
916
|
+
lines.push("", ...formatBaselines(result.baselines));
|
|
729
917
|
if (result.retest && result.retest.open > 0) {
|
|
730
918
|
const { open, results } = result.retest;
|
|
731
919
|
lines.push("", `## Open findings re-tested (${results.length} of ${open})`, "");
|
|
@@ -778,6 +966,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
778
966
|
...(r.auditError !== undefined ? { designNotMeasured: r.auditError } : {}),
|
|
779
967
|
})),
|
|
780
968
|
unvisited: result.unvisited,
|
|
969
|
+
...(result.timeBudget ? { timeBudget: result.timeBudget } : {}),
|
|
781
970
|
ignored: result.ignored,
|
|
782
971
|
flows: result.flows.map((f) => ({
|
|
783
972
|
name: f.name,
|
|
@@ -790,6 +979,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
790
979
|
})),
|
|
791
980
|
skippedFlows: result.skippedFlows,
|
|
792
981
|
retest: result.retest,
|
|
982
|
+
baselines: result.baselines ?? null,
|
|
793
983
|
issues: result.issues,
|
|
794
984
|
// Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
|
|
795
985
|
worthALook: result.worthALook,
|