scenescout 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +5 -1
- package/dist/browsers.js +52 -8
- package/dist/check-run.js +4 -2
- package/dist/engine/bench.js +147 -14
- package/dist/engine/browser.js +174 -7
- package/dist/engine/calibration.js +122 -20
- package/dist/engine/check.js +86 -14
- package/dist/engine/design.js +15 -1
- package/dist/engine/fingerprint.js +69 -7
- package/dist/engine/lane.js +138 -3
- package/dist/engine/memory.js +135 -10
- package/dist/engine/report.js +37 -2
- package/dist/engine/unload.js +201 -0
- package/dist/engine/verify.js +4 -3
- package/dist/mcp-server.js +23 -9
- package/package.json +1 -1
- package/skills/scenescout/SKILL.md +4 -3
package/dist/engine/check.js
CHANGED
|
@@ -81,7 +81,31 @@ export const CHECK_RULES = {
|
|
|
81
81
|
help: "A step of a flow saved in .scenescout/flows could not be done, or what it expected was not there. The evidence names the flow, the step and what happened instead.",
|
|
82
82
|
},
|
|
83
83
|
};
|
|
84
|
-
|
|
84
|
+
/**
|
|
85
|
+
* Rules whose measurement is exact and whose meaning depends on a convention
|
|
86
|
+
* of the project the check cannot see. SceneScout is used against any app, so
|
|
87
|
+
* it does not decide those conventions: these are listed as "worth a look",
|
|
88
|
+
* each naming the convention that would make it a defect, and never counted,
|
|
89
|
+
* given a severity or gated on, at any --fail-on. SARIF reports them at level
|
|
90
|
+
* "note". `convention` finishes the sentence "a defect only if your project
|
|
91
|
+
* uses …". --ignore takes them like any other rule.
|
|
92
|
+
*/
|
|
93
|
+
export const WORTH_A_LOOK_RULES = {
|
|
94
|
+
"off-grid-spacing": {
|
|
95
|
+
title: "Spacing off a 4px grid",
|
|
96
|
+
help: "More than a fifth of the page's paddings or vertical margins are not multiples of 4px. That matters where a project keeps a 4px spacing scale, and not where it uses another scale or none.",
|
|
97
|
+
convention: "a 4px spacing scale",
|
|
98
|
+
},
|
|
99
|
+
"indistinct-link": {
|
|
100
|
+
title: "Link styled like body text",
|
|
101
|
+
help: "Links with no underline, in the same colour as the page's body text. In running text a reader cannot tell them from the text around them; in navigation this styling is common, and the check cannot tell the two apart.",
|
|
102
|
+
convention: "a visible link style (an underline or a distinct colour) wherever links appear, navigation included",
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
export const CHECK_RULE_IDS = [...Object.keys(CHECK_RULES), ...Object.keys(WORTH_A_LOOK_RULES)];
|
|
106
|
+
export function isWorthALookRule(rule) {
|
|
107
|
+
return Object.prototype.hasOwnProperty.call(WORTH_A_LOOK_RULES, rule);
|
|
108
|
+
}
|
|
85
109
|
const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
|
|
86
110
|
/** Snapshot refs (`e12`) are numbered per run; evidence carrying them would never match itself twice. */
|
|
87
111
|
function stripRefs(line) {
|
|
@@ -145,27 +169,36 @@ export function geometryRule(line) {
|
|
|
145
169
|
* oracles caught while it ran. The second kind goes through the same rules as
|
|
146
170
|
* a crawled page's, so a request that fails on load and again inside a flow is
|
|
147
171
|
* one issue, not two.
|
|
172
|
+
*
|
|
173
|
+
* A fact under a worth-a-look rule goes to `worthALook` instead, deduplicated
|
|
174
|
+
* the same way; the two lists never share an entry.
|
|
148
175
|
*/
|
|
149
|
-
export function
|
|
176
|
+
export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
150
177
|
const byKey = new Map();
|
|
178
|
+
const looks = new Map();
|
|
151
179
|
const add = (rule, evidence, route, opts = {}) => {
|
|
152
180
|
if (ignore.includes(rule))
|
|
153
181
|
return;
|
|
154
182
|
const clean = redactSecrets(withoutOrigin(evidence, origin)).slice(0, 300);
|
|
155
183
|
const key = `${rule}\u0000${clean}`;
|
|
156
|
-
const found = byKey.get(key);
|
|
184
|
+
const found = byKey.get(key) ?? looks.get(key);
|
|
157
185
|
if (found) {
|
|
158
186
|
if (!found.routes.includes(route))
|
|
159
187
|
found.routes.push(route);
|
|
160
188
|
return;
|
|
161
189
|
}
|
|
190
|
+
const fingerprint = createHash("sha256").update(key).digest("hex").slice(0, 32);
|
|
191
|
+
if (isWorthALookRule(rule)) {
|
|
192
|
+
looks.set(key, { rule, evidence: clean, routes: [route], convention: WORTH_A_LOOK_RULES[rule].convention, fingerprint });
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
162
195
|
byKey.set(key, {
|
|
163
196
|
rule,
|
|
164
197
|
severity: opts.embed ? "medium" : (opts.severity ?? CHECK_RULES[rule].severity),
|
|
165
198
|
evidence: clean,
|
|
166
199
|
routes: [route],
|
|
167
200
|
...(opts.embed ? { embed: opts.embed } : {}),
|
|
168
|
-
fingerprint
|
|
201
|
+
fingerprint,
|
|
169
202
|
});
|
|
170
203
|
};
|
|
171
204
|
for (const r of routes) {
|
|
@@ -206,7 +239,14 @@ export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
|
|
|
206
239
|
if (f.outcome.status === "failed")
|
|
207
240
|
add("flow-step-failed", flowStepEvidence(f), f.outcome.path);
|
|
208
241
|
}
|
|
209
|
-
return
|
|
242
|
+
return {
|
|
243
|
+
issues: [...byKey.values()].sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.rule.localeCompare(b.rule)),
|
|
244
|
+
worthALook: [...looks.values()].sort((a, b) => a.rule.localeCompare(b.rule)),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
/** The defect tier of `checkFindings`: what counts, and what the gate reads. */
|
|
248
|
+
export function issuesFromRoutes(routes, origin, ignore = [], flows = []) {
|
|
249
|
+
return checkFindings(routes, origin, ignore, flows).issues;
|
|
210
250
|
}
|
|
211
251
|
/**
|
|
212
252
|
* Why a check has nothing to give a verdict on, or null when it measured the
|
|
@@ -383,7 +423,7 @@ export function parseCheckArgs(args, cwd) {
|
|
|
383
423
|
if (paths?.some((p) => !p.startsWith("/")))
|
|
384
424
|
return { ok: false, error: "--paths are paths on the app, each starting with /" };
|
|
385
425
|
const ignore = list(flags.get("ignore")) ?? [];
|
|
386
|
-
const unknownRules = ignore.filter((r) => !(r
|
|
426
|
+
const unknownRules = ignore.filter((r) => !CHECK_RULE_IDS.includes(r));
|
|
387
427
|
if (unknownRules.length > 0)
|
|
388
428
|
return { ok: false, error: `unknown rule(s) in --ignore: ${unknownRules.join(", ")}. Rules: ${CHECK_RULE_IDS.join(", ")}` };
|
|
389
429
|
const retest = flags.get("retest") ?? "on";
|
|
@@ -489,8 +529,17 @@ const RETEST_SARIF_RULE = {
|
|
|
489
529
|
help: { text: "A finding an earlier run filed and left open failed the same way when its page loaded again. It gates according to --gate-retests." },
|
|
490
530
|
defaultConfiguration: { level: "error" },
|
|
491
531
|
};
|
|
532
|
+
function sarifLocation(route) {
|
|
533
|
+
return route === "(shared chrome)"
|
|
534
|
+
? {
|
|
535
|
+
physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
|
|
536
|
+
message: { text: "the app's shared shell, on every page that renders it" },
|
|
537
|
+
}
|
|
538
|
+
: { physicalLocation: { artifactLocation: { uri: route.replace(/^\//, ""), uriBaseId: "APP" } } };
|
|
539
|
+
}
|
|
492
540
|
export function toSarif(result, toolVersion) {
|
|
493
541
|
const used = [...new Set(result.issues.map((i) => i.rule))];
|
|
542
|
+
const lookRules = [...new Set(result.worthALook.map((o) => o.rule))];
|
|
494
543
|
const base = new URL(result.url);
|
|
495
544
|
const reproducing = retestGateFailures(result);
|
|
496
545
|
const refused = result.flows.filter((f) => f.outcome.status === "refused");
|
|
@@ -500,14 +549,20 @@ export function toSarif(result, toolVersion) {
|
|
|
500
549
|
message: {
|
|
501
550
|
text: `${CHECK_RULES[i.rule].title}: ${i.evidence}${i.routes.length > 1 ? ` (on ${i.routes.length} routes)` : ""}${i.embed ? ` (in an embed of ${i.embed})` : ""}`,
|
|
502
551
|
},
|
|
503
|
-
locations: i.routes.slice(0, 10).map(
|
|
504
|
-
? {
|
|
505
|
-
physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
|
|
506
|
-
message: { text: "the app's shared shell, on every page that renders it" },
|
|
507
|
-
}
|
|
508
|
-
: { physicalLocation: { artifactLocation: { uri: r.replace(/^\//, ""), uriBaseId: "APP" } } }),
|
|
552
|
+
locations: i.routes.slice(0, 10).map(sarifLocation),
|
|
509
553
|
partialFingerprints: { "scenescoutCheck/v1": i.fingerprint },
|
|
510
554
|
}));
|
|
555
|
+
// Always "note", whatever --fail-on says: a result a code-scanning dashboard shows, never one that reads as an error.
|
|
556
|
+
const lookResults = result.worthALook.map((o) => ({
|
|
557
|
+
ruleId: o.rule,
|
|
558
|
+
level: "note",
|
|
559
|
+
message: {
|
|
560
|
+
text: `Worth a look — ${WORTH_A_LOOK_RULES[o.rule].title}: ${o.evidence}${o.routes.length > 1 ? ` (on ${o.routes.length} routes)` : ""}. A defect only if your project uses ${o.convention}.`,
|
|
561
|
+
},
|
|
562
|
+
locations: o.routes.slice(0, 10).map(sarifLocation),
|
|
563
|
+
partialFingerprints: { "scenescoutCheck/v1": o.fingerprint },
|
|
564
|
+
properties: { tier: "worth-a-look", convention: o.convention },
|
|
565
|
+
}));
|
|
511
566
|
// Only the re-tests that fail the gate are results: a reviewer reading code scanning sees what failed it.
|
|
512
567
|
const retestResults = reproducing.map((r) => ({
|
|
513
568
|
ruleId: RETEST_SARIF_RULE.id,
|
|
@@ -537,6 +592,14 @@ export function toSarif(result, toolVersion) {
|
|
|
537
592
|
help: { text: CHECK_RULES[id].help },
|
|
538
593
|
defaultConfiguration: { level: SARIF_LEVEL[CHECK_RULES[id].severity] },
|
|
539
594
|
})),
|
|
595
|
+
...lookRules.map((id) => ({
|
|
596
|
+
id,
|
|
597
|
+
name: WORTH_A_LOOK_RULES[id].title,
|
|
598
|
+
shortDescription: { text: WORTH_A_LOOK_RULES[id].title },
|
|
599
|
+
help: { text: `${WORTH_A_LOOK_RULES[id].help} Worth a look: a defect only if your project uses ${WORTH_A_LOOK_RULES[id].convention}.` },
|
|
600
|
+
defaultConfiguration: { level: "note" },
|
|
601
|
+
properties: { tags: ["worth-a-look"] },
|
|
602
|
+
})),
|
|
540
603
|
...(reproducing.length > 0 ? [RETEST_SARIF_RULE] : []),
|
|
541
604
|
],
|
|
542
605
|
},
|
|
@@ -553,7 +616,7 @@ export function toSarif(result, toolVersion) {
|
|
|
553
616
|
},
|
|
554
617
|
],
|
|
555
618
|
originalUriBaseIds: { APP: { uri: `${base.origin}/` } },
|
|
556
|
-
results: [...issueResults, ...retestResults],
|
|
619
|
+
results: [...issueResults, ...lookResults, ...retestResults],
|
|
557
620
|
},
|
|
558
621
|
],
|
|
559
622
|
};
|
|
@@ -596,7 +659,8 @@ export function formatCheck(result) {
|
|
|
596
659
|
(passed
|
|
597
660
|
? `${couldNotRun > 0 ? "passed" : "**PASSED**"} — ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
|
|
598
661
|
: `${couldNotRun > 0 ? "failed" : "**FAILED**"} — ${failingText} · ${counts.high} high · ${counts.medium} medium · ${counts.low} low`) +
|
|
599
|
-
(unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "")
|
|
662
|
+
(unaudited > 0 ? ` · design not measured on ${unaudited} route(s)` : "") +
|
|
663
|
+
(result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
|
|
600
664
|
// Right under the verdict: what a green check was allowed to do is part of what it means.
|
|
601
665
|
lines.push("", `Settings — ${describeSettings(result)}`);
|
|
602
666
|
for (const sev of CHECK_SEVERITIES) {
|
|
@@ -608,6 +672,12 @@ export function formatCheck(result) {
|
|
|
608
672
|
lines.push(`- **${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`);
|
|
609
673
|
}
|
|
610
674
|
}
|
|
675
|
+
if (result.worthALook.length > 0) {
|
|
676
|
+
lines.push("", `## Worth a look (${result.worthALook.length})`, "", "Measured exactly, and defects only under a convention of your project that the check cannot see. They are not counted above and never fail the gate, at any --fail-on.", "");
|
|
677
|
+
for (const o of result.worthALook) {
|
|
678
|
+
lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
611
681
|
lines.push("", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|");
|
|
612
682
|
for (const r of result.routes) {
|
|
613
683
|
const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
|
|
@@ -701,5 +771,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
701
771
|
skippedFlows: result.skippedFlows,
|
|
702
772
|
retest: result.retest,
|
|
703
773
|
issues: result.issues,
|
|
774
|
+
// Apart from `issues` and `counts`, which the gate reads: none of these is counted or gated.
|
|
775
|
+
worthALook: result.worthALook,
|
|
704
776
|
};
|
|
705
777
|
}
|
package/dist/engine/design.js
CHANGED
|
@@ -274,6 +274,14 @@ function contrastFailures(records) {
|
|
|
274
274
|
}
|
|
275
275
|
return out;
|
|
276
276
|
}
|
|
277
|
+
/** The commonest off-grid values, smallest first and without their counts: "7px, 13px". */
|
|
278
|
+
function gridValues(top) {
|
|
279
|
+
return top
|
|
280
|
+
.map(([v]) => v)
|
|
281
|
+
.sort((a, b) => a - b)
|
|
282
|
+
.map((v) => `${v}px`)
|
|
283
|
+
.join(", ");
|
|
284
|
+
}
|
|
277
285
|
/** Below the WCAG 2.2 target-size minimum; inline links are exempt by that rule. */
|
|
278
286
|
function tooSmall(r) {
|
|
279
287
|
return r.tag !== "a" && (r.rect.h < 24 || r.rect.w < 24) && r.rect.h > 0;
|
|
@@ -559,8 +567,8 @@ export function analyzeDesign(payload, viewport, chromeKeys = new Set()) {
|
|
|
559
567
|
if (r.textLen > 40 && r.tag !== "a" && r.color !== "unknown")
|
|
560
568
|
bodyColorFreq.set(r.color, (bodyColorFreq.get(r.color) ?? 0) + 1);
|
|
561
569
|
const dominantBody = [...bodyColorFreq.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
570
|
+
const indistinct = dominantBody ? records.filter((r) => r.tag === "a" && r.textLen > 0 && !r.underline && r.color === dominantBody) : [];
|
|
562
571
|
if (dominantBody) {
|
|
563
|
-
const indistinct = records.filter((r) => r.tag === "a" && r.textLen > 0 && !r.underline && r.color === dominantBody);
|
|
564
572
|
if (indistinct.length > 0) {
|
|
565
573
|
affordances.push(`→ ${indistinct.length} link(s) with no underline AND the same color as body text (e.g. ${label(indistinct[0])}) — invisible as links`);
|
|
566
574
|
}
|
|
@@ -726,6 +734,12 @@ export function analyzeDesign(payload, viewport, chromeKeys = new Set()) {
|
|
|
726
734
|
...(page.scrollW > viewport.width + 8
|
|
727
735
|
? [{ rule: "horizontal-scroll", detail: `content ${page.scrollW}px wide in a ${viewport.width}px viewport` }]
|
|
728
736
|
: []),
|
|
737
|
+
// The same thresholds as the SPACING and AFFORDANCES lines above, so the check and the audit agree on what they saw.
|
|
738
|
+
// Worded without counts or percentages: a spacing scale and a link style are app-wide conventions, so the same
|
|
739
|
+
// values on ten pages are one entry on ten routes, and the entry's fingerprint does not move when content does.
|
|
740
|
+
...(pad.n > 10 && pad.pct > 20 ? [{ rule: "off-grid-spacing", detail: `paddings off a 4px grid: ${gridValues(pad.top)}` }] : []),
|
|
741
|
+
...(mar.n > 10 && mar.pct > 20 ? [{ rule: "off-grid-spacing", detail: `vertical margins off a 4px grid: ${gridValues(mar.top)}` }] : []),
|
|
742
|
+
...(indistinct.length > 0 ? [{ rule: "indistinct-link", detail: `links with no underline in the body-text colour ${dominantBody}` }] : []),
|
|
729
743
|
...chromeDefects,
|
|
730
744
|
];
|
|
731
745
|
return { report, score, signatures, defects };
|
|
@@ -4,6 +4,17 @@ const HEX_RE = /^[0-9a-f]{12,}$/i;
|
|
|
4
4
|
const NUM_RE = /^\d+$/;
|
|
5
5
|
/** Query params that name a UI state (a different screen), not transient data. */
|
|
6
6
|
const UI_STATE_PARAM_RE = /^(tab|view|mode|step|panel|section)$/i;
|
|
7
|
+
/**
|
|
8
|
+
* Whether a path segment is an id that route identity collapses to `:id`: a
|
|
9
|
+
* UUID or long hex string anywhere, a number from the second real segment on.
|
|
10
|
+
* `index` counts as `path.split("/")` does, so the first real segment of a
|
|
11
|
+
* leading-slash path is 1 (see the note on numeric first segments below).
|
|
12
|
+
*/
|
|
13
|
+
export function isIdSegment(seg, index) {
|
|
14
|
+
if (UUID_RE.test(seg) || HEX_RE.test(seg))
|
|
15
|
+
return true;
|
|
16
|
+
return NUM_RE.test(seg) && index > 1;
|
|
17
|
+
}
|
|
7
18
|
/**
|
|
8
19
|
* Normalize a URL into a route-class identity:
|
|
9
20
|
* - path ids collapse (/orders/123 → /orders/:id)
|
|
@@ -38,13 +49,7 @@ export function normalizePath(rawUrl) {
|
|
|
38
49
|
}
|
|
39
50
|
// segments[0] is always "" for a leading-slash path, so the first REAL
|
|
40
51
|
// segment is index 1 — that is the one a bare numeric page occupies.
|
|
41
|
-
const segments = path.split("/").map((seg, i) =>
|
|
42
|
-
if (UUID_RE.test(seg) || HEX_RE.test(seg))
|
|
43
|
-
return ":id";
|
|
44
|
-
if (NUM_RE.test(seg) && i > 1)
|
|
45
|
-
return ":id";
|
|
46
|
-
return seg;
|
|
47
|
-
});
|
|
52
|
+
const segments = path.split("/").map((seg, i) => (isIdSegment(seg, i) ? ":id" : seg));
|
|
48
53
|
let normalized = segments.join("/") || "/";
|
|
49
54
|
if (normalized === "")
|
|
50
55
|
normalized = "/";
|
|
@@ -98,3 +103,60 @@ export function fingerprintState(url, elements) {
|
|
|
98
103
|
export function shortHash(input) {
|
|
99
104
|
return createHash("sha1").update(input).digest("hex").slice(0, 10);
|
|
100
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* A route as a lane wrote it, with every query string and every fragment that
|
|
108
|
+
* is not a hash route removed. A lane copies routes out of the address bar, and
|
|
109
|
+
* an address can carry `?access_token=`, `?code=`, `?session_id=` or a signed
|
|
110
|
+
* URL's `X-Amz-Signature=`; none of it identifies the page, and none of it may
|
|
111
|
+
* be stored or archived. A hash route (`#/things`) keeps its path, not its query.
|
|
112
|
+
* A `name=value` pair whose name reads as a credential is removed wherever it is.
|
|
113
|
+
*/
|
|
114
|
+
export function stripRouteQuery(route) {
|
|
115
|
+
return route
|
|
116
|
+
.replace(/\?[^\s()[\]]*/g, "")
|
|
117
|
+
.replace(/#(?!\/)[^\s()[\]]*/g, "")
|
|
118
|
+
.replace(/[\w-]*(token|signature|secret|password|session|code|key|auth|credential)[\w-]*=[^\s()[\]]*/gi, "")
|
|
119
|
+
.trim();
|
|
120
|
+
}
|
|
121
|
+
/** Page files a lane may name without a leading slash ("orders.html"). */
|
|
122
|
+
const BARE_PAGE_RE = /^[\w-]+(\/[\w.-]+)*\.(html?|php|aspx?|jsp)$/i;
|
|
123
|
+
/** A host with a port, or a loopback or IPv4 host, written without a scheme ("127.0.0.1:4173/orders.html"). */
|
|
124
|
+
const BARE_HOST_RE = /^(?:[\w.-]+:\d+|localhost|\d{1,3}(?:\.\d{1,3}){3})(\/.*)?$/i;
|
|
125
|
+
/**
|
|
126
|
+
* Every path one route from a lane report names, in the form a benchmark key
|
|
127
|
+
* entry's `route` is written in; empty when it names none.
|
|
128
|
+
*
|
|
129
|
+
* A lane writes its routes as free text — "/order.html?id=1042 (from
|
|
130
|
+
* /orders.html link)", "Orders (/orders.html)", "orders.html",
|
|
131
|
+
* "127.0.0.1:4173/orders.html", "/reports.html and /reports-scheduled.html".
|
|
132
|
+
* EVERY path in it counts, including one in a note: the benchmark uses these
|
|
133
|
+
* to set aside a verdict as another lane's, so a page left out makes a wrong
|
|
134
|
+
* verdict disappear, while a page taken in only keeps a verdict scored. A bare
|
|
135
|
+
* origin is "/", a bare page file gains its slash, the query goes, ids
|
|
136
|
+
* collapse as the engine's route identity collapses them, and "/index.html" is
|
|
137
|
+
* "/".
|
|
138
|
+
*/
|
|
139
|
+
export function laneRoutePaths(raw) {
|
|
140
|
+
const out = new Set();
|
|
141
|
+
for (const word of raw.split(/[\s,;|+()[\]]+|→|->|=>/)) {
|
|
142
|
+
const token = word.replace(/^[<"'`]+|[>"'`.:!]+$/g, "");
|
|
143
|
+
if (!token)
|
|
144
|
+
continue;
|
|
145
|
+
let path;
|
|
146
|
+
const url = /^https?:\/\/[^/\s]+(\/.*)?$/i.exec(token) ?? BARE_HOST_RE.exec(token);
|
|
147
|
+
if (url)
|
|
148
|
+
path = url[1] || "/";
|
|
149
|
+
else if (token.startsWith("/") || token.startsWith("#/"))
|
|
150
|
+
path = token;
|
|
151
|
+
else if (BARE_PAGE_RE.test(token.split(/[?#]/)[0]))
|
|
152
|
+
path = `/${token}`;
|
|
153
|
+
if (path === undefined)
|
|
154
|
+
continue;
|
|
155
|
+
let p = normalizePath(path).split("?")[0];
|
|
156
|
+
p = p.replace(/\/index\.html?$/i, "/");
|
|
157
|
+
if (p.length > 1)
|
|
158
|
+
p = p.replace(/\/+$/, "");
|
|
159
|
+
out.add(p || "/");
|
|
160
|
+
}
|
|
161
|
+
return [...out];
|
|
162
|
+
}
|
package/dist/engine/lane.js
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
* Pure logic, no browser, so every rule here is table-tested.
|
|
24
24
|
*/
|
|
25
25
|
import { z } from "zod";
|
|
26
|
+
import { isIdSegment } from "./fingerprint.js";
|
|
26
27
|
import { FINDING_CATEGORIES } from "./memory.js";
|
|
27
28
|
/** Upper bound on the decisions and routes arrays, so a report is always small enough to hold whole. */
|
|
28
29
|
export const LANE_MAX_ITEMS = 255;
|
|
@@ -32,6 +33,8 @@ export const LANE_EVIDENCE_MAX = 160;
|
|
|
32
33
|
export const LANE_OBSERVATION_MAX = 64;
|
|
33
34
|
/** One line saying what blocked the lane; the detail belongs in a finding. */
|
|
34
35
|
export const LANE_BLOCKED_BY_MAX = 200;
|
|
36
|
+
/** One line naming the convention that would decide a "worth_a_look": "a 4px spacing scale", not an essay. */
|
|
37
|
+
export const LANE_CONVENTION_MAX = 160;
|
|
35
38
|
/** The planner chooses the lane name, so this cap is on the planner's side; it is stated all the same. */
|
|
36
39
|
export const LANE_NAME_MAX = 40;
|
|
37
40
|
/** A route as the lane saw it, query string included. */
|
|
@@ -39,7 +42,16 @@ export const LANE_ROUTE_MAX = 200;
|
|
|
39
42
|
export const LANE_SEVERITIES = ["high", "medium", "low"];
|
|
40
43
|
/** The same set scout_finding accepts, so a lane can report every finding it filed. */
|
|
41
44
|
export const LANE_CATEGORIES = FINDING_CATEGORIES;
|
|
42
|
-
|
|
45
|
+
/**
|
|
46
|
+
* "worth_a_look" is the third answer beside defect and not a defect: the
|
|
47
|
+
* observation is real, and whether it is a defect depends on a convention of
|
|
48
|
+
* the project that the run cannot see (a spacing scale, how navigation links
|
|
49
|
+
* are styled, whether controls carry test ids, whether the app ships to touch
|
|
50
|
+
* screens). SceneScout is used against any app, so it does not decide those;
|
|
51
|
+
* it names the convention and leaves the call to the project. It is not "I
|
|
52
|
+
* could not tell", which stays "unsure".
|
|
53
|
+
*/
|
|
54
|
+
export const LANE_VERDICTS = ["defect", "not_a_defect", "unsure", "worth_a_look"];
|
|
43
55
|
export const LANE_STATUSES = ["complete", "partial", "blocked"];
|
|
44
56
|
const Confidence = z.number().min(0).max(1);
|
|
45
57
|
/**
|
|
@@ -57,12 +69,33 @@ export const LaneDecision = z
|
|
|
57
69
|
confidence: Confidence,
|
|
58
70
|
/** The same machine signature scout_finding takes as its cross-run dedup key. */
|
|
59
71
|
evidence: z.string().min(1).max(LANE_EVIDENCE_MAX).nullable(),
|
|
72
|
+
/**
|
|
73
|
+
* The project convention that would make a "worth_a_look" a defect.
|
|
74
|
+
* Optional so a reply written before the verdict existed still parses;
|
|
75
|
+
* required (non-null) on a worth_a_look. On any other verdict the parser
|
|
76
|
+
* drops it and the fold says so: refusing a whole report over a field
|
|
77
|
+
* nothing reads would cost a round trip for nothing.
|
|
78
|
+
*/
|
|
79
|
+
// Any string here: its length is checked only on a worth_a_look (below), because on every other verdict it is ignored.
|
|
80
|
+
convention: z.string().nullable().optional(),
|
|
60
81
|
})
|
|
61
82
|
.strict()
|
|
62
83
|
.superRefine((d, ctx) => {
|
|
63
84
|
if (d.verdict === "defect" && (d.severity === null || d.category === null)) {
|
|
64
85
|
ctx.addIssue({ code: z.ZodIssueCode.custom, message: "a defect needs a severity and a category" });
|
|
65
86
|
}
|
|
87
|
+
if (d.verdict === "worth_a_look") {
|
|
88
|
+
if (d.convention === undefined || d.convention === null || d.convention.trim() === "") {
|
|
89
|
+
ctx.addIssue({ code: z.ZodIssueCode.custom, message: "a worth_a_look must name the convention that would decide it", path: ["convention"] });
|
|
90
|
+
}
|
|
91
|
+
else if (d.convention.length > LANE_CONVENTION_MAX) {
|
|
92
|
+
ctx.addIssue({
|
|
93
|
+
code: z.ZodIssueCode.custom,
|
|
94
|
+
message: `a convention is at most ${LANE_CONVENTION_MAX} characters`,
|
|
95
|
+
path: ["convention"],
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
66
99
|
});
|
|
67
100
|
export const LaneReport = z
|
|
68
101
|
.object({
|
|
@@ -149,7 +182,26 @@ export function parseLaneReport(text, expectedLane) {
|
|
|
149
182
|
if (expectedLane !== undefined && result.data.lane !== expectedLane) {
|
|
150
183
|
return { ok: false, reason: `the report names lane "${result.data.lane}", but this reply was asked of lane "${expectedLane}"` };
|
|
151
184
|
}
|
|
152
|
-
|
|
185
|
+
// A convention means something only on a worth_a_look; elsewhere it is dropped, so nothing stored carries it, and named in the fold.
|
|
186
|
+
const conventionsIgnored = [];
|
|
187
|
+
const decisions = result.data.decisions.map((d) => {
|
|
188
|
+
if (d.verdict === "worth_a_look" || d.convention === undefined || d.convention === null)
|
|
189
|
+
return d;
|
|
190
|
+
conventionsIgnored.push(d.verdict);
|
|
191
|
+
const { convention: _dropped, ...rest } = d;
|
|
192
|
+
return rest;
|
|
193
|
+
});
|
|
194
|
+
return { ok: true, report: { ...result.data, decisions }, aroundIgnored, entitiesDecoded: decoded.count, conventionsIgnored };
|
|
195
|
+
}
|
|
196
|
+
/** The line the fold adds for each verdict whose convention was dropped: "convention ignored on a defect verdict"; empty when none was. */
|
|
197
|
+
export function ignoredConventionsNote(verdicts) {
|
|
198
|
+
if (verdicts.length === 0)
|
|
199
|
+
return "";
|
|
200
|
+
const counts = new Map();
|
|
201
|
+
for (const v of verdicts)
|
|
202
|
+
counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
203
|
+
const parts = [...counts].map(([v, n]) => `convention ignored on ${n === 1 ? `${/^[aeiou]/.test(v) ? "an" : "a"} ${v} verdict` : `${n} ${v} verdicts`}`);
|
|
204
|
+
return `\n(${parts.join("; ")}: only a worth_a_look names a convention.)`;
|
|
153
205
|
}
|
|
154
206
|
/**
|
|
155
207
|
* The HTML character references a relay adds when it escapes text: the five
|
|
@@ -245,8 +297,9 @@ export function laneReportInstruction(lane) {
|
|
|
245
297
|
const LANE_RUBRIC = [
|
|
246
298
|
"Reply with ONE JSON object and nothing else — no prose before or after it, no explanation, no headings. A fenced ```json block is fine; anything outside it is discarded unread, so put nothing there you want kept.",
|
|
247
299
|
`Shape: {"lane":<your lane name>,"status":<${quoteAll(LANE_STATUSES)}>,"decisions":[…],"routes":[…],"blocked_by":<string or null>}.`,
|
|
248
|
-
`Each decision: {"observation":<a short id for what was observed, unique in the report, at most ${LANE_OBSERVATION_MAX} characters>,"verdict":<${quoteAll(LANE_VERDICTS)}>,"severity":<${quoteAll(LANE_SEVERITIES)} or null>,"category":<${quoteAll(LANE_CATEGORIES)} or null>,"confidence":<0..1>,"evidence":<machine signature such as "GET /api/things 500", or null>}.`,
|
|
300
|
+
`Each decision: {"observation":<a short id for what was observed, unique in the report, at most ${LANE_OBSERVATION_MAX} characters>,"verdict":<${quoteAll(LANE_VERDICTS)}>,"severity":<${quoteAll(LANE_SEVERITIES)} or null>,"category":<${quoteAll(LANE_CATEGORIES)} or null>,"confidence":<0..1>,"evidence":<machine signature such as "GET /api/things 500", or null>,"convention":<string or null>}.`,
|
|
249
301
|
`A "defect" must carry a severity and a category. "evidence" is a signature, not a sentence: at most ${LANE_EVIDENCE_MAX} characters. "confidence" is how sure you are of the verdict, calibrated: 0.5 means a coin flip, 0.95 means you would bet on it.`,
|
|
302
|
+
`"worth_a_look" is for an observation that is real but is a defect only under a convention of the project you cannot see (a spacing scale, how navigation links are styled, whether controls carry test ids, whether the app targets touch screens): it must name that convention in "convention", one line of at most ${LANE_CONVENTION_MAX} characters, such as "a 4px spacing scale". On every other verdict leave "convention" null or out: it is ignored there. It is not for "I could not tell": that is "unsure".`,
|
|
250
303
|
`"routes" lists the routes you covered, each at most ${LANE_ROUTE_MAX} characters. "blocked_by" is one line of at most ${LANE_BLOCKED_BY_MAX} characters saying what stopped you: required when the status is "blocked", allowed with "partial", null with "complete"; the detail belongs in a finding. The lane name is at most ${LANE_NAME_MAX} characters. At most ${LANE_MAX_ITEMS} decisions and ${LANE_MAX_ITEMS} routes. Unknown keys are refused.`,
|
|
251
304
|
"The object IS your final report: whatever hands it back must hand back the object verbatim, not a summary of it.",
|
|
252
305
|
];
|
|
@@ -258,12 +311,14 @@ export function summarizeLaneReport(r) {
|
|
|
258
311
|
const defects = r.decisions.filter((d) => d.verdict === "defect");
|
|
259
312
|
const high = defects.filter((d) => d.severity === "high").length;
|
|
260
313
|
const unsure = r.decisions.filter((d) => d.verdict === "unsure").length;
|
|
314
|
+
const worthALook = r.decisions.filter((d) => d.verdict === "worth_a_look").length;
|
|
261
315
|
const mean = r.decisions.length ? r.decisions.reduce((s, d) => s + d.confidence, 0) / r.decisions.length : 0;
|
|
262
316
|
const parts = [
|
|
263
317
|
r.status,
|
|
264
318
|
`${r.decisions.length} judged`,
|
|
265
319
|
`${defects.length} defects (${high} high)`,
|
|
266
320
|
`${unsure} unsure`,
|
|
321
|
+
...(worthALook > 0 ? [`${worthALook} worth a look`] : []),
|
|
267
322
|
`mean confidence ${mean.toFixed(2)}`,
|
|
268
323
|
`${r.routes.length} routes`,
|
|
269
324
|
];
|
|
@@ -271,6 +326,86 @@ export function summarizeLaneReport(r) {
|
|
|
271
326
|
parts.push(`blocked by ${r.blocked_by}`);
|
|
272
327
|
return `${r.lane}: ${parts.join(", ")}`;
|
|
273
328
|
}
|
|
329
|
+
/**
|
|
330
|
+
* Method, optional origin, path (template characters included), optional query
|
|
331
|
+
* and fragment, then a status only when it comes next: after an arrow or a
|
|
332
|
+
* colon, or after spaces, and nothing else. A status further along
|
|
333
|
+
* ("... (expected 403)") is a different claim about the request and is not
|
|
334
|
+
* read as its status.
|
|
335
|
+
*
|
|
336
|
+
* The separator is two alternatives rather than `\s*(?:->|:)?\s*`: with the
|
|
337
|
+
* arrow optional, two adjacent `\s*` can split a run of spaces every way
|
|
338
|
+
* there is, and evidence ending in a long run of whitespace took quadratic
|
|
339
|
+
* time to reject.
|
|
340
|
+
*/
|
|
341
|
+
const STATED_REQUEST_RE = /\b(GET|POST|PUT|PATCH|DELETE)\s+(?:https?:\/\/[^/\s]+)?(\/[A-Za-z0-9/_.:{}$*~%-]*)(?:\?[^\s#]*)?(?:#\S*)?(?:(?:\s*(?:->|→|=>|:)\s*|\s+)([1-5]\d{2})\b)?/gi;
|
|
342
|
+
/** The requests a piece of evidence names, in order. */
|
|
343
|
+
export function statedRequests(text) {
|
|
344
|
+
const out = [];
|
|
345
|
+
for (const m of text.matchAll(STATED_REQUEST_RE)) {
|
|
346
|
+
const raw = m[2].replace(/[.:,]+$/, "");
|
|
347
|
+
const start = (m.index ?? 0) + m[0].indexOf(m[2]);
|
|
348
|
+
out.push({ method: m[1].toUpperCase(), segments: pathSegments(raw), status: m[3] ?? null, start, end: start + raw.length });
|
|
349
|
+
}
|
|
350
|
+
return out;
|
|
351
|
+
}
|
|
352
|
+
function pathSegments(path) {
|
|
353
|
+
const trimmed = path.replace(/\/+$/, "");
|
|
354
|
+
if (trimmed === "")
|
|
355
|
+
return [];
|
|
356
|
+
return trimmed
|
|
357
|
+
.slice(1)
|
|
358
|
+
.split("/")
|
|
359
|
+
.map((seg) => {
|
|
360
|
+
try {
|
|
361
|
+
return decodeURIComponent(seg).toLowerCase();
|
|
362
|
+
}
|
|
363
|
+
catch {
|
|
364
|
+
return seg.toLowerCase();
|
|
365
|
+
}
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* A path-template segment: `{name}`, `:name` or `*`. Whole segments only —
|
|
370
|
+
* `order-{id}` is a literal, and so is `**`, which some routers read as "any
|
|
371
|
+
* number of segments": a template here stands for exactly one.
|
|
372
|
+
*/
|
|
373
|
+
export function isTemplateSegment(seg) {
|
|
374
|
+
return seg === "*" || /^\{[^/{}]+\}$/.test(seg) || /^:[a-z_][a-z0-9_-]*$/i.test(seg);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Whether two paths name the same route. A template segment on either side
|
|
378
|
+
* stands for ONE segment that is an id by route identity's own rule
|
|
379
|
+
* (`isIdSegment`: a number after the first segment, a UUID, a long hex
|
|
380
|
+
* string), or for another template. Never for a word: `/users/{id}` is not
|
|
381
|
+
* `/users/me`, `/things/:id` is not `/things/export`, and a path of three stars does
|
|
382
|
+
* not stand for every three-segment path. Same number of segments, every literal
|
|
383
|
+
* equal (case-insensitive), so a template never absorbs a slash or an empty
|
|
384
|
+
* segment.
|
|
385
|
+
*/
|
|
386
|
+
export function templatedPathsMatch(a, b) {
|
|
387
|
+
if (a.length !== b.length)
|
|
388
|
+
return false;
|
|
389
|
+
// +1: segments start at the first real one; isIdSegment counts as split("/") does.
|
|
390
|
+
const standsFor = (seg, i) => isTemplateSegment(seg) || isIdSegment(seg, i + 1);
|
|
391
|
+
return a.every((x, i) => {
|
|
392
|
+
const y = b[i];
|
|
393
|
+
if (isTemplateSegment(x))
|
|
394
|
+
return standsFor(y, i);
|
|
395
|
+
if (isTemplateSegment(y))
|
|
396
|
+
return standsFor(x, i);
|
|
397
|
+
return x === y;
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Whether two stated requests are the same call: same method, same route with
|
|
402
|
+
* a template read as an id (see templatedPathsMatch), and the same status — both stated and equal, or
|
|
403
|
+
* both unstated. A different status is a different outcome of the call, and
|
|
404
|
+
* pairing a 200 with a 403 would file one claim under its opposite.
|
|
405
|
+
*/
|
|
406
|
+
export function requestsPair(a, b) {
|
|
407
|
+
return a.method === b.method && a.status === b.status && templatedPathsMatch(a.segments, b.segments);
|
|
408
|
+
}
|
|
274
409
|
/**
|
|
275
410
|
* Which sessions are lanes of a parallel run, and whether each one's report
|
|
276
411
|
* has been folded, so a lane's session is not closed before its decisions
|