scenescout 3.14.0 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +27 -4
- package/dist/check-run.js +22 -11
- package/dist/ci-run.js +113 -17
- package/dist/cli.js +150 -75
- package/dist/commands.js +141 -0
- package/dist/engine/browser.js +546 -133
- package/dist/engine/check.js +77 -30
- package/dist/engine/ci.js +97 -0
- package/dist/engine/claims.js +81 -15
- package/dist/engine/collector.js +307 -65
- package/dist/engine/dedup.js +329 -9
- package/dist/engine/design.js +81 -12
- package/dist/engine/forms.js +17 -0
- package/dist/engine/memory.js +180 -36
- package/dist/engine/oracles.js +126 -1
- package/dist/engine/provider.js +4 -3
- package/dist/engine/refresh.js +270 -22
- package/dist/engine/report.js +8 -1
- package/dist/engine/scripted-login.js +342 -62
- package/dist/first-run.js +577 -0
- package/dist/login-run.js +136 -35
- package/dist/mcp-server.js +75 -16
- package/package.json +3 -2
- package/skills/scenescout/SKILL.md +3 -3
package/dist/engine/check.js
CHANGED
|
@@ -107,12 +107,19 @@ export const WORTH_A_LOOK_RULES = {
|
|
|
107
107
|
help: "Links with no underline, in the same colour as the page's body text. In running text a reader cannot tell them from the text around them; in navigation this styling is common, and the check cannot tell the two apart.",
|
|
108
108
|
convention: "a visible link style (an underline or a distinct colour) wherever links appear, navigation included",
|
|
109
109
|
},
|
|
110
|
+
"scrolled-out-controls": {
|
|
111
|
+
title: "Controls scrolled out of view sideways",
|
|
112
|
+
help: "Controls inside a horizontally scrolling container (a wide table's last column, say) lie outside its visible width at this viewport, so only a sideways scroll of the container shows them. A wide table that scrolls is a common, deliberate layout; it matters where a project keeps every row's actions in view.",
|
|
113
|
+
convention: "row actions and other controls that stay in view without a sideways scroll of their container at this viewport width",
|
|
114
|
+
},
|
|
110
115
|
};
|
|
111
116
|
export const CHECK_RULE_IDS = [...Object.keys(CHECK_RULES), ...Object.keys(WORTH_A_LOOK_RULES)];
|
|
112
117
|
export function isWorthALookRule(rule) {
|
|
113
118
|
return Object.prototype.hasOwnProperty.call(WORTH_A_LOOK_RULES, rule);
|
|
114
119
|
}
|
|
115
|
-
const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
|
|
120
|
+
export const SEVERITY_RANK = { high: 0, medium: 1, low: 2 };
|
|
121
|
+
/** The route a defect of the app's shared shell is charged to: one fix, however many pages show it. */
|
|
122
|
+
export const SHARED_CHROME_ROUTE = "(shared chrome)";
|
|
116
123
|
/** Snapshot refs (`e12`) are numbered per run; evidence carrying them would never match itself twice. */
|
|
117
124
|
function stripRefs(line) {
|
|
118
125
|
return line.replace(/\be\d+\s+/g, "").trim();
|
|
@@ -166,6 +173,8 @@ export function geometryRule(line) {
|
|
|
166
173
|
return "offpage-control";
|
|
167
174
|
if (/ overlaps /.test(line))
|
|
168
175
|
return "overlapping-controls";
|
|
176
|
+
if (/ scrolled out of view inside a horizontally scrolling container /.test(line))
|
|
177
|
+
return "scrolled-out-controls";
|
|
169
178
|
return "layout-issue";
|
|
170
179
|
}
|
|
171
180
|
/**
|
|
@@ -238,7 +247,7 @@ export function checkFindings(routes, origin, ignore = [], flows = []) {
|
|
|
238
247
|
for (const p of r.placeholderOnly)
|
|
239
248
|
add("placeholder-only-label", p, route);
|
|
240
249
|
for (const d of r.design)
|
|
241
|
-
add(d.rule, d.detail, d.chrome ?
|
|
250
|
+
add(d.rule, d.detail, d.chrome ? SHARED_CHROME_ROUTE : route);
|
|
242
251
|
}
|
|
243
252
|
for (const f of flows) {
|
|
244
253
|
for (const { path, violation } of f.violations)
|
|
@@ -367,7 +376,13 @@ export const CHECK_OPTION_NAMES = [
|
|
|
367
376
|
"nav-timeout-ms",
|
|
368
377
|
];
|
|
369
378
|
export const MAX_CHECK_ROUTES = 150;
|
|
379
|
+
/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
|
|
380
|
+
export const MAX_DISCOVERY_ROUNDS = 6;
|
|
370
381
|
export const DEFAULT_CHECK_ROUTES = 50;
|
|
382
|
+
/** A path given on the command line, resolved from the directory the command runs in; the same on every platform. */
|
|
383
|
+
export function resolveArgPath(cwd, p) {
|
|
384
|
+
return p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`;
|
|
385
|
+
}
|
|
371
386
|
/** Parse `scenescout check` arguments. Every mistake is a sentence, never a half-configured run. */
|
|
372
387
|
export function parseCheckArgs(args, cwd) {
|
|
373
388
|
const positional = [];
|
|
@@ -461,7 +476,7 @@ export function parseCheckArgs(args, cwd) {
|
|
|
461
476
|
const gateRetests = oneOf("gate-retests", GATE_RETESTS, DEFAULT_SETTINGS.gateRetests);
|
|
462
477
|
if (!gateRetests)
|
|
463
478
|
return { ok: false, error: `--gate-retests must be one of ${GATE_RETESTS.join(", ")}` };
|
|
464
|
-
const resolve = (p) => (
|
|
479
|
+
const resolve = (p) => resolveArgPath(cwd, p);
|
|
465
480
|
return {
|
|
466
481
|
ok: true,
|
|
467
482
|
options: {
|
|
@@ -548,7 +563,7 @@ const RETEST_SARIF_RULE = {
|
|
|
548
563
|
defaultConfiguration: { level: "error" },
|
|
549
564
|
};
|
|
550
565
|
function sarifLocation(route) {
|
|
551
|
-
return route ===
|
|
566
|
+
return route === SHARED_CHROME_ROUTE
|
|
552
567
|
? {
|
|
553
568
|
physicalLocation: { artifactLocation: { uri: "", uriBaseId: "APP" } },
|
|
554
569
|
message: { text: "the app's shared shell, on every page that renders it" },
|
|
@@ -659,10 +674,63 @@ function code(text) {
|
|
|
659
674
|
const pad = longest > 0 ? " " : "";
|
|
660
675
|
return `${fence}${pad}${text}${pad}${fence}`;
|
|
661
676
|
}
|
|
677
|
+
/** The first three routes as code spans, and how many more. */
|
|
662
678
|
function routeList(routes) {
|
|
663
679
|
const shown = routes.slice(0, 3).map(code);
|
|
664
680
|
return shown.join(", ") + (routes.length > 3 ? ` and ${routes.length - 3} more` : "");
|
|
665
681
|
}
|
|
682
|
+
/** One markdown list item for an issue: its title, rule, evidence and routes. */
|
|
683
|
+
export function issueLine(i) {
|
|
684
|
+
return `**${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`;
|
|
685
|
+
}
|
|
686
|
+
/** The issues, one section per severity, worst first. Shared by the check's report and the first run's. */
|
|
687
|
+
export function issueSections(issues) {
|
|
688
|
+
const lines = [];
|
|
689
|
+
for (const sev of CHECK_SEVERITIES) {
|
|
690
|
+
const of = issues.filter((i) => i.severity === sev);
|
|
691
|
+
if (of.length === 0)
|
|
692
|
+
continue;
|
|
693
|
+
lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
|
|
694
|
+
for (const i of of)
|
|
695
|
+
lines.push(`- ${issueLine(i)}`);
|
|
696
|
+
}
|
|
697
|
+
return lines;
|
|
698
|
+
}
|
|
699
|
+
const WORTH_A_LOOK_INTRO = "Measured exactly, and defects only under a convention of your project that the check cannot see.";
|
|
700
|
+
/** The worth-a-look section, or nothing when there is none. `gated`: the report is a gate's, so say these never fail it. */
|
|
701
|
+
export function worthALookSection(observations, gated = true) {
|
|
702
|
+
if (observations.length === 0)
|
|
703
|
+
return [];
|
|
704
|
+
const lines = [
|
|
705
|
+
"",
|
|
706
|
+
`## Worth a look (${observations.length})`,
|
|
707
|
+
"",
|
|
708
|
+
`${WORTH_A_LOOK_INTRO} They are not counted above${gated ? " and never fail the gate, at any --fail-on" : ""}.`,
|
|
709
|
+
"",
|
|
710
|
+
];
|
|
711
|
+
for (const o of observations) {
|
|
712
|
+
lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
|
|
713
|
+
}
|
|
714
|
+
return lines;
|
|
715
|
+
}
|
|
716
|
+
/** The table of routes measured, with each one's status, controls and issue count. */
|
|
717
|
+
export function routesTable(result) {
|
|
718
|
+
const lines = ["", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|"];
|
|
719
|
+
for (const r of result.routes) {
|
|
720
|
+
const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
|
|
721
|
+
const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
|
|
722
|
+
(r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
|
|
723
|
+
// Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
|
|
724
|
+
lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
|
|
725
|
+
}
|
|
726
|
+
return lines;
|
|
727
|
+
}
|
|
728
|
+
/** The routes known and not visited, as one line; nothing when every known route was visited. */
|
|
729
|
+
export function unvisitedLine(unvisited, why) {
|
|
730
|
+
if (unvisited.length === 0)
|
|
731
|
+
return [];
|
|
732
|
+
return ["", `Not visited (${why}): ${unvisited.slice(0, 20).map(code).join(", ")}${unvisited.length > 20 ? " …" : ""}`];
|
|
733
|
+
}
|
|
666
734
|
/** The human report: the verdict first, then what failed it, then everything else. */
|
|
667
735
|
export function formatCheck(result) {
|
|
668
736
|
const { passed, counts, failing, retestsFailing, couldNotRun } = summarise(result);
|
|
@@ -683,32 +751,10 @@ export function formatCheck(result) {
|
|
|
683
751
|
(result.worthALook.length > 0 ? ` · ${result.worthALook.length} worth a look, never gated` : ""));
|
|
684
752
|
// Right under the verdict: what a green check was allowed to do is part of what it means.
|
|
685
753
|
lines.push("", `Settings — ${describeSettings(result)}`);
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
lines.push("", `## ${sev[0].toUpperCase()}${sev.slice(1)} (${of.length})`, "");
|
|
691
|
-
for (const i of of) {
|
|
692
|
-
lines.push(`- **${CHECK_RULES[i.rule].title}** \`${i.rule}\`: ${cell(i.evidence)}${i.embed ? ` _(in an embed of ${i.embed}: its behaviour, not the app's)_` : ""} — ${routeList(i.routes)}`);
|
|
693
|
-
}
|
|
694
|
-
}
|
|
695
|
-
if (result.worthALook.length > 0) {
|
|
696
|
-
lines.push("", `## Worth a look (${result.worthALook.length})`, "", "Measured exactly, and defects only under a convention of your project that the check cannot see. They are not counted above and never fail the gate, at any --fail-on.", "");
|
|
697
|
-
for (const o of result.worthALook) {
|
|
698
|
-
lines.push(`- **${WORTH_A_LOOK_RULES[o.rule].title}** \`${o.rule}\`: ${cell(o.evidence)} — a defect only if your project uses ${o.convention} — ${routeList(o.routes)}`);
|
|
699
|
-
}
|
|
700
|
-
}
|
|
701
|
-
lines.push("", "## Routes", "", "| Route | Status | Controls | Issues |", "|---|---|---|---|");
|
|
702
|
-
for (const r of result.routes) {
|
|
703
|
-
const n = result.issues.filter((i) => i.routes.includes(r.path)).length;
|
|
704
|
-
const status = (r.loadError !== undefined ? "did not load" : r.loginRedirect ? `${r.status ?? "?"} → sign-in` : String(r.status ?? "?")) +
|
|
705
|
-
(r.auditError ? ` (design not measured: ${cell(r.auditError.slice(0, 80))})` : "");
|
|
706
|
-
// Plain text, not a code span: inside a table cell a code span keeps the backslashes cell() adds.
|
|
707
|
-
lines.push(`| ${cell(r.path)} | ${status} | ${r.elements} | ${n} |`);
|
|
708
|
-
}
|
|
709
|
-
if (result.unvisited.length > 0) {
|
|
710
|
-
lines.push("", `Not visited (over --max-routes): ${result.unvisited.slice(0, 20).map(code).join(", ")}${result.unvisited.length > 20 ? " …" : ""}`);
|
|
711
|
-
}
|
|
754
|
+
lines.push(...issueSections(result.issues));
|
|
755
|
+
lines.push(...worthALookSection(result.worthALook));
|
|
756
|
+
lines.push(...routesTable(result));
|
|
757
|
+
lines.push(...unvisitedLine(result.unvisited, "over --max-routes"));
|
|
712
758
|
if (result.flows.length > 0 || result.skippedFlows.length > 0) {
|
|
713
759
|
lines.push("", `## Flows (${result.flows.length})`, "");
|
|
714
760
|
for (const f of result.flows) {
|
|
@@ -778,6 +824,7 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
778
824
|
...(r.auditError !== undefined ? { designNotMeasured: r.auditError } : {}),
|
|
779
825
|
})),
|
|
780
826
|
unvisited: result.unvisited,
|
|
827
|
+
...(result.timeBudget ? { timeBudget: result.timeBudget } : {}),
|
|
781
828
|
ignored: result.ignored,
|
|
782
829
|
flows: result.flows.map((f) => ({
|
|
783
830
|
name: f.name,
|
package/dist/engine/ci.js
CHANGED
|
@@ -36,6 +36,21 @@ export const DEFAULT_EFFORT = "low";
|
|
|
36
36
|
*/
|
|
37
37
|
export const CI_MODES = ["observe", "read-only", "safe-write", "destructive"];
|
|
38
38
|
export const CI_LEVELS = ["minimal", "medium", "extensive"];
|
|
39
|
+
/**
|
|
40
|
+
* How a filed finding is told apart from one already stored. `rule`: the
|
|
41
|
+
* store's rule alone (memory.ts isDuplicateFinding). `judge`: the rule first,
|
|
42
|
+
* then, for a filing the rule keeps apart, a model asked about the open
|
|
43
|
+
* findings on the same page (engine/dedup.ts DedupJudge), with the rule's
|
|
44
|
+
* decision kept whenever the model cannot answer. A CI run already sends
|
|
45
|
+
* pages to a model and holds a key, so it judges by default; the MCP server
|
|
46
|
+
* judges only when asked to (DEDUP_ENV or scout_attach {dedup}).
|
|
47
|
+
*/
|
|
48
|
+
export const DEDUP_MODES = ["rule", "judge"];
|
|
49
|
+
export const DEFAULT_CI_DEDUP = "judge";
|
|
50
|
+
/** Turns the dedup judge on for the MCP server: `rule` (the default) or `judge`. An attach's `dedup` wins over it. */
|
|
51
|
+
export const DEDUP_ENV = "SCENESCOUT_DEDUP";
|
|
52
|
+
/** Which provider the MCP server's dedup judge uses when both keys are in its environment. */
|
|
53
|
+
export const DEDUP_PROVIDER_ENV = "SCENESCOUT_DEDUP_PROVIDER";
|
|
39
54
|
export const DEFAULT_CAPS = { turns: 40, tokens: 1_500_000, wallMs: 20 * 60_000 };
|
|
40
55
|
const CAP_BOUNDS = { turns: [1, 500], tokens: [1_000, 20_000_000], minutes: [1, 360] };
|
|
41
56
|
/** Every option `scenescout ci` accepts; the ci action's inputs are these names (ci-test holds them equal). */
|
|
@@ -62,6 +77,7 @@ export const CI_OPTION_NAMES = [
|
|
|
62
77
|
"out",
|
|
63
78
|
"show",
|
|
64
79
|
"compare-url",
|
|
80
|
+
"dedup",
|
|
65
81
|
];
|
|
66
82
|
/** The longest --show description: it becomes a line of the model's prompt. */
|
|
67
83
|
export const MAX_SHOW = 200;
|
|
@@ -225,6 +241,9 @@ export function parseCiArgs(args, cwd) {
|
|
|
225
241
|
const navTimeout = parseLimitFlag("nav", flags.get("nav-timeout-ms"));
|
|
226
242
|
if (!navTimeout.ok)
|
|
227
243
|
return navTimeout;
|
|
244
|
+
const dedup = flags.get("dedup") ?? DEFAULT_CI_DEDUP;
|
|
245
|
+
if (!DEDUP_MODES.includes(dedup))
|
|
246
|
+
return { ok: false, error: `--dedup must be one of ${DEDUP_MODES.join(", ")}` };
|
|
228
247
|
const resolve = (p) => (p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`);
|
|
229
248
|
return {
|
|
230
249
|
ok: true,
|
|
@@ -247,6 +266,7 @@ export function parseCiArgs(args, cwd) {
|
|
|
247
266
|
...(compareUrl ? { compareUrl } : {}),
|
|
248
267
|
...(actionTimeout.value !== undefined ? { actionTimeoutMs: actionTimeout.value } : {}),
|
|
249
268
|
...(navTimeout.value !== undefined ? { navTimeoutMs: navTimeout.value } : {}),
|
|
269
|
+
dedup: dedup,
|
|
250
270
|
},
|
|
251
271
|
};
|
|
252
272
|
}
|
|
@@ -278,6 +298,57 @@ export function detectProvider(env, options) {
|
|
|
278
298
|
return { ok: false, error: `--effort must be one of ${EFFORTS[provider].join(", ")} for ${provider}` };
|
|
279
299
|
return { ok: true, resolved: { provider, model: options.model ?? DEFAULT_MODEL[provider], effort, baseUrl: options.baseUrl ?? DEFAULT_BASE_URL[provider] } };
|
|
280
300
|
}
|
|
301
|
+
// ── the dedup judge's model ─────────────────────────────────────────────────
|
|
302
|
+
/**
|
|
303
|
+
* The effort the dedup judge asks at: the lowest the provider's API takes.
|
|
304
|
+
* Measured, effort none judged as well as low and changed no verdict
|
|
305
|
+
* (docs/benchmark.md); the Messages API has no none, so Anthropic gets low.
|
|
306
|
+
*/
|
|
307
|
+
export function judgeEffort(provider) {
|
|
308
|
+
return EFFORTS[provider][0];
|
|
309
|
+
}
|
|
310
|
+
/** The MCP server's dedup mode from its environment: `rule` when unset. A value that is neither is refused, naming the variable. */
|
|
311
|
+
export function dedupModeFromEnv(env) {
|
|
312
|
+
const raw = (env[DEDUP_ENV] ?? "").trim();
|
|
313
|
+
if (raw === "")
|
|
314
|
+
return "rule";
|
|
315
|
+
if (!DEDUP_MODES.includes(raw))
|
|
316
|
+
throw new Error(`${DEDUP_ENV} must be one of ${DEDUP_MODES.join(", ")}, not ${JSON.stringify(raw)}`);
|
|
317
|
+
return raw;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* The key and model the MCP server's dedup judge uses when no CI run answers
|
|
321
|
+
* for it: the provider whose key is in the server's environment (with both,
|
|
322
|
+
* the one DEDUP_PROVIDER_ENV names), its default model and its lowest effort.
|
|
323
|
+
* A missing key is an answer (`ok: false`, and the rule decides); a provider
|
|
324
|
+
* name that is neither is refused.
|
|
325
|
+
*/
|
|
326
|
+
export function judgeKeyConfig(env) {
|
|
327
|
+
const named = (env[DEDUP_PROVIDER_ENV] ?? "").trim();
|
|
328
|
+
if (named !== "" && !PROVIDERS.includes(named))
|
|
329
|
+
throw new Error(`${DEDUP_PROVIDER_ENV} must be one of ${PROVIDERS.join(", ")}, not ${JSON.stringify(named)}`);
|
|
330
|
+
const withKey = PROVIDERS.filter((p) => present(env, KEY_ENV[p]));
|
|
331
|
+
let provider;
|
|
332
|
+
if (named !== "") {
|
|
333
|
+
provider = named;
|
|
334
|
+
if (!withKey.includes(provider))
|
|
335
|
+
return { ok: false, error: `${DEDUP_PROVIDER_ENV}=${provider} needs ${KEY_ENV[provider]} in the server's environment` };
|
|
336
|
+
}
|
|
337
|
+
else if (withKey.length === 0) {
|
|
338
|
+
return { ok: false, error: `the judge needs ${KEY_ENV.anthropic} or ${KEY_ENV.openai} in the server's environment` };
|
|
339
|
+
}
|
|
340
|
+
else if (withKey.length > 1) {
|
|
341
|
+
return { ok: false, error: `both ${KEY_ENV.anthropic} and ${KEY_ENV.openai} are set: set ${DEDUP_PROVIDER_ENV} to anthropic or openai to choose` };
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
provider = withKey[0];
|
|
345
|
+
}
|
|
346
|
+
return {
|
|
347
|
+
ok: true,
|
|
348
|
+
resolved: { provider, model: DEFAULT_MODEL[provider], effort: judgeEffort(provider), baseUrl: DEFAULT_BASE_URL[provider] },
|
|
349
|
+
key: (env[KEY_ENV[provider]] ?? "").trim(),
|
|
350
|
+
};
|
|
351
|
+
}
|
|
281
352
|
/**
|
|
282
353
|
* The environment the MCP server and its browser are started with: this
|
|
283
354
|
* process's, without the keys. Nothing on that side needs them, and a key a
|
|
@@ -576,6 +647,14 @@ export function findingsThisRun(before, after) {
|
|
|
576
647
|
const runsBefore = new Map(before.map((f) => [f.id, f.runs]));
|
|
577
648
|
return after.filter((f) => f.status !== "resolved" && (!runsBefore.has(f.id) || f.runs > (runsBefore.get(f.id) ?? 0)));
|
|
578
649
|
}
|
|
650
|
+
/** One line on how a run deduplicated, for the summary. */
|
|
651
|
+
export function dedupLine(d) {
|
|
652
|
+
if (d.by === "rule")
|
|
653
|
+
return "the rule alone";
|
|
654
|
+
const tokens = d.usage.input + d.usage.output;
|
|
655
|
+
return (`the rule, then the model judge${d.effort ? ` at effort ${d.effort}` : ""} for filings it kept apart: ${d.calls} call(s)` +
|
|
656
|
+
`${d.failed ? `, ${d.failed} without an answer (the rule decided those)` : ""}, ${n(tokens)} tokens (in the usage below), ${(d.ms / 1000).toFixed(1)}s`);
|
|
657
|
+
}
|
|
579
658
|
const SEVERITY_ORDER = { high: 0, medium: 1, low: 2 };
|
|
580
659
|
function pathOf(url) {
|
|
581
660
|
try {
|
|
@@ -604,6 +683,7 @@ export function ciSummaryMarkdown(r, secrets = []) {
|
|
|
604
683
|
...(r.capture ? [] : [`| Level | ${r.level} — completion contract ${r.contractMet ? "met" : "not met (the report's gap ledger says what is missing)"} |`]),
|
|
605
684
|
`| Mode | ${r.mode} |`,
|
|
606
685
|
`| Model | ${r.provider} ${cell(r.model, secrets)}, effort ${r.effort} |`,
|
|
686
|
+
...(r.dedup ? [`| Finding dedup | ${dedupLine(r.dedup)} |`] : []),
|
|
607
687
|
`| Usage | ${usageLine(r.spend, r.model, r.endedAt, r.price)} |`,
|
|
608
688
|
``,
|
|
609
689
|
];
|
|
@@ -668,6 +748,23 @@ export function ciSummaryJson(r, version, secrets = []) {
|
|
|
668
748
|
seconds: Math.round((r.endedAt - r.spend.startedAt) / 1000),
|
|
669
749
|
estimatedCostUsd: estimateCost(r.model, r.spend.usage, r.price),
|
|
670
750
|
},
|
|
751
|
+
...(r.dedup
|
|
752
|
+
? {
|
|
753
|
+
dedup: {
|
|
754
|
+
by: r.dedup.by,
|
|
755
|
+
...(r.dedup.by === "judge"
|
|
756
|
+
? {
|
|
757
|
+
effort: r.dedup.effort,
|
|
758
|
+
calls: r.dedup.calls,
|
|
759
|
+
failed: r.dedup.failed,
|
|
760
|
+
inputTokens: r.dedup.usage.input,
|
|
761
|
+
outputTokens: r.dedup.usage.output,
|
|
762
|
+
seconds: Math.round(r.dedup.ms / 100) / 10,
|
|
763
|
+
}
|
|
764
|
+
: {}),
|
|
765
|
+
},
|
|
766
|
+
}
|
|
767
|
+
: {}),
|
|
671
768
|
counts: {
|
|
672
769
|
high: r.findings.filter((f) => !isWorthALook(f) && f.severity === "high").length,
|
|
673
770
|
medium: r.findings.filter((f) => !isWorthALook(f) && f.severity === "medium").length,
|
package/dist/engine/claims.js
CHANGED
|
@@ -86,6 +86,20 @@ const ERROR_RE = /\b(?:error|failed|failure|could ?n[o'’]?t|unable to|went wro
|
|
|
86
86
|
* unreported because the Danger zone said "This cannot be undone."
|
|
87
87
|
*/
|
|
88
88
|
const REFUSAL_RE = /\b(?:can(?:not|[’'`]?t| not)|must be|(?:is|are) already|only (?:an? |the )?\w+(?: \w+)? (?:can|may|be)|may only|can only(?: be)?|(?:nothing|not) (?:was|has been|have been|been) (?:saved|sent|deleted|updated|created|changed|submitted)|(?:was|were|has|have|is|are)(?: not|n[’']t)(?: been)? (?:saved|sent|deleted|updated|created|changed|submitted))\b/i;
|
|
89
|
+
/**
|
|
90
|
+
* Words that name a refusal outright. An admission when the page ANNOUNCES them
|
|
91
|
+
* or puts them up in answer to the action, as when a toast echoes the server's
|
|
92
|
+
* "was refused" message. Not page-wide: "Rejected" and "Blocked" are ordinary
|
|
93
|
+
* status badges and filter tabs on record pages, and counted wherever they
|
|
94
|
+
* stood they would excuse every lie on such a page.
|
|
95
|
+
*/
|
|
96
|
+
const REFUSED_RE = /\b(?:refused|rejected|blocked)\b/i;
|
|
97
|
+
/**
|
|
98
|
+
* Addresses of a page's own infrastructure writes: a token refresh, telemetry,
|
|
99
|
+
* an error monitor, a heartbeat. Refusing one says nothing about whether the
|
|
100
|
+
* change the user made was kept, so it is never paired with a success claim.
|
|
101
|
+
*/
|
|
102
|
+
export const INFRASTRUCTURE_WRITE_RE = /\/auth\/(refresh|token|session)|refresh[-_]?token|\/telemetry|\/analytics|\/heartbeat|\/sentry|\/collect\b|\/logs?\b|\/metrics\b/i;
|
|
89
103
|
/** Whether a piece of announced text explains a refusal. */
|
|
90
104
|
export function isRefusalNotice(text) {
|
|
91
105
|
const trimmed = text.trim();
|
|
@@ -110,6 +124,15 @@ export function classify(text) {
|
|
|
110
124
|
return "empty";
|
|
111
125
|
return null;
|
|
112
126
|
}
|
|
127
|
+
/** What counts as an open dialog: a native one or an ARIA one. */
|
|
128
|
+
const DIALOG_SEL = 'dialog[open], [role="dialog"], [role="alertdialog"]';
|
|
129
|
+
/** Page-side count of the open dialogs alone, for a caller that needs nothing else the claim scan reads. */
|
|
130
|
+
export const OPEN_DIALOGS_SCRIPT = `(() => {
|
|
131
|
+
const visible = ${VISIBLE_SRC};
|
|
132
|
+
let dialogs = 0;
|
|
133
|
+
for (const d of document.querySelectorAll(${JSON.stringify(DIALOG_SEL)})) if (visible(d)) dialogs += 1;
|
|
134
|
+
return dialogs;
|
|
135
|
+
})()`;
|
|
113
136
|
function shortUrl(url) {
|
|
114
137
|
try {
|
|
115
138
|
const u = new URL(url);
|
|
@@ -128,25 +151,42 @@ function standIn(req) {
|
|
|
128
151
|
? ` The refusal was the write policy's stand-in (the server never received the request); what failed is the page's handling of a refusal, which a real one would meet the same way.`
|
|
129
152
|
: "";
|
|
130
153
|
}
|
|
154
|
+
/** Whether an announced sentence admits the failure: an error, a refusal named outright, or one explained in the app's own words. */
|
|
155
|
+
function admitsInAnnouncement(text) {
|
|
156
|
+
return classify(text) === "error" || REFUSED_RE.test(text) || isRefusalNotice(text);
|
|
157
|
+
}
|
|
131
158
|
/**
|
|
132
159
|
* The contradictions this action produced, if any.
|
|
133
160
|
*
|
|
134
161
|
* Both rules are silent whenever the page admits the failure, and both require
|
|
135
162
|
* a genuinely refused request — the page half never fires alone.
|
|
163
|
+
*
|
|
164
|
+
* `before` is what the page said when the action began, when the action was
|
|
165
|
+
* the user's (a click, a keypress, typing). A success claim already on screen
|
|
166
|
+
* then is not the page's answer to this action's write: a status badge reading
|
|
167
|
+
* "Published", a heading, a row from earlier. Only what the action put up can
|
|
168
|
+
* contradict what the action's write met. Without it, every text counts.
|
|
136
169
|
*/
|
|
137
|
-
export function findContradictions(requests, page) {
|
|
170
|
+
export function findContradictions(requests, page, before) {
|
|
138
171
|
const refused = requests.filter(isRefused);
|
|
139
172
|
if (refused.length === 0)
|
|
140
173
|
return [];
|
|
141
174
|
const claims = page.texts.map(classify);
|
|
175
|
+
const earlier = before ? new Set([...before.texts, ...(before.announced ?? [])]) : null;
|
|
176
|
+
const isNew = (text) => earlier === null || !earlier.has(text);
|
|
142
177
|
// An app that says what went wrong has behaved correctly, and nothing below
|
|
143
178
|
// applies. This is checked before anything else so that a page carrying both
|
|
144
179
|
// an error banner and a stale empty state is not reported.
|
|
145
180
|
if (claims.includes("error"))
|
|
146
181
|
return [];
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
|
|
182
|
+
// An admission where the page announces its responses: an error, a refusal
|
|
183
|
+
// named outright (an app echoing the server's "was refused"), or one
|
|
184
|
+
// explained in the app's own words. Help text with the same wording
|
|
185
|
+
// elsewhere excuses nothing.
|
|
186
|
+
if ((page.announced ?? []).some(admitsInAnnouncement))
|
|
187
|
+
return [];
|
|
188
|
+
// A refusal named outright in text the action put up, announced or not.
|
|
189
|
+
if (earlier !== null && page.texts.some((t) => isNew(t) && t.length <= CLAIM_TEXT_MAX && REFUSED_RE.test(t)))
|
|
150
190
|
return [];
|
|
151
191
|
const out = [];
|
|
152
192
|
const reads = refused.filter((r) => !WRITING_METHODS.has(r.method.toUpperCase()));
|
|
@@ -162,15 +202,36 @@ export function findContradictions(requests, page) {
|
|
|
162
202
|
evidence: `refused-empty ${say(worst)}`,
|
|
163
203
|
});
|
|
164
204
|
}
|
|
165
|
-
|
|
166
|
-
|
|
205
|
+
// Only writes this action sent, and not the page's own infrastructure.
|
|
206
|
+
const isActionWrite = (r) => WRITING_METHODS.has(r.method.toUpperCase()) && !r.background && !INFRASTRUCTURE_WRITE_RE.test(r.url);
|
|
207
|
+
const writes = refused.filter(isActionWrite);
|
|
208
|
+
// Writes of the same action that went through. The message may be about
|
|
209
|
+
// one of them, so a success claim beside them is a PARTIAL false success:
|
|
210
|
+
// part of the user's change was refused and the page said nothing about
|
|
211
|
+
// that part. Still reported, at medium, because silent partial loss is a
|
|
212
|
+
// real defect; all refused stays high.
|
|
213
|
+
const kept = requests.filter((r) => isActionWrite(r) && DATA_RESOURCES.has(r.resourceType) && !r.blockedByPolicy && r.status !== null && r.status < 400);
|
|
214
|
+
const successAt = claims.findIndex((c, i) => c === "success" && isNew(page.texts[i]));
|
|
215
|
+
if (writes.length > 0 && successAt >= 0) {
|
|
167
216
|
const worst = writes[0];
|
|
168
|
-
const message = page.texts[
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
217
|
+
const message = JSON.stringify(page.texts[successAt].trim().slice(0, 80));
|
|
218
|
+
if (kept.length === 0) {
|
|
219
|
+
out.push({
|
|
220
|
+
kind: "false_success",
|
|
221
|
+
detail: `${say(worst)} was refused, and the page says ${message}. The user is told their change was kept when the server rejected it.${standIn(worst)}`,
|
|
222
|
+
evidence: `false-success ${say(worst)}`,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
const total = writes.length + kept.length;
|
|
227
|
+
out.push({
|
|
228
|
+
kind: "false_success",
|
|
229
|
+
severity: "medium",
|
|
230
|
+
detail: `partial: ${writes.length} of ${total} writes from this action were refused (${say(worst)}), and the page says ${message} with no word about the refused part. ` +
|
|
231
|
+
`The user is told their change was kept when part of it was rejected.${standIn(worst)}`,
|
|
232
|
+
evidence: `false-success-partial ${say(worst)}`,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
174
235
|
}
|
|
175
236
|
return out;
|
|
176
237
|
}
|
|
@@ -200,6 +261,7 @@ export const CLAIM_SCAN_SCRIPT = `(() => {
|
|
|
200
261
|
// is a container of static text (its form's help, its warning), and counting
|
|
201
262
|
// it brought back the help text this set exists to exclude.
|
|
202
263
|
const ANNOUNCES = "[role~='status'], [role~='alert'], [aria-live]:not([aria-live='off']), output";
|
|
264
|
+
const COLUMN_HEADER = "th, [role~='columnheader'], [aria-sort]";
|
|
203
265
|
const seen = new Set();
|
|
204
266
|
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
|
|
205
267
|
let el = document.body;
|
|
@@ -207,7 +269,8 @@ export const CLAIM_SCAN_SCRIPT = `(() => {
|
|
|
207
269
|
let own = "";
|
|
208
270
|
for (const node of el.childNodes) if (node.nodeType === 3) own += node.nodeValue;
|
|
209
271
|
own = own.replace(/\\s+/g, " ").trim();
|
|
210
|
-
|
|
272
|
+
// A column header names what a column holds ("Updated on", "Completed"); it claims nothing.
|
|
273
|
+
if (own && own.length <= ${CLAIM_TEXT_MAX} && !seen.has(own) && !el.closest(COLUMN_HEADER) && visible(el)) {
|
|
211
274
|
seen.add(own);
|
|
212
275
|
texts.push(own);
|
|
213
276
|
}
|
|
@@ -243,7 +306,10 @@ export const CLAIM_SCAN_SCRIPT = `(() => {
|
|
|
243
306
|
if (s && s.length <= ${CLAIM_TEXT_MAX} && !announced.includes(s)) announced.push(s);
|
|
244
307
|
}
|
|
245
308
|
}
|
|
309
|
+
// Open dialogs, so a page error raised as one opens can be read as the
|
|
310
|
+
// confirmation a router cancelled a route change for (oracles.ts).
|
|
311
|
+
let dialogs = 0;
|
|
312
|
+
for (const d of document.querySelectorAll(${JSON.stringify(DIALOG_SEL)})) if (visible(d)) dialogs += 1;
|
|
246
313
|
|
|
247
|
-
|
|
248
|
-
return { texts, announced, emptyLists };
|
|
314
|
+
return { texts, announced, emptyLists, dialogs };
|
|
249
315
|
})()`;
|