scenescout 3.14.1 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +90 -18
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +211 -24
- package/dist/ci-run.js +374 -62
- package/dist/cli.js +221 -47
- package/dist/commands.js +42 -4
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1660 -386
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +258 -68
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +233 -18
- package/dist/engine/claims.js +234 -12
- package/dist/engine/collector.js +851 -78
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/dedup.js +329 -9
- package/dist/engine/design.js +355 -43
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +198 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +668 -70
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +167 -2
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/provider.js +4 -3
- package/dist/engine/refresh.js +338 -25
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +248 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/scripted-login.js +342 -62
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +581 -0
- package/dist/installer.js +267 -3
- package/dist/intake.js +104 -0
- package/dist/login-run.js +386 -71
- package/dist/mcp-server.js +637 -68
- package/dist/playbook.js +5 -0
- package/package.json +6 -3
- package/skills/scenescout/SKILL.md +51 -18
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opening the live view and the report in the user's default browser.
|
|
3
|
+
*
|
|
4
|
+
* The live view's address and the report's path are printed in tool results,
|
|
5
|
+
* but a person in a chat client may never see or follow them. The `open`
|
|
6
|
+
* setting has the engine open them itself: the live view when a session
|
|
7
|
+
* attaches, the report when scout_report writes it.
|
|
8
|
+
*
|
|
9
|
+
* Whether to open is decided here, by a pure function of the setting and the
|
|
10
|
+
* environment, so every case is table-tested. Opening goes through the
|
|
11
|
+
* platform opener with no shell, and only ever hands it a loopback URL or an
|
|
12
|
+
* absolute file path. The live view's own rules (loopback only, a token on
|
|
13
|
+
* every path, ADR 7) are unchanged: opening an address reaches nobody new.
|
|
14
|
+
*
|
|
15
|
+
* Nothing here imports Playwright, and the spawn is injected, so a test never
|
|
16
|
+
* launches a browser.
|
|
17
|
+
*/
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { isCiEnv } from "./capture.js";
|
|
20
|
+
/** The environment variable that sets `open` for every session of the server. A `scout_attach` `open` wins over it. */
|
|
21
|
+
export const OPEN_ENV = "SCENESCOUT_OPEN";
|
|
22
|
+
export const OPEN_CHOICES = ["live", "report", "both", "none"];
|
|
23
|
+
/** `SCENESCOUT_OPEN`, or undefined when unset. A value outside the choices is refused rather than ignored. */
|
|
24
|
+
export function openChoiceFromEnv(env) {
|
|
25
|
+
const raw = (env[OPEN_ENV] ?? "").trim();
|
|
26
|
+
if (raw === "")
|
|
27
|
+
return undefined;
|
|
28
|
+
if (!OPEN_CHOICES.includes(raw))
|
|
29
|
+
throw new Error(`${OPEN_ENV} must be one of ${OPEN_CHOICES.join(", ")}, not ${JSON.stringify(raw)}`);
|
|
30
|
+
return raw;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Nobody would see a window opened from here: a session over SSH, where the
|
|
34
|
+
* browser would open on the remote machine if at all, or a Linux or other
|
|
35
|
+
* Unix machine with no display server. macOS and Windows always have one.
|
|
36
|
+
*/
|
|
37
|
+
export function noDisplay(env, platform) {
|
|
38
|
+
if (env.SSH_CONNECTION || env.SSH_TTY)
|
|
39
|
+
return true;
|
|
40
|
+
if (platform === "darwin" || platform === "win32")
|
|
41
|
+
return false;
|
|
42
|
+
return !env.DISPLAY && !env.WAYLAND_DISPLAY;
|
|
43
|
+
}
|
|
44
|
+
const both = (choice, why) => ({
|
|
45
|
+
live: choice === "live" || choice === "both",
|
|
46
|
+
report: choice === "report" || choice === "both",
|
|
47
|
+
why,
|
|
48
|
+
});
|
|
49
|
+
/**
|
|
50
|
+
* What opens. A setting, from the attach or the environment, is followed as
|
|
51
|
+
* given. With none, both open on a local desktop session, whether the
|
|
52
|
+
* session's browser is headed or headless and whatever the MCP client
|
|
53
|
+
* declares: what a client declares is outside this project's control, and a
|
|
54
|
+
* person in a chat client is the one who most needs the page opened. Nothing
|
|
55
|
+
* opens in CI or where no display would show it. A developer who wants no
|
|
56
|
+
* tabs sets SCENESCOUT_OPEN=none.
|
|
57
|
+
*/
|
|
58
|
+
export function decideOpen(choice, ctx) {
|
|
59
|
+
if (choice)
|
|
60
|
+
return both(choice, `open is set to ${choice}`);
|
|
61
|
+
if (isCiEnv(ctx.env))
|
|
62
|
+
return both("none", "a CI run");
|
|
63
|
+
if (noDisplay(ctx.env, ctx.platform))
|
|
64
|
+
return both("none", "no display to show a window on");
|
|
65
|
+
return both("both", "a local desktop session");
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Only what the engine itself produced may be opened: the live view's
|
|
69
|
+
* loopback address (plain http, no credentials) or an absolute path to a file.
|
|
70
|
+
* Anything else — a remote URL, a relative path, something that could read as
|
|
71
|
+
* an option to the opener — is refused.
|
|
72
|
+
*/
|
|
73
|
+
export function openableTarget(target) {
|
|
74
|
+
if (target === "" || /[\0-\x1f\x7f]/.test(target) || target.startsWith("-"))
|
|
75
|
+
return false;
|
|
76
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(target)) {
|
|
77
|
+
let u;
|
|
78
|
+
try {
|
|
79
|
+
u = new URL(target);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
return u.protocol === "http:" && ["127.0.0.1", "localhost", "[::1]"].includes(u.hostname) && !u.username && !u.password;
|
|
85
|
+
}
|
|
86
|
+
return path.isAbsolute(target) || path.win32.isAbsolute(target);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The platform's opener, run directly with the target as its only argument.
|
|
90
|
+
* Windows' `start` is a cmd.exe built-in and cannot run without a shell, so
|
|
91
|
+
* Windows uses the URL protocol handler `start` itself relies on.
|
|
92
|
+
*/
|
|
93
|
+
export function openerCommand(platform, target) {
|
|
94
|
+
if (platform === "darwin")
|
|
95
|
+
return { command: "open", args: [target] };
|
|
96
|
+
if (platform === "win32")
|
|
97
|
+
return { command: "rundll32", args: ["url.dll,FileProtocolHandler", target] };
|
|
98
|
+
return { command: "xdg-open", args: [target] };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Hand a target to the platform opener. Refuses a target openableTarget does
|
|
102
|
+
* not accept. The opener runs detached and is not waited for; a failure to
|
|
103
|
+
* start it (no xdg-open installed, say) arrives later through onError.
|
|
104
|
+
*/
|
|
105
|
+
export function openInBrowser(target, deps) {
|
|
106
|
+
if (!openableTarget(target))
|
|
107
|
+
return { ok: false, why: "only the live view's loopback address or an absolute file path is opened" };
|
|
108
|
+
const { command, args } = openerCommand(deps.platform, target);
|
|
109
|
+
try {
|
|
110
|
+
const child = deps.spawn(command, args, { shell: false, stdio: "ignore", detached: true });
|
|
111
|
+
child.on("error", (err) => deps.onError(`${command} could not start: ${err.message}`));
|
|
112
|
+
child.unref();
|
|
113
|
+
return { ok: true };
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
return { ok: false, why: `${command} could not start: ${err instanceof Error ? err.message : String(err)}` };
|
|
117
|
+
}
|
|
118
|
+
}
|
package/dist/engine/oracles.js
CHANGED
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { redactSecrets } from "./memory.js";
|
|
2
|
+
/** Every kind of violation the oracles record. One list, so the report's plain wording can be held equal to it. */
|
|
3
|
+
export const ORACLE_KINDS = [
|
|
4
|
+
"console_error",
|
|
5
|
+
"page_error",
|
|
6
|
+
"request_failed",
|
|
7
|
+
"http_error",
|
|
8
|
+
"dom_injection",
|
|
9
|
+
"refused_empty",
|
|
10
|
+
"false_success",
|
|
11
|
+
"postmessage_token",
|
|
12
|
+
];
|
|
2
13
|
/** How an HTTP error violation states its request. One formatter, so a reader of the detail (the check) cannot drift from its writer. */
|
|
3
14
|
export function httpErrorDetail(method, url, status) {
|
|
4
15
|
return `${method} ${url.slice(0, 200)} → HTTP ${status}`;
|
|
@@ -21,6 +32,19 @@ export function redactViolation(v) {
|
|
|
21
32
|
}
|
|
22
33
|
/** How long after a write-policy block a generic "fetch failed" error is still attributed to it. */
|
|
23
34
|
export const POLICY_BLOCK_WINDOW_MS = 2000;
|
|
35
|
+
/**
|
|
36
|
+
* How a block surfaces in an error's wording: the browser's own lines for a
|
|
37
|
+
* dropped request or the stand-in 403, the status as HTTP clients word their
|
|
38
|
+
* rejection ("Request failed with status code 403", "403 Forbidden"), and the
|
|
39
|
+
* stand-in body's own sentence echoed into an error. A 403 alone with neither
|
|
40
|
+
* word is left out: "Error 403 at line 12" is not the policy.
|
|
41
|
+
*/
|
|
42
|
+
const POLICY_INDUCED_RE = [
|
|
43
|
+
/ERR_BLOCKED_BY_CLIENT|Failed to fetch|NetworkError when attempting to fetch|Load failed|the server responded with a status of 403\b/i,
|
|
44
|
+
/\bstatus(?: code)?:? 403\b/i,
|
|
45
|
+
/\b403\b[^\n]{0,40}\b(?:forbidden|refused)\b|\b(?:forbidden|refused)\b[^\n]{0,40}\b403\b/i,
|
|
46
|
+
/refused by the tester's [a-z-]+ write policy/i,
|
|
47
|
+
];
|
|
24
48
|
/**
|
|
25
49
|
* Is this console or page error a consequence of the tester's OWN write-policy
|
|
26
50
|
* block rather than something the app did?
|
|
@@ -42,7 +66,7 @@ export function isPolicyInduced(v, msSincePolicyBlock) {
|
|
|
42
66
|
return false;
|
|
43
67
|
if (v.kind !== "page_error" && v.kind !== "console_error")
|
|
44
68
|
return false;
|
|
45
|
-
return
|
|
69
|
+
return POLICY_INDUCED_RE.some((re) => re.test(v.detail));
|
|
46
70
|
}
|
|
47
71
|
/**
|
|
48
72
|
* The request a console message is the browser's own echo of, or null when it
|
|
@@ -106,6 +130,78 @@ export class EmbedRequestLog {
|
|
|
106
130
|
return key === null ? null : (this.byKey.get(key) ?? null);
|
|
107
131
|
}
|
|
108
132
|
}
|
|
133
|
+
/** How long after a replay ends the browser's console echo of its failure is still taken for the replay's. */
|
|
134
|
+
export const REPLAY_ECHO_WINDOW_MS = 2000;
|
|
135
|
+
/**
|
|
136
|
+
* Addresses the tester is calling with scout_request, so the browser's console
|
|
137
|
+
* echo of a refused probe ("Failed to load resource: … 403") is known as the
|
|
138
|
+
* tester's and not charged to the page. The request itself is matched by
|
|
139
|
+
* identity in the monitor; the echo carries only an address, so it is matched
|
|
140
|
+
* by address while the replay is in flight and for a short window after, which
|
|
141
|
+
* is when the browser prints it. Bounded, dropping the oldest.
|
|
142
|
+
*/
|
|
143
|
+
export class ReplayLog {
|
|
144
|
+
cap;
|
|
145
|
+
/** Address → until when an echo of it is the replay's (Infinity while in flight). */
|
|
146
|
+
until = new Map();
|
|
147
|
+
constructor(cap = 200) {
|
|
148
|
+
this.cap = cap;
|
|
149
|
+
}
|
|
150
|
+
begin(url) {
|
|
151
|
+
const key = requestKey(url);
|
|
152
|
+
this.until.delete(key);
|
|
153
|
+
this.until.set(key, Infinity);
|
|
154
|
+
if (this.until.size > this.cap)
|
|
155
|
+
this.until.delete(this.until.keys().next().value);
|
|
156
|
+
}
|
|
157
|
+
end(url, now) {
|
|
158
|
+
const key = requestKey(url);
|
|
159
|
+
if (this.until.has(key))
|
|
160
|
+
this.until.set(key, now + REPLAY_ECHO_WINDOW_MS);
|
|
161
|
+
}
|
|
162
|
+
/** Whether a console message is the browser's echo of a replay's failed load. */
|
|
163
|
+
echoes(text, locationUrl, pageUrl, now) {
|
|
164
|
+
const key = failedLoadEchoOf(text, locationUrl, pageUrl);
|
|
165
|
+
if (key === null)
|
|
166
|
+
return false;
|
|
167
|
+
const until = this.until.get(key);
|
|
168
|
+
if (until === undefined)
|
|
169
|
+
return false;
|
|
170
|
+
if (now > until) {
|
|
171
|
+
this.until.delete(key);
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
return true;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Wording of a router cancelling a route change on purpose. Some client-side
|
|
179
|
+
* routers can only stop a navigation (to keep a dirty form, say) by throwing
|
|
180
|
+
* from their route-change event, and the apps that use them filter this
|
|
181
|
+
* sentinel out of their own error monitoring. Narrow on purpose: it must name
|
|
182
|
+
* the route and the cancelling.
|
|
183
|
+
*/
|
|
184
|
+
const ROUTE_CANCEL_RE = /\brout(?:e|ing)\b.{0,40}\b(?:abort|cancel)|\b(?:abort|cancel)\w*\b.{0,40}\brout(?:e|ing)\b/i;
|
|
185
|
+
/**
|
|
186
|
+
* Is this uncaught page error a router deliberately cancelling a route change,
|
|
187
|
+
* rather than a crash? Only for a click that left the URL where it was, and
|
|
188
|
+
* then only when the message says so, or when the click asked for a route
|
|
189
|
+
* change and the page put up a confirmation instead. Still reported, at
|
|
190
|
+
* medium with a note: the error is real and uncaught, but the user never sees
|
|
191
|
+
* it, and ranking it with crashes made every such page argue it away.
|
|
192
|
+
*/
|
|
193
|
+
export function isRouteCancellation(message, c) {
|
|
194
|
+
if (!c.byClick || c.urlChanged)
|
|
195
|
+
return false;
|
|
196
|
+
return ROUTE_CANCEL_RE.test(message) || (c.viaLink && c.dialogOpened);
|
|
197
|
+
}
|
|
198
|
+
/** Appended to a page error read as a route-change cancellation, saying which evidence it was read on. */
|
|
199
|
+
export function routeCancelNote(message) {
|
|
200
|
+
const why = ROUTE_CANCEL_RE.test(message)
|
|
201
|
+
? "the click left the URL unchanged and the error says the route change was cancelled"
|
|
202
|
+
: "the click on a link left the URL unchanged and the page opened a dialog instead";
|
|
203
|
+
return ` (likely a router cancelling the route change on purpose: ${why}; reported at medium, check what the page showed before filing)`;
|
|
204
|
+
}
|
|
109
205
|
/**
|
|
110
206
|
* Invariant oracles: passive listeners that record violations regardless of
|
|
111
207
|
* what the agent is doing. The engine drains the buffer after every action and
|
|
@@ -124,6 +220,11 @@ export class OracleMonitor {
|
|
|
124
220
|
// Benign noise: failed favicon / source map fetches show up as console errors.
|
|
125
221
|
if (/favicon|source map/i.test(text))
|
|
126
222
|
return;
|
|
223
|
+
// The browser's echo of the tester's own scout_request: its answer was in that tool's result.
|
|
224
|
+
if (this.replays.echoes(text, msg.location().url, page.url(), Date.now())) {
|
|
225
|
+
this.replayAttributed += 1;
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
127
228
|
this.record({
|
|
128
229
|
kind: "console_error",
|
|
129
230
|
severity: "high",
|
|
@@ -150,6 +251,10 @@ export class OracleMonitor {
|
|
|
150
251
|
return;
|
|
151
252
|
if (BENIGN_URL_RE.test(req.url()))
|
|
152
253
|
return;
|
|
254
|
+
if (this.isReplay(req)) {
|
|
255
|
+
this.replayAttributed += 1;
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
153
258
|
if (this.refusedByPolicy(req)) {
|
|
154
259
|
this.policyAttributed += 1;
|
|
155
260
|
return;
|
|
@@ -170,6 +275,12 @@ export class OracleMonitor {
|
|
|
170
275
|
// reported here on every page load teaches the driver to ignore http_error.
|
|
171
276
|
if (BENIGN_URL_RE.test(res.url()))
|
|
172
277
|
return;
|
|
278
|
+
// The tester's own scout_request: a probe of a boundary is meant to be
|
|
279
|
+
// refused, and its answer was in that tool's result, not the page's.
|
|
280
|
+
if (this.isReplay(res.request())) {
|
|
281
|
+
this.replayAttributed += 1;
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
173
284
|
// The write policy's own stand-in refusal, matched by request identity
|
|
174
285
|
// like a dropped one: the server never said this.
|
|
175
286
|
if (this.refusedByPolicy(res.request())) {
|
|
@@ -205,6 +316,44 @@ export class OracleMonitor {
|
|
|
205
316
|
* errors look the same.
|
|
206
317
|
*/
|
|
207
318
|
policyAttributed = 0;
|
|
319
|
+
isReplay = () => false;
|
|
320
|
+
replays = new ReplayLog();
|
|
321
|
+
/** Failures of the tester's own scout_request calls, kept out of the page's violations and counted. */
|
|
322
|
+
replayAttributed = 0;
|
|
323
|
+
/**
|
|
324
|
+
* The engine knows which requests are its own scout_request replays, by
|
|
325
|
+
* identity; their failures are the tester's probes, not the page's.
|
|
326
|
+
*/
|
|
327
|
+
setReplayCheck(check) {
|
|
328
|
+
this.isReplay = check;
|
|
329
|
+
}
|
|
330
|
+
/** A scout_request call to this address is starting: the browser's echo of its failure is the tester's. */
|
|
331
|
+
replayStarted(url) {
|
|
332
|
+
this.replays.begin(url);
|
|
333
|
+
}
|
|
334
|
+
/** That call has returned; its echo is still expected for a short window. */
|
|
335
|
+
replayEnded(url) {
|
|
336
|
+
this.replays.end(url, Date.now());
|
|
337
|
+
}
|
|
338
|
+
/** Whether a page error was recorded since `since` (ms) and not yet drained. */
|
|
339
|
+
hasPageErrorSince(since) {
|
|
340
|
+
return this.buffer.some((v) => v.kind === "page_error" && Date.parse(v.at) >= since);
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Re-rank the page errors this click raised that read as a router
|
|
344
|
+
* cancelling a route change (isRouteCancellation). Done before the drain, so
|
|
345
|
+
* the action's result, the session log and the report all see one verdict.
|
|
346
|
+
*/
|
|
347
|
+
downgradeRouteCancellations(since, c) {
|
|
348
|
+
for (const v of this.buffer) {
|
|
349
|
+
if (v.kind !== "page_error" || v.severity !== "high" || Date.parse(v.at) < since)
|
|
350
|
+
continue;
|
|
351
|
+
if (!isRouteCancellation(v.detail, c))
|
|
352
|
+
continue;
|
|
353
|
+
v.severity = "medium";
|
|
354
|
+
v.detail += routeCancelNote(v.detail);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
208
357
|
embedOfRequest = () => null;
|
|
209
358
|
embedRequests = new EmbedRequestLog();
|
|
210
359
|
/**
|
|
@@ -241,7 +390,7 @@ export class OracleMonitor {
|
|
|
241
390
|
* it is reported through here rather than by a page event.
|
|
242
391
|
*/
|
|
243
392
|
noteContradiction(c, url) {
|
|
244
|
-
this.record({ kind: c.kind, severity: "high", detail: c.detail, url });
|
|
393
|
+
this.record({ kind: c.kind, severity: c.severity ?? "high", detail: c.detail, url });
|
|
245
394
|
}
|
|
246
395
|
/**
|
|
247
396
|
* The page posted a token-shaped value with targetOrigin "*" (postmessage.ts).
|
|
@@ -299,6 +448,22 @@ export class OracleMonitor {
|
|
|
299
448
|
return out;
|
|
300
449
|
}
|
|
301
450
|
}
|
|
451
|
+
/**
|
|
452
|
+
* Violations a plan run with onViolation "continue" goes on past: an error
|
|
453
|
+
* status, and the console's echo of one. They say a step's page answered
|
|
454
|
+
* badly, which in a sweep of independent steps (tabs, filters, pages) is a
|
|
455
|
+
* result to list, not a reason the next step cannot run. Anything else (an
|
|
456
|
+
* uncaught exception, a failed request, typed markup coming back, a page
|
|
457
|
+
* contradicting the server, a token posted to any origin) stops a plan either way.
|
|
458
|
+
*/
|
|
459
|
+
export const CONTINUABLE_KINDS = new Set(["http_error", "console_error"]);
|
|
460
|
+
/** Whether these violations, drained after one plan step, end the plan. Repeats of already-reported violations never do. */
|
|
461
|
+
export function planStopsAt(violations, rule) {
|
|
462
|
+
const fresh = violations.filter((v) => !v.repeat);
|
|
463
|
+
if (fresh.length === 0)
|
|
464
|
+
return false;
|
|
465
|
+
return rule === "stop" || fresh.some((v) => !CONTINUABLE_KINDS.has(v.kind));
|
|
466
|
+
}
|
|
302
467
|
export function formatViolations(violations) {
|
|
303
468
|
if (violations.length === 0)
|
|
304
469
|
return "";
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The report in plain words, for a reader who does not read request
|
|
3
|
+
* signatures and route patterns: a QA tester, a product owner, anyone who
|
|
4
|
+
* needs to act on a finding without decoding it first.
|
|
5
|
+
*
|
|
6
|
+
* It is a view over the same findings the technical report prints, never a
|
|
7
|
+
* replacement: each item keeps its technical detail one click away, and the
|
|
8
|
+
* `report` setting chooses which parts a document carries. Everything here is
|
|
9
|
+
* pure string work, so the wording and the step translation are table-tested.
|
|
10
|
+
*/
|
|
11
|
+
import { seenOnOf } from "./memory.js";
|
|
12
|
+
/** Which parts a report carries: the plain view and the technical one ("both"), or one of them. */
|
|
13
|
+
export const REPORT_AUDIENCES = ["both", "qa", "dev"];
|
|
14
|
+
/** Both, so nobody loses the part they rely on unless they ask to. */
|
|
15
|
+
export const DEFAULT_REPORT_AUDIENCE = "both";
|
|
16
|
+
/**
|
|
17
|
+
* The plain wording for every finding category and every oracle kind, in one
|
|
18
|
+
* table. contract-test holds its keys equal to FINDING_CATEGORIES and
|
|
19
|
+
* ORACLE_KINDS, so a new category or oracle cannot ship without its words.
|
|
20
|
+
*/
|
|
21
|
+
export const PLAIN_WORDING = {
|
|
22
|
+
// Finding categories.
|
|
23
|
+
"console-error": { problem: "The page reported an error behind the scenes", expected: "The page works without reporting errors." },
|
|
24
|
+
"page-error": { problem: "Something on the page broke while it was being used", expected: "The page keeps working while it is used." },
|
|
25
|
+
"http-error": { problem: "The server refused or failed a request", expected: "The action completes, or the page says clearly why it could not." },
|
|
26
|
+
network: { problem: "A request to the server did not get through", expected: "The page loads what it needs, or says what is missing." },
|
|
27
|
+
"dead-end": { problem: "A path led nowhere", expected: "Every link and button leads somewhere useful, with a way back." },
|
|
28
|
+
"ux-confusing": { problem: "Something is confusing to use", expected: "It is clear what to do and what happened." },
|
|
29
|
+
"ux-polish": { problem: "Something feels unfinished", expected: "The page looks finished and consistent." },
|
|
30
|
+
visual: { problem: "Something looks wrong on screen", expected: "Everything is visible, readable and in its place." },
|
|
31
|
+
a11y: { problem: "Some people cannot use this part", expected: "Everyone can use it, including with a keyboard or a screen reader." },
|
|
32
|
+
"permission-leak": { problem: "Someone can see or do something they should not", expected: "Each person sees and does only what their role allows." },
|
|
33
|
+
"data-inconsistency": { problem: "The information shown does not add up", expected: "The same information agrees everywhere it is shown." },
|
|
34
|
+
"stale-state": { problem: "The page shows out-of-date information", expected: "The page shows the current state after every change." },
|
|
35
|
+
"data-loss": { problem: "Work or information was lost", expected: "What a person enters is kept." },
|
|
36
|
+
performance: { problem: "Something is too slow", expected: "The page responds without a noticeable wait." },
|
|
37
|
+
security: { problem: "A security weakness", expected: "The app keeps people's data and accounts safe." },
|
|
38
|
+
"missing-testid": { problem: "A control automated tests cannot find reliably", expected: "Every control can be found reliably by automated tests." },
|
|
39
|
+
other: { problem: "Something else is wrong", expected: "The app behaves the way a person using it would expect." },
|
|
40
|
+
// Oracle kinds: what the automatic checks notice on every page.
|
|
41
|
+
console_error: { problem: "The page reported an error behind the scenes", expected: "The page works without reporting errors." },
|
|
42
|
+
page_error: { problem: "Something on the page broke while it was being used", expected: "The page keeps working while it is used." },
|
|
43
|
+
request_failed: { problem: "A request to the server did not get through", expected: "The page loads what it needs, or says what is missing." },
|
|
44
|
+
http_error: { problem: "The server refused or failed a request", expected: "The action completes, or the page says clearly why it could not." },
|
|
45
|
+
dom_injection: { problem: "Text typed into a field came back as part of the page itself", expected: "What a person types is always shown as plain text." },
|
|
46
|
+
refused_empty: {
|
|
47
|
+
problem: "The server refused a request and the page showed nothing instead of an error",
|
|
48
|
+
expected: "When something fails, the page says so.",
|
|
49
|
+
},
|
|
50
|
+
false_success: { problem: "The page said something worked when the server had refused it", expected: "The page reports only what really happened." },
|
|
51
|
+
postmessage_token: { problem: "The page handed a sign-in token to any site that asked", expected: "Sign-in details go only to the app's own pages." },
|
|
52
|
+
};
|
|
53
|
+
/** The words for a category or oracle kind; one the table does not know (an older finding's) reads as "other". */
|
|
54
|
+
export function plainWording(kind) {
|
|
55
|
+
return Object.prototype.hasOwnProperty.call(PLAIN_WORDING, kind) ? PLAIN_WORDING[kind] : PLAIN_WORDING.other;
|
|
56
|
+
}
|
|
57
|
+
/** Severity as its effect on the people using the app. */
|
|
58
|
+
export const IMPACT = { high: "Blocks users", medium: "Annoying", low: "Cosmetic" };
|
|
59
|
+
/** What a control's role is called in a sentence. */
|
|
60
|
+
const ROLE_WORD = {
|
|
61
|
+
button: "button",
|
|
62
|
+
link: "link",
|
|
63
|
+
textbox: "field",
|
|
64
|
+
searchbox: "search box",
|
|
65
|
+
combobox: "list",
|
|
66
|
+
listbox: "list",
|
|
67
|
+
checkbox: "checkbox",
|
|
68
|
+
radio: "option",
|
|
69
|
+
switch: "switch",
|
|
70
|
+
tab: "tab",
|
|
71
|
+
menuitem: "menu item",
|
|
72
|
+
option: "option",
|
|
73
|
+
spinbutton: "field",
|
|
74
|
+
slider: "slider",
|
|
75
|
+
row: "row",
|
|
76
|
+
cell: "cell",
|
|
77
|
+
};
|
|
78
|
+
/** `role "name"` as "the "name" button"; a target that is not that shape is quoted as it is. */
|
|
79
|
+
function control(target) {
|
|
80
|
+
const m = /^(\w+) "(.*)"$/.exec(target.trim());
|
|
81
|
+
if (!m)
|
|
82
|
+
return target.trim();
|
|
83
|
+
const word = ROLE_WORD[m[1]] ?? m[1];
|
|
84
|
+
return m[2] ? `the "${m[2]}" ${word}` : `the unnamed ${word}`;
|
|
85
|
+
}
|
|
86
|
+
/** One action-log step, split into what was done, to what, and on which page. */
|
|
87
|
+
function splitStep(line) {
|
|
88
|
+
const at = line.lastIndexOf(" @ ");
|
|
89
|
+
const head = at >= 0 ? line.slice(0, at) : line;
|
|
90
|
+
const url = at >= 0 ? line.slice(at + 3) : "";
|
|
91
|
+
const space = head.indexOf(" ");
|
|
92
|
+
return space < 0 ? { action: head, target: "", url } : { action: head.slice(0, space), target: head.slice(space + 1), url };
|
|
93
|
+
}
|
|
94
|
+
/** Actions that move to a page, and so start the steps that matter. */
|
|
95
|
+
const OPENS = new Set(["navigate", "crawl"]);
|
|
96
|
+
/**
|
|
97
|
+
* One repro step as a person would do it, or null for a step only the tester
|
|
98
|
+
* takes (a snapshot, a screenshot, an audit, the write policy's own notes).
|
|
99
|
+
*/
|
|
100
|
+
export function plainStep(line) {
|
|
101
|
+
const { action: raw, target, url } = splitStep(line);
|
|
102
|
+
// A plan's step is the same action, run from a list.
|
|
103
|
+
const action = raw.replace(/^plan:/, "");
|
|
104
|
+
if (OPENS.has(action))
|
|
105
|
+
return `Go to ${url || target}`;
|
|
106
|
+
let m;
|
|
107
|
+
if ((m = /^click(?:×(\d+))?$/.exec(action))) {
|
|
108
|
+
const times = Number(m[1] ?? 1);
|
|
109
|
+
const verb = times === 2 ? "Double-click" : times > 2 ? `Click ${times} times on` : "Click";
|
|
110
|
+
return `${verb} ${control(target)}`;
|
|
111
|
+
}
|
|
112
|
+
if (action === "type") {
|
|
113
|
+
const t = /^(\w+ ".*") ← ("(?:[^"\\]|\\.)*")( \+ Enter)?/.exec(target);
|
|
114
|
+
if (!t)
|
|
115
|
+
return `Type into ${control(target)}`;
|
|
116
|
+
return `Type ${t[2]} into ${control(t[1])}${t[3] ? " and press Enter" : ""}`;
|
|
117
|
+
}
|
|
118
|
+
if (action === "select") {
|
|
119
|
+
const s = /^(\w+ ".*") = (.*)$/.exec(target);
|
|
120
|
+
return s ? `Choose "${s[2]}" in ${control(s[1])}` : `Choose an option in ${control(target)}`;
|
|
121
|
+
}
|
|
122
|
+
if (action === "upload") {
|
|
123
|
+
const u = / ← (.+?) \(/.exec(target);
|
|
124
|
+
return u ? `Attach the file "${u[1]}"` : "Attach a file";
|
|
125
|
+
}
|
|
126
|
+
if (action === "hover")
|
|
127
|
+
return `Point at ${control(target)}`;
|
|
128
|
+
if (action === "press")
|
|
129
|
+
return `Press ${target}`;
|
|
130
|
+
if (action === "scroll")
|
|
131
|
+
return target ? `Scroll ${target}` : "Scroll the page";
|
|
132
|
+
if (action === "back")
|
|
133
|
+
return "Go back to the previous page";
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
/** The page a step was on, for the first step when the trace has no navigation of its own. */
|
|
137
|
+
function pageOf(line) {
|
|
138
|
+
return line ? splitStep(line).url : "";
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* A finding's repro trace as numbered steps a person can follow. It starts at
|
|
142
|
+
* the last time the run went to a page, since what came before that was on
|
|
143
|
+
* another page; when it never did, it starts by going to the page the trace
|
|
144
|
+
* began on. Steps only the tester takes are left out.
|
|
145
|
+
*/
|
|
146
|
+
export function plainSteps(repro) {
|
|
147
|
+
let start = -1;
|
|
148
|
+
for (let i = repro.length - 1; i >= 0; i--) {
|
|
149
|
+
if (OPENS.has(splitStep(repro[i]).action.replace(/^plan:/, ""))) {
|
|
150
|
+
start = i;
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const steps = (start >= 0 ? repro.slice(start) : repro).map(plainStep).filter((s) => s !== null);
|
|
155
|
+
if (start < 0) {
|
|
156
|
+
const first = pageOf(repro[0]);
|
|
157
|
+
if (first)
|
|
158
|
+
steps.unshift(`Go to ${first}`);
|
|
159
|
+
}
|
|
160
|
+
return steps;
|
|
161
|
+
}
|
|
162
|
+
/** A relative path inside the run's folder, safe to put in a document: no scheme, no `..`, nothing absolute. */
|
|
163
|
+
export function isSafeRelativePath(p) {
|
|
164
|
+
return /^[A-Za-z0-9._-][A-Za-z0-9._/-]*$/.test(p) && !p.split("/").includes("..") && !p.includes("//");
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* The picture of a finding, as a path relative to the run's folder, or
|
|
168
|
+
* undefined. A finding's own picture comes first: `picture`, a path relative
|
|
169
|
+
* to the run's folder, is the slot a capture taken when a finding is filed
|
|
170
|
+
* fills, and no finding carries one yet. On a recorded run the last frame on
|
|
171
|
+
* screen before the finding was filed stands in. A value that is not a safe relative path is ignored, since the file is
|
|
172
|
+
* read back without a schema and the path goes into a page.
|
|
173
|
+
*/
|
|
174
|
+
export function pictureOf(f, lastFrame) {
|
|
175
|
+
const own = f.picture;
|
|
176
|
+
for (const p of [own, lastFrame])
|
|
177
|
+
if (typeof p === "string" && isSafeRelativePath(p))
|
|
178
|
+
return p;
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
const count = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
182
|
+
/** "a, b and c". */
|
|
183
|
+
function joinAnd(parts) {
|
|
184
|
+
return parts.length <= 1 ? parts.join("") : `${parts.slice(0, -1).join(", ")} and ${parts[parts.length - 1]}`;
|
|
185
|
+
}
|
|
186
|
+
/** The short summary that opens the plain section. */
|
|
187
|
+
export function plainSummary(input) {
|
|
188
|
+
const lines = [];
|
|
189
|
+
const { current } = input;
|
|
190
|
+
const by = (s) => current.filter((f) => f.severity === s).length;
|
|
191
|
+
const where = input.routes && input.routes.total > 0 ? ` It went to ${input.routes.visited} of the ${input.routes.total} pages it knew about.` : "";
|
|
192
|
+
if (current.length === 0) {
|
|
193
|
+
lines.push(`This run found no problems.${where}`);
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
const split = [
|
|
197
|
+
["high", "blocks users", "block users"],
|
|
198
|
+
["medium", "is annoying", "are annoying"],
|
|
199
|
+
["low", "is cosmetic", "are cosmetic"],
|
|
200
|
+
]
|
|
201
|
+
.filter(([s]) => by(s) > 0)
|
|
202
|
+
.map(([s, one, many]) => `${by(s)} ${by(s) === 1 ? one : many}`);
|
|
203
|
+
lines.push(`This run found ${count(current.length, "problem")}: ${joinAnd(split)}.${where}`);
|
|
204
|
+
}
|
|
205
|
+
const elsewhere = input.audience === "qa" ? "" : " They are listed in the technical detail below.";
|
|
206
|
+
if (input.historical > 0) {
|
|
207
|
+
lines.push(``, `${count(input.historical, "problem")} found by earlier runs ${input.historical === 1 ? "was" : "were"} not checked again this time.${elsewhere}`);
|
|
208
|
+
}
|
|
209
|
+
if (input.worthALook > 0) {
|
|
210
|
+
lines.push(``, `${count(input.worthALook, "observation")} ${input.worthALook === 1 ? "is a problem" : "are problems"} only if your project follows a particular convention, so ${input.worthALook === 1 ? "it is" : "they are"} not counted here.${elsewhere}`);
|
|
211
|
+
}
|
|
212
|
+
const kinds = new Map();
|
|
213
|
+
for (const v of input.violations)
|
|
214
|
+
kinds.set(v.kind, (kinds.get(v.kind) ?? 0) + 1);
|
|
215
|
+
if (kinds.size > 0) {
|
|
216
|
+
lines.push(``, `While it worked, the automatic checks noticed:`, ``);
|
|
217
|
+
for (const [kind, n] of [...kinds.entries()].sort((a, b) => b[1] - a[1]))
|
|
218
|
+
lines.push(`- ${plainWording(kind).problem} (${n === 1 ? "once" : `${n} times`})`);
|
|
219
|
+
}
|
|
220
|
+
if (input.gaps.length > 0) {
|
|
221
|
+
lines.push(``, `The run left ${count(input.gaps.length, "thing")} unchecked.${input.audience === "qa" ? "" : " The gap ledger below says which."}`);
|
|
222
|
+
if (input.audience === "qa") {
|
|
223
|
+
lines.push(``);
|
|
224
|
+
for (const g of input.gaps)
|
|
225
|
+
lines.push(`- ${g}`);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return lines;
|
|
229
|
+
}
|
|
230
|
+
/** One finding in plain words, with its technical detail folded beneath it. */
|
|
231
|
+
export function plainFinding(f, index, input) {
|
|
232
|
+
const words = plainWording(f.category);
|
|
233
|
+
// One cause in a shared part of the app is seen on every page that shows it.
|
|
234
|
+
const also = seenOnOf(f);
|
|
235
|
+
const lines = [
|
|
236
|
+
`### ${index}. ${f.title}`,
|
|
237
|
+
``,
|
|
238
|
+
`**${IMPACT[f.severity]}** · ${words.problem} · on ${f.url}${also.length > 0 ? `, and also seen on ${also.join(", ")}` : ""}`,
|
|
239
|
+
``,
|
|
240
|
+
];
|
|
241
|
+
const picture = pictureOf(f, input.lastFrames?.get(f.id));
|
|
242
|
+
if (picture)
|
|
243
|
+
lines.push(``, ``);
|
|
244
|
+
const steps = plainSteps(f.repro);
|
|
245
|
+
if (steps.length > 0) {
|
|
246
|
+
lines.push(`**What was done:**`, ``);
|
|
247
|
+
steps.forEach((s, i) => lines.push(`${i + 1}. ${s}`));
|
|
248
|
+
lines.push(``);
|
|
249
|
+
}
|
|
250
|
+
lines.push(`**What was expected:** ${words.expected}`, ``, `**What happened:** ${f.detail.replace(/\r?\n+/g, " ")}`, ``);
|
|
251
|
+
lines.push(`<details><summary>Technical detail</summary>`, ``);
|
|
252
|
+
// "Finding id", not "Id": the HTML and the live view hang a finding's recorded
|
|
253
|
+
// frames under the list that names its "Id", and that is the full entry's.
|
|
254
|
+
lines.push(`- **Finding id:** \`${f.id}\` · **Category:** ${f.category} · **Severity:** ${f.severity}`);
|
|
255
|
+
if (f.evidence)
|
|
256
|
+
lines.push(`- **Evidence:** \`${f.evidence}\``);
|
|
257
|
+
lines.push(`- **Route:** \`${f.state}\``);
|
|
258
|
+
if (input.audience !== "qa")
|
|
259
|
+
lines.push(`- The full entry, with the repro trace and a test skeleton, is under the same id in the technical detail below.`);
|
|
260
|
+
lines.push(``, `</details>`, ``);
|
|
261
|
+
return lines;
|
|
262
|
+
}
|
|
263
|
+
/** The whole plain section: the summary, then each problem this run found, worst first. */
|
|
264
|
+
export function formatPlainSection(input) {
|
|
265
|
+
const lines = [`## In plain words`, ``, ...plainSummary(input), ``, ...(input.tickets ?? [])];
|
|
266
|
+
input.current.forEach((f, i) => lines.push(...plainFinding(f, i + 1, input)));
|
|
267
|
+
return lines;
|
|
268
|
+
}
|