scenescout 3.10.0 → 3.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +12 -3
- package/dist/browsers.js +64 -0
- package/dist/check-run.js +79 -5
- package/dist/cli.js +32 -6
- package/dist/engine/bench.js +154 -17
- package/dist/engine/browser.js +426 -26
- package/dist/engine/calibration.js +99 -14
- package/dist/engine/check.js +251 -33
- package/dist/engine/fingerprint.js +12 -7
- package/dist/engine/flow.js +398 -0
- package/dist/engine/lane.js +81 -0
- package/dist/engine/memory.js +48 -6
- package/dist/engine/unload.js +201 -0
- package/dist/engine/verify.js +119 -0
- package/dist/mcp-server.js +4 -4
- package/package.json +4 -2
- package/skills/scenescout/SKILL.md +1 -1
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Saved flows: the steps `scenescout check` replays, as data.
|
|
3
|
+
*
|
|
4
|
+
* A flow is a short script a person or an agent wrote down once because it
|
|
5
|
+
* matters that it keeps working: open a page, click a control, see a result.
|
|
6
|
+
* The check replays every flow in `.scenescout/flows/*.json` after its crawl,
|
|
7
|
+
* with no model, so a flow that passed yesterday and breaks today fails the
|
|
8
|
+
* gate naming the flow and the step (ADR 12).
|
|
9
|
+
*
|
|
10
|
+
* The steps are `scout_run_plan`'s steps, so a plan that worked in an
|
|
11
|
+
* exploratory run can be saved as it is, plus three assertions a plan has no
|
|
12
|
+
* need for: text is visible, the URL matches, a request answered with a status.
|
|
13
|
+
*
|
|
14
|
+
* Everything here is pure: the schema and its error messages, the target
|
|
15
|
+
* grammar, and the rules that decide whether a URL or a request matched. The
|
|
16
|
+
* browser half is BrowserEngine.replayFlow, which only acts and reports.
|
|
17
|
+
*/
|
|
18
|
+
import fs from "node:fs";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import { z } from "zod";
|
|
21
|
+
import { beaconResourceType, BROWSER_ENGINES } from "../browsers.js";
|
|
22
|
+
import { MEMORY_DIRNAME, SELF_IGNORE_KEEP } from "./memory.js";
|
|
23
|
+
/** Most steps one flow may hold. A longer flow is several flows, and each would fail more usefully on its own. */
|
|
24
|
+
export const MAX_FLOW_STEPS = 50;
|
|
25
|
+
/** Most flows one check replays. */
|
|
26
|
+
export const MAX_FLOWS = 50;
|
|
27
|
+
/** How long a step waits for its target, text, URL or request before it counts as broken. */
|
|
28
|
+
export const FLOW_STEP_TIMEOUT_MS = 5000;
|
|
29
|
+
/**
|
|
30
|
+
* How long a flow waits after its last step for a write that step set off
|
|
31
|
+
* late (a save on a timer, a debounced autosave), before it counts as passed.
|
|
32
|
+
*/
|
|
33
|
+
export const FLOW_AFTER_LAST_STEP_MS = 750;
|
|
34
|
+
/** Where a project keeps its flows, relative to the project directory. */
|
|
35
|
+
export const FLOWS_SUBDIR = path.join(MEMORY_DIRNAME, SELF_IGNORE_KEEP);
|
|
36
|
+
export const TARGET_HELP = 'testid=…, text=…, label=… or role=<role>[name="…"]';
|
|
37
|
+
/**
|
|
38
|
+
* The ARIA roles a `role=` target may name (WAI-ARIA 1.2, as Playwright's
|
|
39
|
+
* getByRole accepts them). A misspelt role matches nothing, so a flow using
|
|
40
|
+
* one would only ever fail at replay; it is refused when the file is read.
|
|
41
|
+
*/
|
|
42
|
+
export const ARIA_ROLE_LIST = [
|
|
43
|
+
"alert",
|
|
44
|
+
"alertdialog",
|
|
45
|
+
"application",
|
|
46
|
+
"article",
|
|
47
|
+
"banner",
|
|
48
|
+
"blockquote",
|
|
49
|
+
"button",
|
|
50
|
+
"caption",
|
|
51
|
+
"cell",
|
|
52
|
+
"checkbox",
|
|
53
|
+
"code",
|
|
54
|
+
"columnheader",
|
|
55
|
+
"combobox",
|
|
56
|
+
"complementary",
|
|
57
|
+
"contentinfo",
|
|
58
|
+
"definition",
|
|
59
|
+
"deletion",
|
|
60
|
+
"dialog",
|
|
61
|
+
"directory",
|
|
62
|
+
"document",
|
|
63
|
+
"emphasis",
|
|
64
|
+
"feed",
|
|
65
|
+
"figure",
|
|
66
|
+
"form",
|
|
67
|
+
"generic",
|
|
68
|
+
"grid",
|
|
69
|
+
"gridcell",
|
|
70
|
+
"group",
|
|
71
|
+
"heading",
|
|
72
|
+
"img",
|
|
73
|
+
"insertion",
|
|
74
|
+
"link",
|
|
75
|
+
"list",
|
|
76
|
+
"listbox",
|
|
77
|
+
"listitem",
|
|
78
|
+
"log",
|
|
79
|
+
"main",
|
|
80
|
+
"marquee",
|
|
81
|
+
"math",
|
|
82
|
+
"meter",
|
|
83
|
+
"menu",
|
|
84
|
+
"menubar",
|
|
85
|
+
"menuitem",
|
|
86
|
+
"menuitemcheckbox",
|
|
87
|
+
"menuitemradio",
|
|
88
|
+
"navigation",
|
|
89
|
+
"none",
|
|
90
|
+
"note",
|
|
91
|
+
"option",
|
|
92
|
+
"paragraph",
|
|
93
|
+
"presentation",
|
|
94
|
+
"progressbar",
|
|
95
|
+
"radio",
|
|
96
|
+
"radiogroup",
|
|
97
|
+
"region",
|
|
98
|
+
"row",
|
|
99
|
+
"rowgroup",
|
|
100
|
+
"rowheader",
|
|
101
|
+
"scrollbar",
|
|
102
|
+
"search",
|
|
103
|
+
"searchbox",
|
|
104
|
+
"separator",
|
|
105
|
+
"slider",
|
|
106
|
+
"spinbutton",
|
|
107
|
+
"status",
|
|
108
|
+
"strong",
|
|
109
|
+
"subscript",
|
|
110
|
+
"superscript",
|
|
111
|
+
"switch",
|
|
112
|
+
"tab",
|
|
113
|
+
"table",
|
|
114
|
+
"tablist",
|
|
115
|
+
"tabpanel",
|
|
116
|
+
"term",
|
|
117
|
+
"textbox",
|
|
118
|
+
"time",
|
|
119
|
+
"timer",
|
|
120
|
+
"toolbar",
|
|
121
|
+
"tooltip",
|
|
122
|
+
"tree",
|
|
123
|
+
"treegrid",
|
|
124
|
+
"treeitem",
|
|
125
|
+
];
|
|
126
|
+
export const ARIA_ROLES = new Set(ARIA_ROLE_LIST);
|
|
127
|
+
export const ARIA_ROLES_MATCH_PLAYWRIGHT = ARIA_ROLE_LIST;
|
|
128
|
+
/** Parse a target string, or null when it is not one. */
|
|
129
|
+
export function parseTarget(target) {
|
|
130
|
+
for (const by of ["testid", "text", "label"]) {
|
|
131
|
+
if (target.startsWith(`${by}=`)) {
|
|
132
|
+
const value = target.slice(by.length + 1);
|
|
133
|
+
return value ? { by, value } : null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const role = /^role=([a-z]+)(?:\[name=(?:"([^"]*)"|'([^']*)'|([^\]"']*))\])?$/i.exec(target);
|
|
137
|
+
if (!role)
|
|
138
|
+
return null;
|
|
139
|
+
const name = role[2] ?? role[3] ?? role[4];
|
|
140
|
+
return name === undefined ? { by: "role", role: role[1].toLowerCase() } : { by: "role", role: role[1].toLowerCase(), name };
|
|
141
|
+
}
|
|
142
|
+
const REQUEST_RE = /^(GET|POST|PUT|PATCH|DELETE|HEAD) (\/\S*)$/;
|
|
143
|
+
function isRegex(pattern) {
|
|
144
|
+
try {
|
|
145
|
+
new RegExp(pattern);
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const target = z
|
|
153
|
+
.string()
|
|
154
|
+
.refine((t) => parseTarget(t) !== null, { message: `must be ${TARGET_HELP}` })
|
|
155
|
+
.superRefine((t, ctx) => {
|
|
156
|
+
const parsed = parseTarget(t);
|
|
157
|
+
if (parsed?.by === "role" && !ARIA_ROLES.has(parsed.role)) {
|
|
158
|
+
ctx.addIssue({
|
|
159
|
+
code: z.ZodIssueCode.custom,
|
|
160
|
+
message: `names the role "${parsed.role}", which is not an ARIA role (e.g. button, link, textbox, checkbox, tab)`,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
const STEP_SCHEMAS = [
|
|
165
|
+
z.object({ action: z.literal("navigate"), target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
|
|
166
|
+
z.object({ action: z.literal("click"), target }).strict(),
|
|
167
|
+
z.object({ action: z.literal("type"), target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
|
|
168
|
+
z.object({ action: z.literal("select"), target, value: z.string() }).strict(),
|
|
169
|
+
z.object({ action: z.literal("press"), value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
|
|
170
|
+
z.object({ action: z.literal("expect-text"), text: z.string().min(1, "is the text that must be visible") }).strict(),
|
|
171
|
+
z.object({ action: z.literal("expect-url"), pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) }).strict(),
|
|
172
|
+
z
|
|
173
|
+
.object({
|
|
174
|
+
action: z.literal("expect-request"),
|
|
175
|
+
request: z.string().regex(REQUEST_RE, 'must be a method and a path, e.g. "GET /api/things"'),
|
|
176
|
+
status: z.union([z.number().int().min(100).max(599), z.string().regex(/^[1-5]xx$/, 'must be a status such as 200, or a class such as "2xx"')]),
|
|
177
|
+
})
|
|
178
|
+
.strict(),
|
|
179
|
+
];
|
|
180
|
+
const stepSchema = z.discriminatedUnion("action", [...STEP_SCHEMAS]);
|
|
181
|
+
export const FLOW_ACTIONS = STEP_SCHEMAS.map((s) => s.shape.action.value);
|
|
182
|
+
const flowSchema = z
|
|
183
|
+
.object({
|
|
184
|
+
name: z.string().min(1).max(100).optional(),
|
|
185
|
+
description: z.string().max(500).optional(),
|
|
186
|
+
steps: z.array(stepSchema).min(1, "needs at least one step").max(MAX_FLOW_STEPS, `holds at most ${MAX_FLOW_STEPS} steps`),
|
|
187
|
+
})
|
|
188
|
+
.strict()
|
|
189
|
+
.superRefine((flow, ctx) => {
|
|
190
|
+
// A flow starts from a page it names, never from wherever the crawl left the browser.
|
|
191
|
+
if (flow.steps[0] && flow.steps[0].action !== "navigate") {
|
|
192
|
+
ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["steps", 0, "action"], message: 'must be "navigate": a flow starts from a page it names' });
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
/** `steps[2].target`, the way a person would look for it in the file. */
|
|
196
|
+
function fieldPath(p) {
|
|
197
|
+
if (p.length === 0)
|
|
198
|
+
return "(the whole file)";
|
|
199
|
+
return p.map((part, i) => (typeof part === "number" ? `[${part}]` : i === 0 ? part : `.${part}`)).join("");
|
|
200
|
+
}
|
|
201
|
+
function issueMessage(issue) {
|
|
202
|
+
if (issue.code === "unrecognized_keys")
|
|
203
|
+
return `unknown field(s) ${issue.keys.map((k) => `"${k}"`).join(", ")}`;
|
|
204
|
+
if (issue.code === "invalid_union_discriminator")
|
|
205
|
+
return `must be one of ${FLOW_ACTIONS.join(", ")}`;
|
|
206
|
+
if (issue.code === "invalid_type" && issue.received === "undefined")
|
|
207
|
+
return "is required";
|
|
208
|
+
return issue.message;
|
|
209
|
+
}
|
|
210
|
+
/** Validate one flow file's text. Every mistake names the file and the field. */
|
|
211
|
+
export function parseFlow(text, file) {
|
|
212
|
+
let json;
|
|
213
|
+
try {
|
|
214
|
+
json = JSON.parse(text);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
return { ok: false, error: `${file}: not valid JSON (${err instanceof Error ? err.message : String(err)})` };
|
|
218
|
+
}
|
|
219
|
+
const parsed = flowSchema.safeParse(json);
|
|
220
|
+
if (!parsed.success) {
|
|
221
|
+
const issues = parsed.error.issues.slice(0, 5).map((i) => `${fieldPath(i.path)} ${issueMessage(i)}`);
|
|
222
|
+
const more = parsed.error.issues.length > 5 ? ` (and ${parsed.error.issues.length - 5} more)` : "";
|
|
223
|
+
return { ok: false, error: `${file}: ${issues.join("; ")}${more}` };
|
|
224
|
+
}
|
|
225
|
+
return { ok: true, flow: { name: parsed.data.name ?? file.replace(/\.json$/i, ""), file, steps: parsed.data.steps } };
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Read every `*.json` in a flows directory, in name order. Throws on the first
|
|
229
|
+
* file that is not a valid flow: a flow that cannot be read would otherwise
|
|
230
|
+
* pass by never running. Everything else in the directory is returned as
|
|
231
|
+
* skipped with its reason, so nothing there is left out without saying so. A
|
|
232
|
+
* symbolic link is followed when it resolves to a file inside the directory,
|
|
233
|
+
* and skipped when it resolves anywhere else.
|
|
234
|
+
*/
|
|
235
|
+
export function loadFlows(dir) {
|
|
236
|
+
const realDir = fs.realpathSync(dir);
|
|
237
|
+
const files = [];
|
|
238
|
+
const skipped = [];
|
|
239
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true }).sort((x, y) => (x.name < y.name ? -1 : x.name > y.name ? 1 : 0))) {
|
|
240
|
+
if (e.isDirectory()) {
|
|
241
|
+
skipped.push({ file: e.name, reason: "a directory: flows are not read from subdirectories" });
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
if (!e.name.toLowerCase().endsWith(".json")) {
|
|
245
|
+
skipped.push({ file: e.name, reason: "not a .json file" });
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
if (e.isSymbolicLink()) {
|
|
249
|
+
let target;
|
|
250
|
+
try {
|
|
251
|
+
target = fs.realpathSync(path.join(dir, e.name));
|
|
252
|
+
}
|
|
253
|
+
catch {
|
|
254
|
+
skipped.push({ file: e.name, reason: "a symbolic link to nothing" });
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (!target.startsWith(realDir + path.sep)) {
|
|
258
|
+
skipped.push({ file: e.name, reason: "a symbolic link to a file outside the flows directory" });
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
if (!fs.statSync(target).isFile()) {
|
|
262
|
+
skipped.push({ file: e.name, reason: "a symbolic link to something that is not a file" });
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
else if (!e.isFile()) {
|
|
267
|
+
skipped.push({ file: e.name, reason: "not a regular file" });
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
files.push(e.name);
|
|
271
|
+
}
|
|
272
|
+
if (files.length > MAX_FLOWS)
|
|
273
|
+
throw new Error(`${dir} holds ${files.length} flows; a check replays at most ${MAX_FLOWS}`);
|
|
274
|
+
const flows = files.map((file) => {
|
|
275
|
+
const parsed = parseFlow(fs.readFileSync(path.join(dir, file), "utf8"), file);
|
|
276
|
+
if (!parsed.ok)
|
|
277
|
+
throw new Error(`flow ${parsed.error}`);
|
|
278
|
+
return parsed.flow;
|
|
279
|
+
});
|
|
280
|
+
return { flows, skipped };
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Which flows directory a check uses: the one named, none for "off", and
|
|
284
|
+
* otherwise the project's own when it exists. A directory that was named and
|
|
285
|
+
* is missing is an error, never a quiet check with no flows.
|
|
286
|
+
*/
|
|
287
|
+
export function resolveFlowsDir(flag, projectDir, exists) {
|
|
288
|
+
if (flag === "off")
|
|
289
|
+
return { dir: null };
|
|
290
|
+
if (flag !== undefined)
|
|
291
|
+
return exists(flag) ? { dir: flag } : { error: `--flows: no directory at ${flag}` };
|
|
292
|
+
const own = path.join(projectDir, FLOWS_SUBDIR);
|
|
293
|
+
return { dir: exists(own) ? own : null };
|
|
294
|
+
}
|
|
295
|
+
/** A step as the report names it. */
|
|
296
|
+
export function describeStep(step) {
|
|
297
|
+
switch (step.action) {
|
|
298
|
+
case "navigate":
|
|
299
|
+
case "click":
|
|
300
|
+
case "select":
|
|
301
|
+
return `${step.action} ${step.target}`;
|
|
302
|
+
case "type":
|
|
303
|
+
return `type into ${step.target}${step.pressEnter ? " and press Enter" : ""}`;
|
|
304
|
+
case "press":
|
|
305
|
+
return `press ${step.value}`;
|
|
306
|
+
case "expect-text":
|
|
307
|
+
return `expect text ${JSON.stringify(step.text)}`;
|
|
308
|
+
case "expect-url":
|
|
309
|
+
return `expect the URL to match /${step.pattern}/`;
|
|
310
|
+
case "expect-request":
|
|
311
|
+
return `expect ${step.request} → ${step.status}`;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
/** Whether a step acts on the page (and so opens a new window for expect-request) rather than reading it. */
|
|
315
|
+
export function isAction(step) {
|
|
316
|
+
return !step.action.startsWith("expect-");
|
|
317
|
+
}
|
|
318
|
+
/** Path, query and hash: what an expect-url pattern is tested against, so the same flow passes on a laptop and a preview deployment. */
|
|
319
|
+
export function urlMatches(pattern, url) {
|
|
320
|
+
let tail;
|
|
321
|
+
try {
|
|
322
|
+
const u = new URL(url);
|
|
323
|
+
tail = `${u.pathname}${u.search}${u.hash}`;
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
tail = url;
|
|
327
|
+
}
|
|
328
|
+
return new RegExp(pattern).test(tail);
|
|
329
|
+
}
|
|
330
|
+
/** A path pattern against a request's path: equal segment for segment, where `*` stands for any one segment. A trailing slash is ignored. */
|
|
331
|
+
export function requestPathMatches(pattern, pathname) {
|
|
332
|
+
const split = (p) => p.replace(/\/+$/, "").split("/");
|
|
333
|
+
const want = split(pattern);
|
|
334
|
+
const got = split(pathname);
|
|
335
|
+
return want.length === got.length && want.every((seg, i) => seg === "*" || seg === got[i]);
|
|
336
|
+
}
|
|
337
|
+
export function statusMatches(expected, status) {
|
|
338
|
+
return typeof expected === "number" ? status === expected : String(status)[0] === expected[0];
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Whether the requests seen since the last action include the one a step
|
|
342
|
+
* expects. The reason says what was seen instead, which is what a person
|
|
343
|
+
* fixing it needs: the right request with the wrong status, or none at all.
|
|
344
|
+
*/
|
|
345
|
+
export function matchRequest(expect, seen) {
|
|
346
|
+
const [, method, pattern] = REQUEST_RE.exec(expect.request) ?? [];
|
|
347
|
+
const same = seen.filter((r) => {
|
|
348
|
+
if (r.method.toUpperCase() !== method)
|
|
349
|
+
return false;
|
|
350
|
+
try {
|
|
351
|
+
return requestPathMatches(pattern, new URL(r.url).pathname);
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
if (same.some((r) => statusMatches(expect.status, r.status)))
|
|
358
|
+
return { ok: true };
|
|
359
|
+
if (same.length > 0) {
|
|
360
|
+
const statuses = [...new Set(same.map((r) => r.status))].join(", ");
|
|
361
|
+
return { ok: false, reason: `${expect.request} answered ${statuses}, expected ${expect.status}` };
|
|
362
|
+
}
|
|
363
|
+
return { ok: false, reason: `no ${expect.request} request was sent since the last action` };
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* The request kinds a refused write may be excused from a step for: what
|
|
367
|
+
* navigator.sendBeacon and <a ping> send, under every engine's name for it
|
|
368
|
+
* (browsers.ts `beaconResourceType`). They are fire-and-forget by design, a
|
|
369
|
+
* page sends them on its own schedule, and no step waits for one.
|
|
370
|
+
*/
|
|
371
|
+
export const BACKGROUND_REQUEST_TYPES = new Set(["ping", ...BROWSER_ENGINES.map(beaconResourceType)]);
|
|
372
|
+
/**
|
|
373
|
+
* Split what the write policy refused during a step into what the step is
|
|
374
|
+
* charged with and background requests, which are only listed. The rule is
|
|
375
|
+
* the request's kind, never its origin: an app's API is often on another port
|
|
376
|
+
* or subdomain, and deciding by origin either hid a step's own write or
|
|
377
|
+
* blamed one on the app. So a fetch, an XHR or a form post is charged to the
|
|
378
|
+
* step wherever it goes, telemetry sent with fetch included; only a beacon or
|
|
379
|
+
* a ping is excused. A refusal with no kind recorded is charged. The app's
|
|
380
|
+
* origin is dropped from what is charged, for a shorter reason.
|
|
381
|
+
*/
|
|
382
|
+
export function splitRefusals(refusals, appOrigin) {
|
|
383
|
+
const charged = [];
|
|
384
|
+
const background = [];
|
|
385
|
+
for (const { sig, type } of refusals) {
|
|
386
|
+
if (type !== undefined && BACKGROUND_REQUEST_TYPES.has(type))
|
|
387
|
+
background.push(sig);
|
|
388
|
+
else
|
|
389
|
+
charged.push(sig.split(appOrigin).join(""));
|
|
390
|
+
}
|
|
391
|
+
return { charged, background };
|
|
392
|
+
}
|
|
393
|
+
/** The step a run stopped at, as the evidence of its issue or the reason the check could not run. */
|
|
394
|
+
export function flowStepEvidence(run) {
|
|
395
|
+
if (run.outcome.status === "passed")
|
|
396
|
+
return "";
|
|
397
|
+
return `flow "${run.name}" (${run.file}) step ${run.outcome.step} of ${run.steps}, ${run.outcome.did}: ${run.outcome.reason}`;
|
|
398
|
+
}
|
package/dist/engine/lane.js
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
* Pure logic, no browser, so every rule here is table-tested.
|
|
24
24
|
*/
|
|
25
25
|
import { z } from "zod";
|
|
26
|
+
import { isIdSegment } from "./fingerprint.js";
|
|
26
27
|
import { FINDING_CATEGORIES } from "./memory.js";
|
|
27
28
|
/** Upper bound on the decisions and routes arrays, so a report is always small enough to hold whole. */
|
|
28
29
|
export const LANE_MAX_ITEMS = 255;
|
|
@@ -271,6 +272,86 @@ export function summarizeLaneReport(r) {
|
|
|
271
272
|
parts.push(`blocked by ${r.blocked_by}`);
|
|
272
273
|
return `${r.lane}: ${parts.join(", ")}`;
|
|
273
274
|
}
|
|
275
|
+
/**
|
|
276
|
+
* Method, optional origin, path (template characters included), optional query
|
|
277
|
+
* and fragment, then a status only when it comes next: after an arrow or a
|
|
278
|
+
* colon, or after spaces, and nothing else. A status further along
|
|
279
|
+
* ("... (expected 403)") is a different claim about the request and is not
|
|
280
|
+
* read as its status.
|
|
281
|
+
*
|
|
282
|
+
* The separator is two alternatives rather than `\s*(?:->|:)?\s*`: with the
|
|
283
|
+
* arrow optional, two adjacent `\s*` can split a run of spaces every way
|
|
284
|
+
* there is, and evidence ending in a long run of whitespace took quadratic
|
|
285
|
+
* time to reject.
|
|
286
|
+
*/
|
|
287
|
+
const STATED_REQUEST_RE = /\b(GET|POST|PUT|PATCH|DELETE)\s+(?:https?:\/\/[^/\s]+)?(\/[A-Za-z0-9/_.:{}$*~%-]*)(?:\?[^\s#]*)?(?:#\S*)?(?:(?:\s*(?:->|→|=>|:)\s*|\s+)([1-5]\d{2})\b)?/gi;
|
|
288
|
+
/** The requests a piece of evidence names, in order. */
|
|
289
|
+
export function statedRequests(text) {
|
|
290
|
+
const out = [];
|
|
291
|
+
for (const m of text.matchAll(STATED_REQUEST_RE)) {
|
|
292
|
+
const raw = m[2].replace(/[.:,]+$/, "");
|
|
293
|
+
const start = (m.index ?? 0) + m[0].indexOf(m[2]);
|
|
294
|
+
out.push({ method: m[1].toUpperCase(), segments: pathSegments(raw), status: m[3] ?? null, start, end: start + raw.length });
|
|
295
|
+
}
|
|
296
|
+
return out;
|
|
297
|
+
}
|
|
298
|
+
function pathSegments(path) {
|
|
299
|
+
const trimmed = path.replace(/\/+$/, "");
|
|
300
|
+
if (trimmed === "")
|
|
301
|
+
return [];
|
|
302
|
+
return trimmed
|
|
303
|
+
.slice(1)
|
|
304
|
+
.split("/")
|
|
305
|
+
.map((seg) => {
|
|
306
|
+
try {
|
|
307
|
+
return decodeURIComponent(seg).toLowerCase();
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
return seg.toLowerCase();
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* A path-template segment: `{name}`, `:name` or `*`. Whole segments only —
|
|
316
|
+
* `order-{id}` is a literal, and so is `**`, which some routers read as "any
|
|
317
|
+
* number of segments": a template here stands for exactly one.
|
|
318
|
+
*/
|
|
319
|
+
export function isTemplateSegment(seg) {
|
|
320
|
+
return seg === "*" || /^\{[^/{}]+\}$/.test(seg) || /^:[a-z_][a-z0-9_-]*$/i.test(seg);
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Whether two paths name the same route. A template segment on either side
|
|
324
|
+
* stands for ONE segment that is an id by route identity's own rule
|
|
325
|
+
* (`isIdSegment`: a number after the first segment, a UUID, a long hex
|
|
326
|
+
* string), or for another template. Never for a word: `/users/{id}` is not
|
|
327
|
+
* `/users/me`, `/things/:id` is not `/things/export`, and a path of three stars does
|
|
328
|
+
* not stand for every three-segment path. Same number of segments, every literal
|
|
329
|
+
* equal (case-insensitive), so a template never absorbs a slash or an empty
|
|
330
|
+
* segment.
|
|
331
|
+
*/
|
|
332
|
+
export function templatedPathsMatch(a, b) {
|
|
333
|
+
if (a.length !== b.length)
|
|
334
|
+
return false;
|
|
335
|
+
// +1: segments start at the first real one; isIdSegment counts as split("/") does.
|
|
336
|
+
const standsFor = (seg, i) => isTemplateSegment(seg) || isIdSegment(seg, i + 1);
|
|
337
|
+
return a.every((x, i) => {
|
|
338
|
+
const y = b[i];
|
|
339
|
+
if (isTemplateSegment(x))
|
|
340
|
+
return standsFor(y, i);
|
|
341
|
+
if (isTemplateSegment(y))
|
|
342
|
+
return standsFor(x, i);
|
|
343
|
+
return x === y;
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Whether two stated requests are the same call: same method, same route with
|
|
348
|
+
* a template read as an id (see templatedPathsMatch), and the same status — both stated and equal, or
|
|
349
|
+
* both unstated. A different status is a different outcome of the call, and
|
|
350
|
+
* pairing a 200 with a 403 would file one claim under its opposite.
|
|
351
|
+
*/
|
|
352
|
+
export function requestsPair(a, b) {
|
|
353
|
+
return a.method === b.method && a.status === b.status && templatedPathsMatch(a.segments, b.segments);
|
|
354
|
+
}
|
|
274
355
|
/**
|
|
275
356
|
* Which sessions are lanes of a parallel run, and whether each one's report
|
|
276
357
|
* has been folded, so a lane's session is not closed before its decisions
|
package/dist/engine/memory.js
CHANGED
|
@@ -451,6 +451,14 @@ export function requestsDisagree(a, b) {
|
|
|
451
451
|
* refused save is data-loss to one and data-inconsistency to another. Across
|
|
452
452
|
* families — a layout defect and a data defect — a shared quoted string names
|
|
453
453
|
* a place on the page, not a bug.
|
|
454
|
+
*
|
|
455
|
+
* The presentation kinds are each a family of their own. They are not
|
|
456
|
+
* alternative labels for one observation: one element is routinely wrong in
|
|
457
|
+
* several independent ways at once — clipped out of view (visual), styled like
|
|
458
|
+
* body text (ux-polish), unnamed to a screen reader (a11y), missing its test id
|
|
459
|
+
* — and the findings about it quote the same control name and name the same
|
|
460
|
+
* test id. As one family, a link "styled like body text" was merged into the
|
|
461
|
+
* finding that the same link was clipped, and the second finding was lost.
|
|
454
462
|
*/
|
|
455
463
|
const CATEGORY_FAMILY = {
|
|
456
464
|
"data-inconsistency": "data",
|
|
@@ -460,10 +468,10 @@ const CATEGORY_FAMILY = {
|
|
|
460
468
|
network: "failure",
|
|
461
469
|
"console-error": "failure",
|
|
462
470
|
"page-error": "failure",
|
|
463
|
-
visual: "
|
|
464
|
-
"ux-polish": "
|
|
465
|
-
a11y: "
|
|
466
|
-
"missing-testid": "
|
|
471
|
+
visual: "visual",
|
|
472
|
+
"ux-polish": "ux-polish",
|
|
473
|
+
a11y: "a11y",
|
|
474
|
+
"missing-testid": "missing-testid",
|
|
467
475
|
"ux-confusing": "flow",
|
|
468
476
|
"dead-end": "flow",
|
|
469
477
|
security: "security",
|
|
@@ -485,7 +493,30 @@ export function sameFamily(a, b) {
|
|
|
485
493
|
const fa = CATEGORY_FAMILY[a];
|
|
486
494
|
return fa !== undefined && fa === CATEGORY_FAMILY[b];
|
|
487
495
|
}
|
|
496
|
+
/**
|
|
497
|
+
* Whether two findings may be one defect by KIND — what is wrong, as opposed
|
|
498
|
+
* to where. The same category, or two categories of one family. Every
|
|
499
|
+
* route-scoped rule below asks this first: evidence, a quoted literal and a
|
|
500
|
+
* title all say where a defect is as often as what it is, so none of them may
|
|
501
|
+
* merge a finding of one kind into a finding of another.
|
|
502
|
+
*/
|
|
503
|
+
function sameKind(a, b) {
|
|
504
|
+
return (!!a && a === b) || sameFamily(a, b);
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* The categories a finding of `category` can merge with, in list order —
|
|
508
|
+
* so scout_finding can tell a lane whose filing was merged which kinds would
|
|
509
|
+
* have been kept apart. An unknown category merges only with itself.
|
|
510
|
+
*/
|
|
511
|
+
export function mergeableCategories(category) {
|
|
512
|
+
const known = FINDING_CATEGORIES.filter((c) => sameKind(category, c));
|
|
513
|
+
return known.length > 0 ? known : [category];
|
|
514
|
+
}
|
|
488
515
|
function sameFinding(a, b) {
|
|
516
|
+
// What is wrong before where: two kinds of defect are two findings, however
|
|
517
|
+
// alike their evidence, quoted text or titles.
|
|
518
|
+
if (!sameKind(a.category, b.category))
|
|
519
|
+
return false;
|
|
489
520
|
const aEv = a.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
|
|
490
521
|
const bEv = b.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
|
|
491
522
|
if (aEv && bEv && aEv === bEv)
|
|
@@ -506,7 +537,9 @@ function sameFinding(a, b) {
|
|
|
506
537
|
// literal with the data defect describing what the button does — in its
|
|
507
538
|
// detail or its own title. Merged, the layout defect was filed and then lost
|
|
508
539
|
// from the report on most runs of a benchmark. Within a family, one bug
|
|
509
|
-
// filed twice under neighbouring categories still merges.
|
|
540
|
+
// filed twice under neighbouring categories still merges. Stricter than the
|
|
541
|
+
// kind check above in one way: an unknown category read from an older file
|
|
542
|
+
// is never merged on a quoted string, even with itself.
|
|
510
543
|
//
|
|
511
544
|
// And never when both findings' evidence names requests with no endpoint in
|
|
512
545
|
// common. A quoted control name bridges two findings about that control
|
|
@@ -541,6 +574,8 @@ function sameFinding(a, b) {
|
|
|
541
574
|
}
|
|
542
575
|
/** Per-project directory for memory, session logs, and reports. */
|
|
543
576
|
export const MEMORY_DIRNAME = ".scenescout";
|
|
577
|
+
/** The one directory under it that is committed rather than ignored: saved flows (flow.ts). */
|
|
578
|
+
export const SELF_IGNORE_KEEP = "flows";
|
|
544
579
|
/** What the directory was called before the tool was renamed. */
|
|
545
580
|
export const LEGACY_MEMORY_DIRNAME = ".scenecraft";
|
|
546
581
|
/**
|
|
@@ -611,6 +646,10 @@ function liveOwnerPid(dir, isAlive) {
|
|
|
611
646
|
* `git status`. That gets the same result without editing a file we do not
|
|
612
647
|
* own, and it disappears cleanly when someone deletes the directory.
|
|
613
648
|
*
|
|
649
|
+
* The one exception is `flows/`: saved flows are inputs a team commits so
|
|
650
|
+
* CI replays them, not output. A file written before that exception existed
|
|
651
|
+
* is still left alone; docs/ci.md says which two lines to add to it.
|
|
652
|
+
*
|
|
614
653
|
* Best-effort and idempotent — a read-only checkout must never break attach,
|
|
615
654
|
* and an existing file is left exactly as the user left it.
|
|
616
655
|
*/
|
|
@@ -621,7 +660,10 @@ export function writeSelfIgnore(dir) {
|
|
|
621
660
|
return null;
|
|
622
661
|
fs.writeFileSync(ignorePath, "# SceneScout exploratory-test artifacts: memory, session logs, status, report.\n" +
|
|
623
662
|
"# Self-ignoring so a `git add -A` in the tested project can never commit them.\n" +
|
|
624
|
-
"*\n"
|
|
663
|
+
"*\n" +
|
|
664
|
+
"# Except the flows `scenescout check` replays: they are written to be committed.\n" +
|
|
665
|
+
`!${SELF_IGNORE_KEEP}/\n` +
|
|
666
|
+
`!${SELF_IGNORE_KEEP}/*.json\n`);
|
|
625
667
|
return `Wrote ${MEMORY_DIRNAME}/.gitignore so these test artifacts stay out of your commits.`;
|
|
626
668
|
}
|
|
627
669
|
catch {
|