scenescout 3.19.2 → 3.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +4 -1
- package/dist/check-run.js +192 -2
- package/dist/cli.js +30 -4
- package/dist/engine/browser.js +71 -8
- package/dist/engine/check-replay.js +315 -0
- package/dist/engine/check.js +24 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# scenescout
|
|
2
2
|
|
|
3
|
+
## 3.20.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 5d5240e: `scenescout check --video` now films each saved flow on the session's own page, so sessionStorage one flow writes is there for the next, and only the flows are filmed, not the crawl. The replay page marks and counts every step and visit left without a frame because its session reached the frame cap, has no scripts or inline event handlers, and carries a generator mark: a check removes only a `replay.html` that has it, so a file of that name the project keeps in the output folder stays.
|
|
8
|
+
|
|
9
|
+
## 3.20.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- eb79ce8: `scenescout check --record` (or `SCENESCOUT_RECORD=on`) keeps a frame after each route visit and each saved-flow step and writes `replay.html` beside the report: each role, each journey with a pass or fail badge, each step with its caption, result and frame, the first failing step highlighted, and the run's version, times, origin and commit in the header. `--video` records a WebM of each saved flow, each on a page of its own, and the replay page plays it beside the journey's steps. The GitHub Action gains `record` and `video` inputs and a `replay` output, and keeps the page, its frames and its videos in the uploaded artifact.
|
|
14
|
+
|
|
3
15
|
## 3.19.2
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -277,7 +277,9 @@ Use SceneScout to test http://localhost:3000, record the run
|
|
|
277
277
|
```
|
|
278
278
|
|
|
279
279
|
or, on the tool directly, `scout_attach {record: true}`. `SCENESCOUT_RECORD=on` in
|
|
280
|
-
the server's environment records every run.
|
|
280
|
+
the server's environment records every run. A CI gate records too:
|
|
281
|
+
`scenescout check --record` writes `replay.html`, every journey step by step with
|
|
282
|
+
its frames ([recording a check](docs/ci.md#recording-a-check)).
|
|
281
283
|
|
|
282
284
|
Then `scout_report` writes two files side by side in `.scenescout/`:
|
|
283
285
|
`report.md` as always, and `report.html` — the whole run as one self-contained
|
|
@@ -454,6 +456,7 @@ With the default settings its saved flows send no HTTP write (they replay under
|
|
|
454
456
|
- `--on-refused-step report|stop` (default `report`): `report` marks a flow whose step was refused "could not run", keeps every other verdict and exits 2; `stop` exits 2 at that step with no results.
|
|
455
457
|
- `--gate-retests never|high|all` (default `high`): which still-reproducing re-tested findings fail the gate.
|
|
456
458
|
- `--baseline off|compare|update` (default `off`), with `--baselines <dir>` and `--baseline-threshold <percent>` (default `0.1`, so small anti-aliasing noise between machines passes): visual baselines, below.
|
|
459
|
+
- `--record` (or `SCENESCOUT_RECORD=on`; the action's `record: on`) keeps a frame after each route visit and each saved-flow step and writes `replay.html` beside the report: each role, each journey with a pass or fail badge, each step with its caption, result and frame, the first failing step highlighted. `--video` (the action's `video: on`) adds a WebM of each journey, played on that page beside its steps. Both off by default. [docs/ci.md](docs/ci.md#recording-a-check) covers size, privacy and publishing it.
|
|
457
460
|
|
|
458
461
|
The defaults are what an unconfigured check does, for a first try or an AI agent running it unattended: its flows send no HTTP write and it never silently hides a result. Each setting is a choice for the project; the report and `check.json` print the values a check ran with.
|
|
459
462
|
|
package/dist/check-run.js
CHANGED
|
@@ -19,6 +19,7 @@ import { decodePng, encodePng } from "./engine/png.js";
|
|
|
19
19
|
import { firstLineOf } from "./engine/limits.js";
|
|
20
20
|
import { isNonPageResource } from "./engine/crawl.js";
|
|
21
21
|
import { checkRetestPlan, retestResults, wellFormedFindings } from "./engine/verify.js";
|
|
22
|
+
import { capFrames, isReplayFrameFile, journeyOf, journeyVideoPath, isGeneratedReplay, isJourneyVideoFile, redactReplay, REPLAY_VIDEOS_DIRNAME, replayVideos, REPLAY_FILE, REPLAY_FRAMES_DIRNAME, replayFramePath, replayFrames, replaySessionKey, visitOf, } from "./engine/check-replay.js";
|
|
22
23
|
/** The baselines folder a check uses: the one --baselines names, else the project's own. */
|
|
23
24
|
export function baselinesDirOf(options) {
|
|
24
25
|
return options.baselinesDir ?? defaultBaselinesDir(options.projectDir);
|
|
@@ -99,6 +100,22 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
99
100
|
const engine = new BrowserEngine();
|
|
100
101
|
/** One browser per role a flow runs as, each attached once, on first use, and closed with the check's own. */
|
|
101
102
|
const roleEngines = new Map();
|
|
103
|
+
// --record: every browser keeps a frame after each step, under its own session name, in its own memory folder.
|
|
104
|
+
const record = options.record === true;
|
|
105
|
+
const startedAt = new Date().toISOString();
|
|
106
|
+
const store = new MemoryStore(scratch);
|
|
107
|
+
if (record)
|
|
108
|
+
engine.sessionKey = replaySessionKey();
|
|
109
|
+
// --video: each flow is filmed on its session's page, from its first step to its hand-back, and saved beside the report.
|
|
110
|
+
const video = options.video === true;
|
|
111
|
+
const frameCap = options.maxFrames;
|
|
112
|
+
const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
|
|
113
|
+
// The default folder is inside .scenescout/, which ignores itself; its .gitignore goes first, so no frame or video is ever unignored.
|
|
114
|
+
if ((record || video) && !options.outDir) {
|
|
115
|
+
const scenescoutDir = path.join(options.projectDir, MEMORY_DIRNAME);
|
|
116
|
+
fs.mkdirSync(scenescoutDir, { recursive: true });
|
|
117
|
+
writeSelfIgnore(scenescoutDir);
|
|
118
|
+
}
|
|
102
119
|
const start = new URL(options.url);
|
|
103
120
|
// From the start of the run, browser launch included: the budget is wall-clock time a person waits.
|
|
104
121
|
const deadline = options.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined;
|
|
@@ -109,9 +126,10 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
109
126
|
const attached = await engine.attach({
|
|
110
127
|
url: start.origin,
|
|
111
128
|
projectDir: options.projectDir,
|
|
112
|
-
memoryStore:
|
|
129
|
+
memoryStore: store,
|
|
113
130
|
mode: options.mode,
|
|
114
131
|
storageStatePath: options.storageStatePath,
|
|
132
|
+
...(record ? { record: true, ...(frameCap !== undefined ? { maxFrames: frameCap } : {}) } : {}),
|
|
115
133
|
...(options.browser ? { browser: options.browser } : {}),
|
|
116
134
|
actionTimeoutMs: options.actionTimeoutMs,
|
|
117
135
|
navTimeoutMs: options.navTimeoutMs,
|
|
@@ -178,6 +196,8 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
178
196
|
return known;
|
|
179
197
|
const roleEngine = new BrowserEngine();
|
|
180
198
|
roleEngines.set(role, roleEngine);
|
|
199
|
+
if (record)
|
|
200
|
+
roleEngine.sessionKey = replaySessionKey(role);
|
|
181
201
|
const roleDir = path.join(scratch, `role-${role}`);
|
|
182
202
|
fs.mkdirSync(roleDir, { recursive: true });
|
|
183
203
|
const attachedAs = await roleEngine.attach({
|
|
@@ -186,6 +206,7 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
186
206
|
memoryStore: new MemoryStore(roleDir),
|
|
187
207
|
mode: options.mode,
|
|
188
208
|
role,
|
|
209
|
+
...(record ? { record: true, ...(frameCap !== undefined ? { maxFrames: frameCap } : {}) } : {}),
|
|
189
210
|
...(options.browser ? { browser: options.browser } : {}),
|
|
190
211
|
actionTimeoutMs: options.actionTimeoutMs,
|
|
191
212
|
navTimeoutMs: options.navTimeoutMs,
|
|
@@ -197,10 +218,29 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
197
218
|
throw new Error(`the saved sign-in for role "${role}" no longer signs in: ${lost.replace(/ Continuing now tests a logged-out app\.$/, "")}`);
|
|
198
219
|
return roleEngine;
|
|
199
220
|
};
|
|
221
|
+
/** On a recorded check, the frames each flow's steps left, beside the flow. */
|
|
222
|
+
const flowFrames = [];
|
|
223
|
+
/** On a check with --video, each flow's video beside the report (relative to it), or null where none was saved. */
|
|
224
|
+
const flowVideos = [];
|
|
200
225
|
for (const flow of runFlows) {
|
|
201
226
|
const runner = await engineFor(flow.role);
|
|
202
227
|
// --flow-writes never: observe's rule, whatever --mode lets the crawl do.
|
|
203
|
-
const
|
|
228
|
+
const walk = () => runner.replayFlow(flow.steps, options.flowWrites === "never" ? "observe" : options.mode);
|
|
229
|
+
let walked;
|
|
230
|
+
if (video) {
|
|
231
|
+
// Filmed on the session's own page, so the flow runs exactly as it would unfilmed and the video holds it alone.
|
|
232
|
+
const rel = journeyVideoPath(flowRuns.length + 1, flow.file);
|
|
233
|
+
const own = await runner.filming(path.join(outDir, ...rel.split("/")), walk);
|
|
234
|
+
walked = own.value;
|
|
235
|
+
flowVideos.push(own.video ? rel : null);
|
|
236
|
+
if (own.videoError)
|
|
237
|
+
log(` flow ${flow.name}: its video could not be saved (${own.videoError})`);
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
walked = await walk();
|
|
241
|
+
}
|
|
242
|
+
const { frames, ...replay } = walked;
|
|
243
|
+
flowFrames.push(frames ?? []);
|
|
204
244
|
flowRuns.push({ name: flow.name, file: flow.file, steps: flow.steps.length, ...(flow.role === undefined ? {} : { role: flow.role }), ...replay });
|
|
205
245
|
log(` flow ${flow.name}${flow.role === undefined ? "" : ` (as ${flow.role})`}: ${replay.outcome.status}${replay.outcome.status === "passed" ? "" : ` at step ${replay.outcome.step}`}`);
|
|
206
246
|
// --on-refused-step stop: nothing after a refused step runs, and the check ends without a verdict.
|
|
@@ -222,6 +262,23 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
222
262
|
}))),
|
|
223
263
|
}
|
|
224
264
|
: null;
|
|
265
|
+
const replay = record || video
|
|
266
|
+
? recordReplay({
|
|
267
|
+
options,
|
|
268
|
+
startedAt,
|
|
269
|
+
ownRole: engine.role,
|
|
270
|
+
log: store.actionLog,
|
|
271
|
+
visited: [...routes, ...retestPages],
|
|
272
|
+
flows: runFlows
|
|
273
|
+
.slice(0, flowRuns.length)
|
|
274
|
+
.map((flow, i) => ({ flow, outcome: flowRuns[i].outcome, frames: flowFrames[i], video: flowVideos[i] ?? null })),
|
|
275
|
+
scratch,
|
|
276
|
+
say: log,
|
|
277
|
+
...(frameCap !== undefined ? { frameCap } : {}),
|
|
278
|
+
})
|
|
279
|
+
: null;
|
|
280
|
+
if (replay)
|
|
281
|
+
log(` kept ${replayFrames(replay).length} frame(s) and ${replayVideos(replay).length} video(s) for ${REPLAY_FILE}`);
|
|
225
282
|
const { pages: measured, resources } = splitResources(redactRoutes(routes.map(withoutOwnResponse)));
|
|
226
283
|
const flows = redactFlowRuns(flowRuns);
|
|
227
284
|
const pictured = baselines ? redactBaselineRun(baselines) : null;
|
|
@@ -248,6 +305,7 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
|
|
|
248
305
|
retest,
|
|
249
306
|
settings: settingsOf(options),
|
|
250
307
|
baselines: pictured,
|
|
308
|
+
...(replay ? { replay } : {}),
|
|
251
309
|
};
|
|
252
310
|
// A value taken from the environment (a code, a password) is never written, wherever the page echoed it.
|
|
253
311
|
return envValues.size > 0 ? JSON.parse(maskEnvValues(JSON.stringify(result), envValues)) : result;
|
|
@@ -403,3 +461,135 @@ async function takeBaselines(engine, options, mode, targets, log) {
|
|
|
403
461
|
}
|
|
404
462
|
return { mode, engine: engineName, threshold: options.baselineThreshold, dir: shownDir(dir, options.projectDir), results };
|
|
405
463
|
}
|
|
464
|
+
/**
|
|
465
|
+
* Remove what an earlier recorded check left beside the report: replay.html,
|
|
466
|
+
* the frames under replay-frames/ and the journey videos under replay-videos/
|
|
467
|
+
* that a check names as it names them (isReplayFrameFile, isJourneyVideoFile),
|
|
468
|
+
* then any folder that leaves empty. replay.html goes only when it carries
|
|
469
|
+
* the check's generator mark (isGeneratedReplay). Run before every
|
|
470
|
+
* check, recorded or not, so a page from an earlier run is never read, or
|
|
471
|
+
* uploaded, as this one's; whatever else the folder holds stays.
|
|
472
|
+
*/
|
|
473
|
+
export function clearReplayOutput(outDir) {
|
|
474
|
+
// Only a page a check wrote: a file of the same name the project keeps there is its own.
|
|
475
|
+
const page = path.join(outDir, REPLAY_FILE);
|
|
476
|
+
let text = null;
|
|
477
|
+
try {
|
|
478
|
+
text = fs.readFileSync(page, "utf8");
|
|
479
|
+
}
|
|
480
|
+
catch (err) {
|
|
481
|
+
if (err.code !== "ENOENT")
|
|
482
|
+
throw err;
|
|
483
|
+
}
|
|
484
|
+
if (text !== null && isGeneratedReplay(text))
|
|
485
|
+
fs.rmSync(page);
|
|
486
|
+
const videos = path.join(outDir, REPLAY_VIDEOS_DIRNAME);
|
|
487
|
+
if (fs.existsSync(videos)) {
|
|
488
|
+
for (const file of fs.readdirSync(videos, { withFileTypes: true }))
|
|
489
|
+
if (file.isFile() && isJourneyVideoFile(file.name))
|
|
490
|
+
fs.rmSync(path.join(videos, file.name));
|
|
491
|
+
if (fs.readdirSync(videos).length === 0)
|
|
492
|
+
fs.rmdirSync(videos);
|
|
493
|
+
}
|
|
494
|
+
const root = path.join(outDir, REPLAY_FRAMES_DIRNAME);
|
|
495
|
+
let folders;
|
|
496
|
+
try {
|
|
497
|
+
folders = fs.readdirSync(root, { withFileTypes: true });
|
|
498
|
+
}
|
|
499
|
+
catch (err) {
|
|
500
|
+
if (err.code === "ENOENT")
|
|
501
|
+
return;
|
|
502
|
+
throw err;
|
|
503
|
+
}
|
|
504
|
+
for (const folder of folders.filter((f) => f.isDirectory())) {
|
|
505
|
+
const dir = path.join(root, folder.name);
|
|
506
|
+
for (const file of fs.readdirSync(dir, { withFileTypes: true }))
|
|
507
|
+
if (file.isFile() && isReplayFrameFile(file.name))
|
|
508
|
+
fs.rmSync(path.join(dir, file.name));
|
|
509
|
+
if (fs.readdirSync(dir).length === 0)
|
|
510
|
+
fs.rmdirSync(dir);
|
|
511
|
+
}
|
|
512
|
+
if (fs.readdirSync(root).length === 0)
|
|
513
|
+
fs.rmdirSync(root);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* The replay page's model for a recorded check, with its frames copied from
|
|
517
|
+
* the browsers' scratch folders to replay-frames/ beside the report: routes
|
|
518
|
+
* visited in the check's own session, then each flow under the role it ran
|
|
519
|
+
* as. Capped and redacted here, so what is copied is exactly what the page shows.
|
|
520
|
+
*/
|
|
521
|
+
function recordReplay(o) {
|
|
522
|
+
const outDir = o.options.outDir ?? defaultCheckDir(o.options.projectDir);
|
|
523
|
+
/** Where each frame the page names is copied from. */
|
|
524
|
+
const sources = new Map();
|
|
525
|
+
/** A shot with its frame's path moved to where it is copied beside the page. */
|
|
526
|
+
const beside = (shot, memoryDir) => {
|
|
527
|
+
if (!shot?.frame)
|
|
528
|
+
return shot?.pastCap ? { pastCap: true } : {};
|
|
529
|
+
const to = replayFramePath(shot.frame);
|
|
530
|
+
if (!to)
|
|
531
|
+
return {};
|
|
532
|
+
sources.set(to, path.join(memoryDir, ...shot.frame.split("/")));
|
|
533
|
+
return { frame: to };
|
|
534
|
+
};
|
|
535
|
+
// What the crawl kept for each visit, by the path it asked for, in the order the visits ran.
|
|
536
|
+
const own = replaySessionKey();
|
|
537
|
+
const crawled = new Map();
|
|
538
|
+
for (const e of o.log) {
|
|
539
|
+
if (e.action !== "crawl" || e.target === undefined || (e.session ?? own) !== own)
|
|
540
|
+
continue;
|
|
541
|
+
const shot = e.frame ? { frame: e.frame } : e.framePastCap ? { pastCap: true } : {};
|
|
542
|
+
crawled.set(e.target, [...(crawled.get(e.target) ?? []), shot]);
|
|
543
|
+
}
|
|
544
|
+
const ownSession = {
|
|
545
|
+
role: o.ownRole,
|
|
546
|
+
own: true,
|
|
547
|
+
// A MemoryStore keeps its files, frames included, in the .scenescout folder of the directory it is given.
|
|
548
|
+
visits: o.visited.map((route) => visitOf(route, beside(crawled.get(route.path)?.shift(), path.join(o.scratch, MEMORY_DIRNAME)))),
|
|
549
|
+
journeys: [],
|
|
550
|
+
};
|
|
551
|
+
const roles = [ownSession];
|
|
552
|
+
for (const { flow, outcome, frames, video } of o.flows) {
|
|
553
|
+
const memoryDir = path.join(flow.role === undefined ? o.scratch : path.join(o.scratch, `role-${flow.role}`), MEMORY_DIRNAME);
|
|
554
|
+
const journey = journeyOf(flow, outcome, frames.map((f) => beside(f, memoryDir)));
|
|
555
|
+
if (video)
|
|
556
|
+
journey.video = video;
|
|
557
|
+
let role = flow.role === undefined ? ownSession : roles.find((r) => !r.own && r.role === flow.role);
|
|
558
|
+
if (!role) {
|
|
559
|
+
role = { role: flow.role, own: false, visits: [], journeys: [] };
|
|
560
|
+
roles.push(role);
|
|
561
|
+
}
|
|
562
|
+
role.journeys.push(journey);
|
|
563
|
+
}
|
|
564
|
+
const replay = redactReplay(capFrames({ startedAt: o.startedAt, roles, framesLeftOut: 0, ...(o.frameCap !== undefined ? { frameCap: o.frameCap } : {}) }));
|
|
565
|
+
// A frame that cannot be copied is left off the page rather than shown broken.
|
|
566
|
+
const copied = new Set();
|
|
567
|
+
let failed = false;
|
|
568
|
+
for (const rel of replayFrames(replay)) {
|
|
569
|
+
const from = sources.get(rel);
|
|
570
|
+
if (!from)
|
|
571
|
+
continue;
|
|
572
|
+
try {
|
|
573
|
+
const to = path.join(outDir, ...rel.split("/"));
|
|
574
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
575
|
+
fs.copyFileSync(from, to);
|
|
576
|
+
copied.add(rel);
|
|
577
|
+
}
|
|
578
|
+
catch (err) {
|
|
579
|
+
// Said once: a folder that refuses one frame usually refuses them all.
|
|
580
|
+
if (!failed)
|
|
581
|
+
o.say(` could not copy a recorded frame to ${outDir}: ${firstLineOf(err)}; frames that could not be copied are left off ${REPLAY_FILE}`);
|
|
582
|
+
failed = true;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
const keep = (item) => {
|
|
586
|
+
if (!item.frame || copied.has(item.frame))
|
|
587
|
+
return item;
|
|
588
|
+
const { frame: _gone, ...rest } = item;
|
|
589
|
+
return rest;
|
|
590
|
+
};
|
|
591
|
+
return {
|
|
592
|
+
...replay,
|
|
593
|
+
roles: replay.roles.map((r) => ({ ...r, visits: r.visits.map(keep), journeys: r.journeys.map((j) => ({ ...j, steps: j.steps.map(keep) })) })),
|
|
594
|
+
};
|
|
595
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -21,7 +21,9 @@ import { APPROX_DISK_MB, defaultAttachNote, defaultEngine, launchTarget, parseBr
|
|
|
21
21
|
import { CLIENT_LABELS, firstMessageHint, manualFor, parseClients, registerWithClient, vscodeBinary } from "./clients.js";
|
|
22
22
|
import { CLAUDE_CODE_NOT_NEEDED, CLI_NAME, desktopExtensionRoots, diagnose, doctorAllGood, findDesktopExtension, installClosing, ensureCommand, findOnUserPath, installSkill, isEphemeralRoot, launchCommand, manualRegisterCommand, planCommand, registerMcp, resolveClaudeDir, spawnRunner, } from "./installer.js";
|
|
23
23
|
import { downloadBrowsers, presentBrowsers } from "./installer.js";
|
|
24
|
-
import { baselinesDirOf, defaultCheckDir, readCheckInputs, runCheck } from "./check-run.js";
|
|
24
|
+
import { baselinesDirOf, clearReplayOutput, defaultCheckDir, readCheckInputs, runCheck } from "./check-run.js";
|
|
25
|
+
import { buildCheckReplayHtml, commitOf, REPLAY_FILE, replayFrames, replayVideos } from "./engine/check-replay.js";
|
|
26
|
+
import { recordChoice } from "./engine/capture.js";
|
|
25
27
|
import { httpClient, httpJudgeAsk, runCi } from "./ci-run.js";
|
|
26
28
|
import { runExport } from "./export-run.js";
|
|
27
29
|
import { runLogin, runScriptedLogin, savedLine } from "./login-run.js";
|
|
@@ -29,7 +31,7 @@ import { credentialRedactor, LOGIN_ENV, readScriptedLogin } from "./engine/scrip
|
|
|
29
31
|
import { parseLoginArgs } from "./engine/profiles.js";
|
|
30
32
|
import { detectProvider, EXIT_CI, judgeEffort, KEY_ENV, parseCiArgs, redactKeys, secretValues } from "./engine/ci.js";
|
|
31
33
|
import { VISUAL_DIRNAME } from "./engine/baseline.js";
|
|
32
|
-
import { EXIT, exitCodeOf, formatCheck, parseCheckArgs, refusedFlowReason, toSarif, toSummaryJson, unmeasuredReason, } from "./engine/check.js";
|
|
34
|
+
import { EXIT, exitCodeOf, formatCheck, parseCheckArgs, refusedFlowReason, summarise, toSarif, toSummaryJson, unmeasuredReason, } from "./engine/check.js";
|
|
33
35
|
import { downloadLine, EXIT_FIRST_RUN, FIRST_RUN_DIRNAME, firstRunCheckOptions, firstRunDownloads, firstRunSummary, formatFirstRun, modeSentence, parseFirstRunArgs, reportFolderProblem, unreachableReason, writeFirstRunReport, } from "./first-run.js";
|
|
34
36
|
import { credentialSecrets, EXIT_EXPORT, parseExportArgs } from "./engine/export.js";
|
|
35
37
|
import { LEGACY_MEMORY_DIRNAME, MEMORY_DIRNAME, writeSelfIgnore } from "./engine/memory.js";
|
|
@@ -105,7 +107,12 @@ Usage:
|
|
|
105
107
|
passes; 0 counts every changed pixel);
|
|
106
108
|
--sarif-file-anchor path: the repository file a SARIF result points at when
|
|
107
109
|
no saved flow raised it (default: the running workflow's file on GitHub
|
|
108
|
-
Actions, else package.json, else README.md)
|
|
110
|
+
Actions, else package.json, else README.md);
|
|
111
|
+
--record [on|off]: keep a frame after each route visit and each flow step
|
|
112
|
+
and write replay.html beside the report, role → journey → step (default:
|
|
113
|
+
SCENESCOUT_RECORD, else off; the frames go in replay-frames/);
|
|
114
|
+
--video [on|off]: record a WebM of each saved flow, and only of the flows,
|
|
115
|
+
into replay-videos/, played on replay.html beside its steps (default off))
|
|
109
116
|
Exit code: 0 passed, 1 failed the gate, 2 could not run.
|
|
110
117
|
scenescout ci <url> An exploratory run with no person present: a model reached through its API
|
|
111
118
|
drives the tools by the SceneScout method and the run ends in the report.
|
|
@@ -570,10 +577,14 @@ async function check(args) {
|
|
|
570
577
|
console.error(`scenescout check: ${parsed.error}`);
|
|
571
578
|
process.exit(EXIT.error);
|
|
572
579
|
}
|
|
573
|
-
|
|
580
|
+
let options = parsed.options;
|
|
574
581
|
const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
|
|
575
582
|
let inputs;
|
|
576
583
|
try {
|
|
584
|
+
// --record, else SCENESCOUT_RECORD, else off.
|
|
585
|
+
options = { ...options, record: recordChoice(options.record, process.env) };
|
|
586
|
+
// A replay page an earlier run left must never be read, or uploaded, as this run's.
|
|
587
|
+
clearReplayOutput(outDir);
|
|
577
588
|
inputs = readCheckInputs(options);
|
|
578
589
|
}
|
|
579
590
|
catch (err) {
|
|
@@ -618,6 +629,19 @@ async function check(args) {
|
|
|
618
629
|
console.error(`scenescout check: ${sarifFiles.warning}`);
|
|
619
630
|
fs.writeFileSync(path.join(outDir, "check.sarif"), JSON.stringify(toSarif(result, version, sarifFiles), null, 2) + "\n");
|
|
620
631
|
fs.writeFileSync(path.join(outDir, "check.json"), JSON.stringify(toSummaryJson(result, version), null, 2) + "\n");
|
|
632
|
+
if (result.replay) {
|
|
633
|
+
const { passed, couldNotRun } = summarise(result);
|
|
634
|
+
const html = buildCheckReplayHtml(result.replay, {
|
|
635
|
+
version,
|
|
636
|
+
origin: new URL(result.url).origin,
|
|
637
|
+
startedAt: result.replay.startedAt,
|
|
638
|
+
endedAt: result.generatedAt,
|
|
639
|
+
commit: commitOf(process.env),
|
|
640
|
+
passed,
|
|
641
|
+
couldNotRun,
|
|
642
|
+
});
|
|
643
|
+
fs.writeFileSync(path.join(outDir, REPLAY_FILE), html);
|
|
644
|
+
}
|
|
621
645
|
// On GitHub Actions the verdict also goes on the run's summary page.
|
|
622
646
|
if (process.env.GITHUB_STEP_SUMMARY)
|
|
623
647
|
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown);
|
|
@@ -629,6 +653,8 @@ async function check(args) {
|
|
|
629
653
|
}
|
|
630
654
|
console.log("\n" + markdown);
|
|
631
655
|
console.log(`Wrote report.md, check.sarif and check.json to ${outDir}`);
|
|
656
|
+
if (result.replay)
|
|
657
|
+
console.log(`Wrote ${REPLAY_FILE} to ${outDir}, with ${replayFrames(result.replay).length} frame(s) and ${replayVideos(result.replay).length} journey video(s) beside it: open it in a browser to see each step`);
|
|
632
658
|
const pictured = result.baselines?.results.filter((r) => r.files).length ?? 0;
|
|
633
659
|
if (pictured > 0)
|
|
634
660
|
console.log(`Wrote the pictures of ${pictured} changed baseline(s) under ${path.join(outDir, VISUAL_DIRNAME)}`);
|
package/dist/engine/browser.js
CHANGED
|
@@ -460,6 +460,8 @@ export class BrowserEngine {
|
|
|
460
460
|
* checked, a recording shows it.
|
|
461
461
|
*/
|
|
462
462
|
recording = false;
|
|
463
|
+
/** Most frames this recorded session keeps (attach's maxFrames). */
|
|
464
|
+
maxFrames = RECORD_MAX_FRAMES;
|
|
463
465
|
framesKept = 0;
|
|
464
466
|
/** How many frames could not be written. The first one says so in the log; the rest are counted. */
|
|
465
467
|
framesFailed = 0;
|
|
@@ -592,8 +594,13 @@ export class BrowserEngine {
|
|
|
592
594
|
* page looked like: `...(await this.frameFor("crawl"))`.
|
|
593
595
|
*/
|
|
594
596
|
async frameFor(action) {
|
|
597
|
+
const pastCap = this.framePastCap();
|
|
595
598
|
const frame = await this.recordFrame(action);
|
|
596
|
-
return frame ? { frame } : {};
|
|
599
|
+
return frame ? { frame } : pastCap ? { framePastCap: true } : {};
|
|
600
|
+
}
|
|
601
|
+
/** On a recorded session, whether the next step gets no frame because the session already keeps as many as it may. */
|
|
602
|
+
framePastCap() {
|
|
603
|
+
return this.recording && this.framesKept >= this.maxFrames;
|
|
597
604
|
}
|
|
598
605
|
/**
|
|
599
606
|
* The frame for the step just taken, as a path relative to the memory
|
|
@@ -603,7 +610,7 @@ export class BrowserEngine {
|
|
|
603
610
|
*/
|
|
604
611
|
async recordFrame(action) {
|
|
605
612
|
const dir = this.memory?.dir;
|
|
606
|
-
if (!this.recording || !dir || this.framesKept >=
|
|
613
|
+
if (!this.recording || !dir || this.framesKept >= this.maxFrames)
|
|
607
614
|
return undefined;
|
|
608
615
|
const jpeg = await this.liveShot(RECORD_SHOT_TIMEOUT_MS);
|
|
609
616
|
if (!jpeg)
|
|
@@ -630,8 +637,8 @@ export class BrowserEngine {
|
|
|
630
637
|
return undefined;
|
|
631
638
|
}
|
|
632
639
|
this.framesKept += 1;
|
|
633
|
-
if (this.framesKept ===
|
|
634
|
-
this.logAction({ action: "record:full", target: `${
|
|
640
|
+
if (this.framesKept === this.maxFrames) {
|
|
641
|
+
this.logAction({ action: "record:full", target: `${this.maxFrames} frames kept; later steps have none`, url: this.page?.url() ?? "" });
|
|
635
642
|
}
|
|
636
643
|
return rel;
|
|
637
644
|
}
|
|
@@ -1104,6 +1111,7 @@ export class BrowserEngine {
|
|
|
1104
1111
|
this.setTask(opts.task ?? "Attaching and taking stock", opts.task !== undefined);
|
|
1105
1112
|
this.setPace(opts.paceMs);
|
|
1106
1113
|
this.recording = opts.record === true;
|
|
1114
|
+
this.maxFrames = opts.maxFrames ?? RECORD_MAX_FRAMES;
|
|
1107
1115
|
// A re-attached engine starts a new recording: numbering from where the
|
|
1108
1116
|
// last one stopped would run into the cap with frames it never took.
|
|
1109
1117
|
this.framesKept = 0;
|
|
@@ -1671,6 +1679,41 @@ export class BrowserEngine {
|
|
|
1671
1679
|
nativeDialogAt = 0;
|
|
1672
1680
|
/** When a native dialog that asks something (confirm, prompt, a leave confirmation) last opened; an alert only tells. */
|
|
1673
1681
|
nativeQuestionAt = 0;
|
|
1682
|
+
/**
|
|
1683
|
+
* Run `work` while the session's page is filmed into `videoTo` (a WebM,
|
|
1684
|
+
* Playwright's page screencast), so the video holds that work and nothing
|
|
1685
|
+
* the session did before or after it. The page, its tab and so its
|
|
1686
|
+
* sessionStorage are the session's own: the work runs exactly as it would
|
|
1687
|
+
* unfilmed. The video is null when it could not be started or saved
|
|
1688
|
+
* (`videoError` says why); the work's own result and errors are untouched.
|
|
1689
|
+
*/
|
|
1690
|
+
async filming(videoTo, work) {
|
|
1691
|
+
const page = this.requirePage();
|
|
1692
|
+
const why = (err) => (err instanceof Error ? err.message.split("\n")[0] : String(err));
|
|
1693
|
+
let videoError;
|
|
1694
|
+
let filming = false;
|
|
1695
|
+
try {
|
|
1696
|
+
await fs.promises.mkdir(path.dirname(videoTo), { recursive: true });
|
|
1697
|
+
await page.screencast.start({ path: videoTo, ...(page.viewportSize() ? { size: page.viewportSize() } : {}) });
|
|
1698
|
+
filming = true;
|
|
1699
|
+
}
|
|
1700
|
+
catch (err) {
|
|
1701
|
+
videoError = `the video could not be started: ${why(err)}`;
|
|
1702
|
+
}
|
|
1703
|
+
let value;
|
|
1704
|
+
try {
|
|
1705
|
+
value = await work();
|
|
1706
|
+
}
|
|
1707
|
+
finally {
|
|
1708
|
+
// The page the filming began on, even if the work moved the session to a popup.
|
|
1709
|
+
if (filming)
|
|
1710
|
+
await page.screencast.stop().catch((err) => (videoError = `the video could not be saved: ${why(err)}`));
|
|
1711
|
+
}
|
|
1712
|
+
if (videoError)
|
|
1713
|
+
return { value, video: null, videoError };
|
|
1714
|
+
const saved = await fs.promises.stat(videoTo).then((st) => st.size > 0, () => false);
|
|
1715
|
+
return saved ? { value, video: videoTo } : { value, video: null, videoError: "no video was written" };
|
|
1716
|
+
}
|
|
1674
1717
|
requirePage() {
|
|
1675
1718
|
if (!this.page || !this.memory) {
|
|
1676
1719
|
throw new Error("Not attached. Call scout_attach first with the app URL and project path.");
|
|
@@ -4405,7 +4448,8 @@ export class BrowserEngine {
|
|
|
4405
4448
|
async inspectRoute(page, elements, url) {
|
|
4406
4449
|
const { geometry, brokenImages } = await this.measureLayout(page, elements, url);
|
|
4407
4450
|
try {
|
|
4408
|
-
|
|
4451
|
+
// The crawl has just kept this page's frame; a second of the same page would only use up the session's cap.
|
|
4452
|
+
const { defects, sampled } = await this.auditPage({ frame: false });
|
|
4409
4453
|
// Nothing styled to read is not a clean page: contrast, focus and target size went unmeasured.
|
|
4410
4454
|
return sampled > 0
|
|
4411
4455
|
? { geometry, brokenImages, design: defects }
|
|
@@ -5085,7 +5129,15 @@ export class BrowserEngine {
|
|
|
5085
5129
|
refusedBackground.push(sig);
|
|
5086
5130
|
return charged;
|
|
5087
5131
|
};
|
|
5088
|
-
|
|
5132
|
+
/** On a recorded session, the frame after each step that ran (null where none was kept), for the check's replay page. */
|
|
5133
|
+
const frames = [];
|
|
5134
|
+
const done = (outcome) => ({
|
|
5135
|
+
outcome,
|
|
5136
|
+
violations,
|
|
5137
|
+
refusedBackground,
|
|
5138
|
+
websockets: [...websockets],
|
|
5139
|
+
...(this.recording ? { frames } : {}),
|
|
5140
|
+
});
|
|
5089
5141
|
// Nothing from the crawl before this flow is charged to it.
|
|
5090
5142
|
this.oracles.drain(false);
|
|
5091
5143
|
this.blockedRequests = [];
|
|
@@ -5269,6 +5321,12 @@ export class BrowserEngine {
|
|
|
5269
5321
|
const n = i + 1;
|
|
5270
5322
|
const did = describeStep(step);
|
|
5271
5323
|
const { failure, refusal } = step.action === "repeat" ? await runRepeat(step) : await runStep(step);
|
|
5324
|
+
// Taken whatever the step's result: the frame of a step that broke is the evidence of how it broke.
|
|
5325
|
+
if (this.recording) {
|
|
5326
|
+
const pastCap = this.framePastCap();
|
|
5327
|
+
const frame = await this.recordFrame(`flow-${step.action}`);
|
|
5328
|
+
frames.push(frame ? { frame } : pastCap ? { pastCap: true } : {});
|
|
5329
|
+
}
|
|
5272
5330
|
if (refusal)
|
|
5273
5331
|
return done({ status: "refused", step: n, did, reason: refusal.split("\n")[0], path: here() });
|
|
5274
5332
|
if (failure)
|
|
@@ -5327,7 +5385,7 @@ export class BrowserEngine {
|
|
|
5327
5385
|
return `URL: ${url}\n` + report;
|
|
5328
5386
|
}
|
|
5329
5387
|
/** The design audit's measurements, as data as well as prose; scout_design_audit and a check's crawl share it. */
|
|
5330
|
-
async auditPage() {
|
|
5388
|
+
async auditPage(opts = {}) {
|
|
5331
5389
|
const page = this.requirePage();
|
|
5332
5390
|
await this.settle();
|
|
5333
5391
|
const payload = (await page.evaluate(DESIGN_COLLECT_SCRIPT));
|
|
@@ -5349,7 +5407,12 @@ export class BrowserEngine {
|
|
|
5349
5407
|
this.memory?.setPageScore(route, { ...score, at: new Date().toISOString(), url: page.url() });
|
|
5350
5408
|
this.memory?.markRouteFact(route, { audited: true });
|
|
5351
5409
|
}
|
|
5352
|
-
this.logAction({
|
|
5410
|
+
this.logAction({
|
|
5411
|
+
action: "design-audit",
|
|
5412
|
+
url: page.url(),
|
|
5413
|
+
result: score ? `score:${score.overall}` : undefined,
|
|
5414
|
+
...(opts.frame === false ? {} : await this.frameFor("design-audit")),
|
|
5415
|
+
});
|
|
5353
5416
|
return { url: page.url(), report, defects, sampled: payload.records.length };
|
|
5354
5417
|
}
|
|
5355
5418
|
/**
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The replay page of a recorded `scenescout check` (`--record`): what each
|
|
3
|
+
* role did, journey by journey and step by step, with the frame the page
|
|
4
|
+
* showed after each step, so a green run leaves evidence of what passed and a
|
|
5
|
+
* red one shows where it broke.
|
|
6
|
+
*
|
|
7
|
+
* A journey is a saved flow (flow.ts); a role is the session it ran in, the
|
|
8
|
+
* check's own or the one its `role` names. The check's own session also lists
|
|
9
|
+
* the routes it visited, one frame each.
|
|
10
|
+
*
|
|
11
|
+
* Everything here is pure: the mapping of steps to captions and results, the
|
|
12
|
+
* frame cap, the redaction and the page itself are table-tested. check-run.ts
|
|
13
|
+
* gathers the frames from the browser and copies them beside the page.
|
|
14
|
+
*/
|
|
15
|
+
import { redactRoute } from "./memory.js";
|
|
16
|
+
import { describeStep } from "./flow.js";
|
|
17
|
+
import { escapeHtml, plainSegment, RECORD_MAX_FRAMES } from "./replay.js";
|
|
18
|
+
/** The file the page is written to, beside report.md. */
|
|
19
|
+
export const REPLAY_FILE = "replay.html";
|
|
20
|
+
/** The page's generator mark: an earlier run's page is removed only when it carries it, never a file of the same name. */
|
|
21
|
+
export const REPLAY_GENERATOR = "scenescout-check-replay";
|
|
22
|
+
/** Whether a replay.html is one a check wrote. */
|
|
23
|
+
export function isGeneratedReplay(html) {
|
|
24
|
+
return html.includes(`<meta name="generator" content="${REPLAY_GENERATOR}">`);
|
|
25
|
+
}
|
|
26
|
+
/** The folder beside it that holds the frames the page shows. */
|
|
27
|
+
export const REPLAY_FRAMES_DIRNAME = "replay-frames";
|
|
28
|
+
/** The folder beside it that holds one video per journey (--video). */
|
|
29
|
+
export const REPLAY_VIDEOS_DIRNAME = "replay-videos";
|
|
30
|
+
/** A shot's fields as an item carries them. */
|
|
31
|
+
function shotFields(shot) {
|
|
32
|
+
if (shot?.frame)
|
|
33
|
+
return { frame: shot.frame };
|
|
34
|
+
return shot?.pastCap ? { pastCap: true } : {};
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Each step of a replayed flow with its caption, its result and its frame.
|
|
38
|
+
* The replay stops at the first step that breaks, so every step before it
|
|
39
|
+
* passed, it is the failing one, and those after it never ran (and have no
|
|
40
|
+
* frame). `frames` holds one entry per executed step: its frame, or why it has none.
|
|
41
|
+
*/
|
|
42
|
+
export function journeySteps(steps, outcome, frames = []) {
|
|
43
|
+
const broke = outcome.status === "passed" ? null : outcome.step;
|
|
44
|
+
return steps.map((step, i) => {
|
|
45
|
+
const n = i + 1;
|
|
46
|
+
const shot = broke === null || n <= broke ? shotFields(frames[i]) : {};
|
|
47
|
+
const base = { n, caption: describeStep(step), ...shot };
|
|
48
|
+
if (broke === null || n < broke)
|
|
49
|
+
return { ...base, result: "passed" };
|
|
50
|
+
if (n > broke)
|
|
51
|
+
return { n, caption: base.caption, result: "not-run" };
|
|
52
|
+
// `broke` is set only when the outcome is not "passed".
|
|
53
|
+
const failed = outcome;
|
|
54
|
+
return { ...base, result: failed.status, reason: failed.reason, path: failed.path };
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/** One journey of the page, from a flow and how its replay went. */
|
|
58
|
+
export function journeyOf(flow, outcome, frames) {
|
|
59
|
+
return {
|
|
60
|
+
name: flow.name,
|
|
61
|
+
file: flow.file,
|
|
62
|
+
status: outcome.status,
|
|
63
|
+
steps: journeySteps(flow.steps, outcome, frames),
|
|
64
|
+
...(outcome.status === "passed" ? {} : { firstFailing: outcome.step }),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** One visited route, from what the crawl measured. */
|
|
68
|
+
export function visitOf(route, shot) {
|
|
69
|
+
const result = route.loadError !== undefined ? "not-loaded" : route.loginRedirect ? "sign-in" : route.status !== null && route.status >= 400 ? "http-error" : "loaded";
|
|
70
|
+
return {
|
|
71
|
+
path: route.path,
|
|
72
|
+
status: route.status,
|
|
73
|
+
result,
|
|
74
|
+
...(route.loadError !== undefined ? { reason: route.loadError } : {}),
|
|
75
|
+
...shotFields(shot),
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* At most `max` frames per role, the first ones kept: the same cap a recorded
|
|
80
|
+
* session keeps (RECORD_MAX_FRAMES), held here too so the page never shows
|
|
81
|
+
* more than the run was allowed to take, whatever it is handed. A step
|
|
82
|
+
* dropped here is marked past the cap like one the session did not take, and
|
|
83
|
+
* `framesLeftOut` counts both, so the page says why each has no frame.
|
|
84
|
+
*/
|
|
85
|
+
export function capFrames(replay, max = replay.frameCap ?? RECORD_MAX_FRAMES) {
|
|
86
|
+
let leftOut = 0;
|
|
87
|
+
const roles = replay.roles.map((role) => {
|
|
88
|
+
let kept = 0;
|
|
89
|
+
const take = (item) => {
|
|
90
|
+
if (item.pastCap)
|
|
91
|
+
leftOut += 1;
|
|
92
|
+
if (!item.frame)
|
|
93
|
+
return item;
|
|
94
|
+
if (kept < max) {
|
|
95
|
+
kept += 1;
|
|
96
|
+
return item;
|
|
97
|
+
}
|
|
98
|
+
leftOut += 1;
|
|
99
|
+
const { frame: _dropped, ...rest } = item;
|
|
100
|
+
return { ...rest, pastCap: true };
|
|
101
|
+
};
|
|
102
|
+
return {
|
|
103
|
+
...role,
|
|
104
|
+
visits: role.visits.map(take),
|
|
105
|
+
journeys: role.journeys.map((j) => ({ ...j, steps: j.steps.map(take) })),
|
|
106
|
+
};
|
|
107
|
+
});
|
|
108
|
+
return { ...replay, roles, framesLeftOut: leftOut, frameCap: max };
|
|
109
|
+
}
|
|
110
|
+
/** The report's redaction (memory.ts redactRoute) on every path, caption and reason the page shows. */
|
|
111
|
+
export function redactReplay(replay) {
|
|
112
|
+
return {
|
|
113
|
+
...replay,
|
|
114
|
+
roles: replay.roles.map((role) => ({
|
|
115
|
+
...role,
|
|
116
|
+
visits: role.visits.map((v) => ({ ...v, path: redactRoute(v.path), ...(v.reason !== undefined ? { reason: redactRoute(v.reason) } : {}) })),
|
|
117
|
+
journeys: role.journeys.map((j) => ({
|
|
118
|
+
...j,
|
|
119
|
+
steps: j.steps.map((s) => ({
|
|
120
|
+
...s,
|
|
121
|
+
caption: redactRoute(s.caption),
|
|
122
|
+
...(s.reason !== undefined ? { reason: redactRoute(s.reason) } : {}),
|
|
123
|
+
...(s.path !== undefined ? { path: redactRoute(s.path) } : {}),
|
|
124
|
+
})),
|
|
125
|
+
})),
|
|
126
|
+
})),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Where the video of the n-th journey run (1-based) goes beside the page. The
|
|
131
|
+
* flow's file name is reduced to a plain segment, so it decides nothing about
|
|
132
|
+
* where the file is written; the number keeps two flows' videos apart.
|
|
133
|
+
*/
|
|
134
|
+
export function journeyVideoPath(n, file) {
|
|
135
|
+
return `${REPLAY_VIDEOS_DIRNAME}/journey-${String(n).padStart(2, "0")}-${plainSegment(file.replace(/\.json$/i, ""), "flow")}.webm`;
|
|
136
|
+
}
|
|
137
|
+
/** Whether a file under replay-videos/ is a video a check wrote, and so one an earlier run's clean-up may remove. */
|
|
138
|
+
export function isJourneyVideoFile(name) {
|
|
139
|
+
return /^journey-\d{2,}-[a-z0-9._-]+\.webm$/i.test(name);
|
|
140
|
+
}
|
|
141
|
+
/** Every journey video the page links, in page order. */
|
|
142
|
+
export function replayVideos(replay) {
|
|
143
|
+
return replay.roles.flatMap((r) => r.journeys.flatMap((j) => (j.video ? [j.video] : [])));
|
|
144
|
+
}
|
|
145
|
+
/** Every frame the page shows, in page order. */
|
|
146
|
+
export function replayFrames(replay) {
|
|
147
|
+
return replay.roles.flatMap((r) => [
|
|
148
|
+
...r.visits.flatMap((v) => (v.frame ? [v.frame] : [])),
|
|
149
|
+
...r.journeys.flatMap((j) => j.steps.flatMap((s) => (s.frame ? [s.frame] : []))),
|
|
150
|
+
]);
|
|
151
|
+
}
|
|
152
|
+
const RECORDED_FRAME = /^recordings\/([a-z0-9._-]+)\/(\d{4}-[a-z0-9._-]+\.jpg)$/i;
|
|
153
|
+
/**
|
|
154
|
+
* Where a frame a session recorded (replay.ts framePath) goes beside the
|
|
155
|
+
* page, or null for anything that is not one: the path is used to write a
|
|
156
|
+
* file, so nothing else is let through.
|
|
157
|
+
*/
|
|
158
|
+
export function replayFramePath(recorded) {
|
|
159
|
+
const m = RECORDED_FRAME.exec(recorded);
|
|
160
|
+
return m ? `${REPLAY_FRAMES_DIRNAME}/${m[1]}/${m[2]}` : null;
|
|
161
|
+
}
|
|
162
|
+
/** Whether a file in a session folder under replay-frames/ is a frame a check wrote, and so one an earlier run's clean-up may remove. */
|
|
163
|
+
export function isReplayFrameFile(name) {
|
|
164
|
+
return /^\d{4}-[a-z0-9._-]+\.jpg$/i.test(name);
|
|
165
|
+
}
|
|
166
|
+
/** The session name a check's own browser records its frames under, and one for each role's. Distinct, so their frames never share a folder. */
|
|
167
|
+
export function replaySessionKey(role) {
|
|
168
|
+
return role === undefined ? "check" : `role-${plainSegment(role, "role")}`;
|
|
169
|
+
}
|
|
170
|
+
/** A commit to name in the page's header: GITHUB_SHA when it is one, else none. */
|
|
171
|
+
export function commitOf(env) {
|
|
172
|
+
const sha = (env.GITHUB_SHA ?? "").trim();
|
|
173
|
+
return /^[0-9a-f]{7,64}$/i.test(sha) ? sha.toLowerCase() : undefined;
|
|
174
|
+
}
|
|
175
|
+
const RESULT_WORD = { passed: "passed", failed: "failed", refused: "refused", "not-run": "not run" };
|
|
176
|
+
const VISIT_WORD = {
|
|
177
|
+
loaded: "loaded",
|
|
178
|
+
"http-error": "HTTP error",
|
|
179
|
+
"sign-in": "sent to sign-in",
|
|
180
|
+
"not-loaded": "did not load",
|
|
181
|
+
};
|
|
182
|
+
/** A UTC time as the page shows it: the reader's own clock is not this page's to assume. */
|
|
183
|
+
function stamp(iso) {
|
|
184
|
+
return iso.length >= 19 ? `${iso.slice(0, 10)} ${iso.slice(11, 19)} UTC` : iso;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* A frame, or a line saying why there is none. No script: the note sits in the
|
|
188
|
+
* same grid cell under the picture, so a picture that loads covers it and one
|
|
189
|
+
* that is missing (a copy sent without replay-frames/) leaves it showing.
|
|
190
|
+
*/
|
|
191
|
+
function frameHtml(item, alt, cap) {
|
|
192
|
+
if (!item.frame)
|
|
193
|
+
return item.pastCap
|
|
194
|
+
? `<p class="noframe">No frame: the session had already kept ${cap}, the most one session keeps.</p>`
|
|
195
|
+
: `<p class="noframe">No frame</p>`;
|
|
196
|
+
const src = escapeHtml(item.frame);
|
|
197
|
+
return (`<a class="frame" href="${src}" target="_blank" rel="noreferrer" data-testid="replay-frame-open">` +
|
|
198
|
+
`<span class="gone-note">If no picture shows here, this frame is not beside this file. Frames live in the <code>${REPLAY_FRAMES_DIRNAME}/</code> folder, which travels with it.</span>` +
|
|
199
|
+
`<img loading="lazy" src="${src}" alt="${escapeHtml(alt)}"></a>`);
|
|
200
|
+
}
|
|
201
|
+
function stepHtml(s, anchor, cap) {
|
|
202
|
+
const first = s.result === "failed" || s.result === "refused";
|
|
203
|
+
const detail = [s.reason, s.path ? `on ${s.path}` : ""].filter(Boolean).join(" ");
|
|
204
|
+
return (`<li class="step ${s.result}${first ? " first-failing" : ""}"${first ? ` id="${anchor}"` : ""} data-result="${s.result}">` +
|
|
205
|
+
`<div class="caption"><span class="n">${s.n}</span> <span class="what">${escapeHtml(s.caption)}</span> <span class="result r-${s.result}">${RESULT_WORD[s.result]}</span></div>` +
|
|
206
|
+
(detail ? `<p class="why">${escapeHtml(detail)}</p>` : "") +
|
|
207
|
+
(s.result === "not-run" ? "" : frameHtml(s, `The page after step ${s.n}: ${s.caption}`, cap)) +
|
|
208
|
+
`</li>`);
|
|
209
|
+
}
|
|
210
|
+
function journeyHtml(j, id, cap) {
|
|
211
|
+
const ok = j.status === "passed";
|
|
212
|
+
const anchor = `${id}-first-failing`;
|
|
213
|
+
const badge = ok ? `<span class="badge pass">passed</span>` : `<span class="badge fail">${j.status}</span>`;
|
|
214
|
+
const jump = j.firstFailing !== undefined
|
|
215
|
+
? `<a class="jump" href="#${anchor}" data-testid="replay-first-failing-link">Step ${j.firstFailing} ${j.status}</a>`
|
|
216
|
+
: `<span class="count">${j.steps.length} step${j.steps.length === 1 ? "" : "s"}</span>`;
|
|
217
|
+
return (`<details class="journey ${ok ? "pass" : "fail"}"${ok ? "" : " open"} data-status="${j.status}"><summary data-testid="replay-journey-toggle">${badge} <b>${escapeHtml(j.name)}</b> <span class="file">${escapeHtml(j.file)}</span> ${jump}</summary>` +
|
|
218
|
+
(j.video
|
|
219
|
+
? `<figure class="video"><video controls preload="metadata" src="${escapeHtml(j.video)}" data-testid="replay-journey-video"></video>` +
|
|
220
|
+
`<figcaption>The whole journey as it ran. <a href="${escapeHtml(j.video)}" target="_blank" rel="noreferrer" data-testid="replay-video-open">Open the video</a></figcaption></figure>`
|
|
221
|
+
: "") +
|
|
222
|
+
`<ol class="steps">${j.steps.map((s) => stepHtml(s, anchor, cap)).join("")}</ol></details>`);
|
|
223
|
+
}
|
|
224
|
+
function visitsHtml(visits, cap) {
|
|
225
|
+
if (visits.length === 0)
|
|
226
|
+
return "";
|
|
227
|
+
const items = visits
|
|
228
|
+
.map((v) => `<li class="visit ${v.result}"><div class="caption"><span class="what">visit ${escapeHtml(v.path)}</span> <span class="result v-${v.result}">${v.status ?? "—"} · ${VISIT_WORD[v.result]}</span></div>` +
|
|
229
|
+
(v.reason ? `<p class="why">${escapeHtml(v.reason)}</p>` : "") +
|
|
230
|
+
frameHtml(v, `The page at ${v.path}`, cap) +
|
|
231
|
+
`</li>`)
|
|
232
|
+
.join("");
|
|
233
|
+
return `<details class="visits"><summary data-testid="replay-visits-toggle">Routes visited <span class="count">${visits.length}</span></summary><ol class="steps">${items}</ol></details>`;
|
|
234
|
+
}
|
|
235
|
+
function roleHtml(r, i, cap) {
|
|
236
|
+
const failing = r.journeys.filter((j) => j.status !== "passed").length;
|
|
237
|
+
const tally = `${r.journeys.length} journey${r.journeys.length === 1 ? "" : "s"}${failing ? `, ${failing} not passed` : ""}`;
|
|
238
|
+
return (`<section class="role" id="role-${i}"><h2>${escapeHtml(r.role)} <span class="count">${r.own ? "the check's own session · " : ""}${tally}</span></h2>` +
|
|
239
|
+
visitsHtml(r.visits, cap) +
|
|
240
|
+
(r.journeys.length > 0
|
|
241
|
+
? r.journeys.map((j, k) => journeyHtml(j, `role-${i}-journey-${k}`, cap)).join("")
|
|
242
|
+
: `<p class="none">No journey ran as this role.</p>`) +
|
|
243
|
+
`</section>`);
|
|
244
|
+
}
|
|
245
|
+
const STYLE = `
|
|
246
|
+
:root { color-scheme: light dark; --bg:#f6f7f9; --panel:#fff; --line:#d9dde3; --text:#15181d; --muted:#5d6673; --accent:#2563eb; --pass:#047857; --pass-bg:#d1fae5; --fail:#b91c1c; --fail-bg:#fee2e2; }
|
|
247
|
+
@media (prefers-color-scheme: dark) { :root { --bg:#0e1116; --panel:#161a21; --line:#2a303a; --text:#e6e9ee; --muted:#98a2b3; --accent:#7aa2ff; --pass:#6ee7b7; --pass-bg:#063b2c; --fail:#fca5a5; --fail-bg:#3f1010; } }
|
|
248
|
+
* { box-sizing:border-box; }
|
|
249
|
+
body { margin:0; background:var(--bg); color:var(--text); font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif; }
|
|
250
|
+
header { padding:14px 20px; background:var(--panel); border-bottom:1px solid var(--line); }
|
|
251
|
+
header h1 { margin:0 0 4px; font-size:18px; display:flex; gap:10px; align-items:center; flex-wrap:wrap; }
|
|
252
|
+
header dl { display:flex; flex-wrap:wrap; gap:4px 18px; margin:0; font-size:13px; color:var(--muted); }
|
|
253
|
+
header dt { font-weight:600; } header dd { margin:0 0 0 4px; overflow-wrap:anywhere; }
|
|
254
|
+
header .pair { display:flex; }
|
|
255
|
+
main { max-width:1000px; margin:0 auto; padding:20px 16px 64px; }
|
|
256
|
+
h2 { font-size:19px; margin:28px 0 10px; }
|
|
257
|
+
.count, .file { color:var(--muted); font-size:13px; font-weight:400; }
|
|
258
|
+
.badge { display:inline-block; padding:1px 8px; border-radius:999px; font-size:12px; font-weight:700; text-transform:uppercase; letter-spacing:.03em; }
|
|
259
|
+
.badge.pass { color:var(--pass); background:var(--pass-bg); } .badge.fail { color:var(--fail); background:var(--fail-bg); }
|
|
260
|
+
details.journey, details.visits { background:var(--panel); border:1px solid var(--line); border-radius:8px; margin:10px 0; padding:10px 14px; }
|
|
261
|
+
details.journey.fail { border-color:var(--fail); }
|
|
262
|
+
summary { cursor:pointer; }
|
|
263
|
+
.jump { color:var(--fail); font-size:13px; margin-left:6px; }
|
|
264
|
+
ol.steps { list-style:none; margin:10px 0 0; padding:0; }
|
|
265
|
+
.step, .visit { border-left:3px solid var(--line); padding:6px 10px; margin:0 0 10px; }
|
|
266
|
+
.step.passed, .visit.loaded { border-color:var(--pass); }
|
|
267
|
+
.step.failed, .step.refused, .visit.http-error, .visit.not-loaded, .visit.sign-in { border-color:var(--fail); }
|
|
268
|
+
.step.first-failing { background:var(--fail-bg); border-left-width:6px; }
|
|
269
|
+
.step.not-run { opacity:.6; }
|
|
270
|
+
.caption { display:flex; flex-wrap:wrap; gap:6px 10px; align-items:baseline; }
|
|
271
|
+
.caption .n { font-weight:700; color:var(--muted); min-width:1.5em; }
|
|
272
|
+
.caption .what { font:13px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; overflow-wrap:anywhere; }
|
|
273
|
+
.result { margin-left:auto; font-size:12px; font-weight:700; }
|
|
274
|
+
.r-passed, .v-loaded { color:var(--pass); } .r-failed, .r-refused, .v-http-error, .v-not-loaded, .v-sign-in { color:var(--fail); } .r-not-run { color:var(--muted); }
|
|
275
|
+
.why { margin:4px 0; font-size:13px; overflow-wrap:anywhere; }
|
|
276
|
+
a.frame { display:grid; margin:6px 0 2px; max-width:min(100%,720px); color:var(--muted); font-size:12px; }
|
|
277
|
+
a.frame > * { grid-area:1 / 1; }
|
|
278
|
+
a.frame img { position:relative; max-width:100%; max-height:400px; object-fit:cover; object-position:top; border:1px solid var(--line); border-radius:6px; display:block; background:var(--panel); }
|
|
279
|
+
.gone-note { align-self:end; padding:2.6em 12px 12px; border:1px dashed var(--line); border-radius:6px; }
|
|
280
|
+
.noframe, .none { color:var(--muted); font-size:12px; font-style:italic; margin:4px 0; }
|
|
281
|
+
.note { color:var(--muted); font-size:13px; }
|
|
282
|
+
figure.video { margin:10px 0; max-width:min(100%,720px); }
|
|
283
|
+
figure.video video { width:100%; border:1px solid var(--line); border-radius:6px; display:block; background:#000; }
|
|
284
|
+
figure.video figcaption { color:var(--muted); font-size:12px; margin-top:4px; }
|
|
285
|
+
`;
|
|
286
|
+
/** The whole page: one HTML file with no scripts, no event handlers and no external assets; its frames and videos sit beside it. */
|
|
287
|
+
export function buildCheckReplayHtml(replay, meta) {
|
|
288
|
+
const verdict = meta.couldNotRun > 0 ? "could not run every flow" : meta.passed ? "passed" : "failed";
|
|
289
|
+
const badge = `<span class="badge ${meta.passed && meta.couldNotRun === 0 ? "pass" : "fail"}" data-testid="replay-verdict">${verdict}</span>`;
|
|
290
|
+
const pair = (term, value) => `<div class="pair"><dt>${term}</dt><dd>${escapeHtml(value)}</dd></div>`;
|
|
291
|
+
const frames = replayFrames(replay).length;
|
|
292
|
+
return `<!doctype html>
|
|
293
|
+
<html lang="en">
|
|
294
|
+
<head>
|
|
295
|
+
<meta charset="utf-8">
|
|
296
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
297
|
+
<meta name="generator" content="${REPLAY_GENERATOR}">
|
|
298
|
+
<title>SceneScout check replay — ${escapeHtml(meta.origin)}</title>
|
|
299
|
+
<style>${STYLE}</style>
|
|
300
|
+
</head>
|
|
301
|
+
<body>
|
|
302
|
+
<header>
|
|
303
|
+
<h1>SceneScout check replay ${badge}</h1>
|
|
304
|
+
<dl>${pair("App", meta.origin)}${pair("Started", stamp(meta.startedAt))}${pair("Ended", stamp(meta.endedAt))}${meta.commit ? pair("Commit", meta.commit) : ""}${pair("SceneScout", `v${meta.version}`)}${pair("Frames", String(frames))}${replayVideos(replay).length > 0 ? pair("Videos", String(replayVideos(replay).length)) : ""}</dl>
|
|
305
|
+
</header>
|
|
306
|
+
<main>
|
|
307
|
+
<p class="note">Each role, then each journey it walked, step by step, with the page as it was after the step${frames === 0 ? " when the check was recorded (--record)" : ""}. Typed values are never shown, and secrets in addresses are redacted as in the report.${replay.framesLeftOut > 0
|
|
308
|
+
? ` ${replay.framesLeftOut} step(s) and visit(s) have no frame because their session had already kept ${replay.frameCap ?? RECORD_MAX_FRAMES}, the most one session keeps.`
|
|
309
|
+
: ""}</p>
|
|
310
|
+
${replay.roles.map((r, i) => roleHtml(r, i, replay.frameCap ?? RECORD_MAX_FRAMES)).join("\n") || '<p class="none">Nothing was recorded.</p>'}
|
|
311
|
+
</main>
|
|
312
|
+
</body>
|
|
313
|
+
</html>
|
|
314
|
+
`;
|
|
315
|
+
}
|
package/dist/engine/check.js
CHANGED
|
@@ -501,7 +501,13 @@ export const CHECK_OPTION_NAMES = [
|
|
|
501
501
|
"baselines",
|
|
502
502
|
"baseline-threshold",
|
|
503
503
|
"sarif-file-anchor",
|
|
504
|
+
"record",
|
|
505
|
+
"video",
|
|
504
506
|
];
|
|
507
|
+
/** Options that may be given alone, meaning on: `--record`, as well as `--record on` and `--record=off`. */
|
|
508
|
+
const SWITCH_OPTIONS = new Set(["record", "video"]);
|
|
509
|
+
const SWITCH_ON = ["on", "true", "1"];
|
|
510
|
+
const SWITCH_OFF = ["off", "false", "0"];
|
|
505
511
|
export const MAX_CHECK_ROUTES = 150;
|
|
506
512
|
/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
|
|
507
513
|
export const MAX_DISCOVERY_ROUNDS = 6;
|
|
@@ -522,6 +528,14 @@ export function parseCheckArgs(args, cwd) {
|
|
|
522
528
|
}
|
|
523
529
|
const eq = a.indexOf("=");
|
|
524
530
|
const name = eq > 0 ? a.slice(2, eq) : a.slice(2);
|
|
531
|
+
// A switch takes the next argument only when it is on or off: `--record http://…` records, and checks that URL.
|
|
532
|
+
if (eq < 0 && SWITCH_OPTIONS.has(name)) {
|
|
533
|
+
const next = args[i + 1]?.toLowerCase();
|
|
534
|
+
if (next !== undefined && [...SWITCH_ON, ...SWITCH_OFF].includes(next))
|
|
535
|
+
i += 1;
|
|
536
|
+
flags.set(name, next !== undefined && [...SWITCH_ON, ...SWITCH_OFF].includes(next) ? next : "on");
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
525
539
|
const value = eq > 0 ? a.slice(eq + 1) : args[i + 1];
|
|
526
540
|
if (value === undefined || (eq < 0 && value.startsWith("--")))
|
|
527
541
|
return { ok: false, error: `--${name} needs a value` };
|
|
@@ -663,6 +677,14 @@ export function parseCheckArgs(args, cwd) {
|
|
|
663
677
|
const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
|
|
664
678
|
if (anchor && !anchor.ok)
|
|
665
679
|
return anchor;
|
|
680
|
+
const recordRaw = flags.get("record")?.trim().toLowerCase();
|
|
681
|
+
if (recordRaw !== undefined && !SWITCH_ON.includes(recordRaw) && !SWITCH_OFF.includes(recordRaw)) {
|
|
682
|
+
return { ok: false, error: "--record is on or off, or given alone for on" };
|
|
683
|
+
}
|
|
684
|
+
const videoRaw = flags.get("video")?.trim().toLowerCase();
|
|
685
|
+
if (videoRaw !== undefined && !SWITCH_ON.includes(videoRaw) && !SWITCH_OFF.includes(videoRaw)) {
|
|
686
|
+
return { ok: false, error: "--video is on or off, or given alone for on" };
|
|
687
|
+
}
|
|
666
688
|
const resolve = (p) => resolveArgPath(cwd, p);
|
|
667
689
|
return {
|
|
668
690
|
ok: true,
|
|
@@ -689,6 +711,8 @@ export function parseCheckArgs(args, cwd) {
|
|
|
689
711
|
...(baselinesDir !== undefined ? { baselinesDir: resolve(baselinesDir) } : {}),
|
|
690
712
|
baselineThreshold: threshold.value,
|
|
691
713
|
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
714
|
+
...(recordRaw !== undefined ? { record: SWITCH_ON.includes(recordRaw) } : {}),
|
|
715
|
+
...(videoRaw !== undefined && SWITCH_ON.includes(videoRaw) ? { video: true } : {}),
|
|
692
716
|
},
|
|
693
717
|
};
|
|
694
718
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.20.1",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|