scenescout 3.21.3 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +4 -1
- package/dist/check-run.js +62 -1
- package/dist/cli.js +18 -3
- package/dist/engine/check-replay.js +12 -2
- package/dist/engine/check-report.js +503 -0
- package/dist/engine/check.js +9 -0
- package/dist/engine/flow.js +28 -8
- package/package.json +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# scenescout
|
|
2
2
|
|
|
3
|
+
## 3.22.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 072b68b: `scenescout check --record --template <file.json>` (the action's `template` input) also writes the recorded run up as a test report laid out by the template: each saved flow a test and each step a row with its expected and actual result and its frame, failed steps as deviations, a SHA-256 manifest of the evidence and blank sign-off rows. Flows gain optional `id`, `requirements` and a step's `expected` for traceability; an example template is in `examples/report-template.json`.
|
|
8
|
+
|
|
3
9
|
## 3.21.3
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -279,7 +279,10 @@ Use SceneScout to test http://localhost:3000, record the run
|
|
|
279
279
|
or, on the tool directly, `scout_attach {record: true}`. `SCENESCOUT_RECORD=on` in
|
|
280
280
|
the server's environment records every run. A CI gate records too:
|
|
281
281
|
`scenescout check --record` writes `replay.html`, every journey step by step with
|
|
282
|
-
its frames ([recording a check](docs/ci.md#recording-a-check))
|
|
282
|
+
its frames ([recording a check](docs/ci.md#recording-a-check)), and
|
|
283
|
+
`--template <file.json>` also writes it up as a test report laid out as a template
|
|
284
|
+
says, with expected and actual results, deviations, blank sign-off rows and a
|
|
285
|
+
SHA-256 manifest of the evidence ([test reports](docs/ci.md#a-test-report-from-a-template)).
|
|
283
286
|
|
|
284
287
|
Then `scout_report` writes two files side by side in `.scenescout/`:
|
|
285
288
|
`report.md` as always, and `report.html` — the whole run as one self-contained
|
package/dist/check-run.js
CHANGED
|
@@ -20,6 +20,7 @@ import { firstLineOf } from "./engine/limits.js";
|
|
|
20
20
|
import { isNonPageResource } from "./engine/crawl.js";
|
|
21
21
|
import { checkRetestPlan, retestResults, wellFormedFindings } from "./engine/verify.js";
|
|
22
22
|
import { capFrames, isReplayFrameFile, journeyOf, journeyVideoPath, isGeneratedReplay, isJourneyVideoFile, redactReplay, REPLAY_VIDEOS_DIRNAME, replayVideos, REPLAY_FILE, REPLAY_FRAMES_DIRNAME, replayFramePath, replayFrames, replaySessionKey, visitOf, } from "./engine/check-replay.js";
|
|
23
|
+
import { buildTemplateReport, evidenceFiles, evidenceIndex, isGeneratedReport, readReportTemplate, recordedReportFile, reportFileOf, templateRecordError, } from "./engine/check-report.js";
|
|
23
24
|
/** The baselines folder a check uses: the one --baselines names, else the project's own. */
|
|
24
25
|
export function baselinesDirOf(options) {
|
|
25
26
|
return options.baselinesDir ?? defaultBaselinesDir(options.projectDir);
|
|
@@ -470,7 +471,34 @@ async function takeBaselines(engine, options, mode, targets, log) {
|
|
|
470
471
|
* check, recorded or not, so a page from an earlier run is never read, or
|
|
471
472
|
* uploaded, as this one's; whatever else the folder holds stays.
|
|
472
473
|
*/
|
|
473
|
-
export function clearReplayOutput(outDir) {
|
|
474
|
+
export function clearReplayOutput(outDir, reportFile) {
|
|
475
|
+
// The test report the earlier run's check.json says it wrote (--template), and the one this run's template names:
|
|
476
|
+
// each only when it still carries the mark. A copy renamed or kept under any other name is the project's own.
|
|
477
|
+
const reports = new Set();
|
|
478
|
+
try {
|
|
479
|
+
const earlier = recordedReportFile(JSON.parse(fs.readFileSync(path.join(outDir, "check.json"), "utf8")));
|
|
480
|
+
if (earlier)
|
|
481
|
+
reports.add(earlier);
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
// No earlier check.json, or one that cannot be read: it names no report to remove.
|
|
485
|
+
}
|
|
486
|
+
if (reportFile)
|
|
487
|
+
reports.add(reportFile);
|
|
488
|
+
for (const name of reports) {
|
|
489
|
+
const at = path.join(outDir, name);
|
|
490
|
+
let text;
|
|
491
|
+
try {
|
|
492
|
+
text = fs.readFileSync(at, "utf8");
|
|
493
|
+
}
|
|
494
|
+
catch (err) {
|
|
495
|
+
if (err.code === "ENOENT")
|
|
496
|
+
continue;
|
|
497
|
+
throw err;
|
|
498
|
+
}
|
|
499
|
+
if (isGeneratedReport(text))
|
|
500
|
+
fs.rmSync(at);
|
|
501
|
+
}
|
|
474
502
|
// Only a page a check wrote: a file of the same name the project keeps there is its own.
|
|
475
503
|
const page = path.join(outDir, REPLAY_FILE);
|
|
476
504
|
let text = null;
|
|
@@ -512,6 +540,39 @@ export function clearReplayOutput(outDir) {
|
|
|
512
540
|
if (fs.readdirSync(root).length === 0)
|
|
513
541
|
fs.rmdirSync(root);
|
|
514
542
|
}
|
|
543
|
+
/**
|
|
544
|
+
* The report template a check was given (--template), read and validated, or
|
|
545
|
+
* null for none. Throws, before the browser starts, when the check is not
|
|
546
|
+
* recorded, when the template is not valid, or when the file it would write is
|
|
547
|
+
* one the project keeps there and a check did not write.
|
|
548
|
+
*/
|
|
549
|
+
export function prepareTemplateReport(options, outDir) {
|
|
550
|
+
if (options.template === undefined)
|
|
551
|
+
return null;
|
|
552
|
+
const needsRecord = templateRecordError(options);
|
|
553
|
+
if (needsRecord)
|
|
554
|
+
throw new Error(needsRecord);
|
|
555
|
+
const template = readReportTemplate(options.template);
|
|
556
|
+
const file = path.join(outDir, reportFileOf(template));
|
|
557
|
+
let text = null;
|
|
558
|
+
try {
|
|
559
|
+
text = fs.readFileSync(file, "utf8");
|
|
560
|
+
}
|
|
561
|
+
catch (err) {
|
|
562
|
+
if (err.code !== "ENOENT")
|
|
563
|
+
throw err;
|
|
564
|
+
}
|
|
565
|
+
if (text !== null && !isGeneratedReport(text))
|
|
566
|
+
throw new Error(`${file} is not a report SceneScout wrote, so a check will not overwrite it. Move or rename it, name another file in the template's "file", or pass --out to write the check somewhere else`);
|
|
567
|
+
return template;
|
|
568
|
+
}
|
|
569
|
+
/** Write the template's report beside replay.html, after check.json and the replay page, so the manifest hashes the files as written. Returns its file name. */
|
|
570
|
+
export function writeTemplateReport(outDir, result, template, meta) {
|
|
571
|
+
const file = reportFileOf(template);
|
|
572
|
+
const evidence = evidenceIndex(outDir, evidenceFiles(result));
|
|
573
|
+
fs.writeFileSync(path.join(outDir, file), buildTemplateReport(result, template, evidence, meta));
|
|
574
|
+
return file;
|
|
575
|
+
}
|
|
515
576
|
/**
|
|
516
577
|
* Why a recorded check (--record or --video) must not start, or null: a
|
|
517
578
|
* replay.html in the output folder that a check did not write is the
|
package/dist/cli.js
CHANGED
|
@@ -21,7 +21,8 @@ import { APPROX_DISK_MB, defaultAttachNote, defaultEngine, launchTarget, parseBr
|
|
|
21
21
|
import { CLIENT_LABELS, firstMessageHint, manualFor, parseClients, registerWithClient, vscodeBinary } from "./clients.js";
|
|
22
22
|
import { CLAUDE_CODE_NOT_NEEDED, CLI_NAME, desktopExtensionRoots, diagnose, doctorAllGood, findDesktopExtension, installClosing, ensureCommand, findOnUserPath, installSkill, isEphemeralRoot, launchCommand, manualRegisterCommand, planCommand, registerMcp, resolveClaudeDir, spawnRunner, } from "./installer.js";
|
|
23
23
|
import { downloadBrowsers, presentBrowsers } from "./installer.js";
|
|
24
|
-
import { baselinesDirOf, clearReplayOutput, replayPageConflict, defaultCheckDir, readCheckInputs, runCheck } from "./check-run.js";
|
|
24
|
+
import { baselinesDirOf, clearReplayOutput, replayPageConflict, defaultCheckDir, prepareTemplateReport, readCheckInputs, runCheck, writeTemplateReport, } from "./check-run.js";
|
|
25
|
+
import { reportFileOf } from "./engine/check-report.js";
|
|
25
26
|
import { buildCheckReplayHtml, commitOf, REPLAY_FILE, replayFrames, replayVideos } from "./engine/check-replay.js";
|
|
26
27
|
import { recordChoice } from "./engine/capture.js";
|
|
27
28
|
import { httpClient, httpJudgeAsk, runCi } from "./ci-run.js";
|
|
@@ -112,7 +113,10 @@ Usage:
|
|
|
112
113
|
and write replay.html beside the report, role → journey → step (default:
|
|
113
114
|
SCENESCOUT_RECORD, else off; the frames go in replay-frames/);
|
|
114
115
|
--video [on|off]: record a WebM of each saved flow, and only of the flows,
|
|
115
|
-
into replay-videos/, played on replay.html beside its steps (default off)
|
|
116
|
+
into replay-videos/, played on replay.html beside its steps (default off);
|
|
117
|
+
--template file.json: on a recorded check, also write a test report laid out
|
|
118
|
+
as the template says (report.html unless it names a file), with the frames,
|
|
119
|
+
deviations and a SHA-256 manifest of the evidence; needs --record)
|
|
116
120
|
Exit code: 0 passed, 1 failed the gate, 2 could not run.
|
|
117
121
|
scenescout ci <url> An exploratory run with no person present: a model reached through its API
|
|
118
122
|
drives the tools by the SceneScout method and the run ends in the report.
|
|
@@ -580,15 +584,18 @@ async function check(args) {
|
|
|
580
584
|
let options = parsed.options;
|
|
581
585
|
const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
|
|
582
586
|
let inputs;
|
|
587
|
+
let template = null;
|
|
583
588
|
try {
|
|
584
589
|
// --record, else SCENESCOUT_RECORD, else off.
|
|
585
590
|
options = { ...options, record: recordChoice(options.record, process.env) };
|
|
591
|
+
// --template: read and checked before the browser starts, so a bad template never costs a run.
|
|
592
|
+
template = prepareTemplateReport(options, outDir);
|
|
586
593
|
// A replay.html the project keeps there is its own: a recorded check refuses to start rather than overwrite it.
|
|
587
594
|
const conflict = options.record || options.video ? replayPageConflict(outDir) : null;
|
|
588
595
|
if (conflict)
|
|
589
596
|
throw new Error(conflict);
|
|
590
597
|
// A replay page an earlier run left must never be read, or uploaded, as this run's.
|
|
591
|
-
clearReplayOutput(outDir);
|
|
598
|
+
clearReplayOutput(outDir, template ? reportFileOf(template) : undefined);
|
|
592
599
|
inputs = readCheckInputs(options);
|
|
593
600
|
}
|
|
594
601
|
catch (err) {
|
|
@@ -615,7 +622,11 @@ async function check(args) {
|
|
|
615
622
|
console.error(`scenescout check: could not run a saved flow: ${refused}`);
|
|
616
623
|
process.exit(EXIT.error);
|
|
617
624
|
}
|
|
625
|
+
// Recorded in check.json, so the next check removes exactly this file and the action finds it.
|
|
626
|
+
if (template && result.replay)
|
|
627
|
+
result = { ...result, testReport: reportFileOf(template) };
|
|
618
628
|
const markdown = formatCheck(result);
|
|
629
|
+
let reportFile = null;
|
|
619
630
|
try {
|
|
620
631
|
const version = packageVersion();
|
|
621
632
|
if (!options.outDir)
|
|
@@ -645,6 +656,8 @@ async function check(args) {
|
|
|
645
656
|
couldNotRun,
|
|
646
657
|
});
|
|
647
658
|
fs.writeFileSync(path.join(outDir, REPLAY_FILE), html);
|
|
659
|
+
if (template)
|
|
660
|
+
reportFile = writeTemplateReport(outDir, result, template, { version, commit: commitOf(process.env) });
|
|
648
661
|
}
|
|
649
662
|
// On GitHub Actions the verdict also goes on the run's summary page.
|
|
650
663
|
if (process.env.GITHUB_STEP_SUMMARY)
|
|
@@ -659,6 +672,8 @@ async function check(args) {
|
|
|
659
672
|
console.log(`Wrote report.md, check.sarif and check.json to ${outDir}`);
|
|
660
673
|
if (result.replay)
|
|
661
674
|
console.log(`Wrote ${REPLAY_FILE} to ${outDir}, with ${replayFrames(result.replay).length} frame(s) and ${replayVideos(result.replay).length} journey video(s) beside it: open it in a browser to see each step`);
|
|
675
|
+
if (reportFile)
|
|
676
|
+
console.log(`Wrote ${reportFile} to ${outDir}: the test report the template lays out, with a SHA-256 manifest of its evidence`);
|
|
662
677
|
const pictured = result.baselines?.results.filter((r) => r.files).length ?? 0;
|
|
663
678
|
if (pictured > 0)
|
|
664
679
|
console.log(`Wrote the pictures of ${pictured} changed baseline(s) under ${path.join(outDir, VISUAL_DIRNAME)}`);
|
|
@@ -33,6 +33,12 @@ function shotFields(shot) {
|
|
|
33
33
|
return { frame: shot.frame };
|
|
34
34
|
return shot?.pastCap ? { pastCap: true } : {};
|
|
35
35
|
}
|
|
36
|
+
/** What a step should bring about: its own `expected`, else an expect-* step's assertion, else nothing. */
|
|
37
|
+
export function expectedOf(step) {
|
|
38
|
+
if (step.expected !== undefined)
|
|
39
|
+
return step.expected;
|
|
40
|
+
return step.action.startsWith("expect-") ? describeStep(step) : undefined;
|
|
41
|
+
}
|
|
36
42
|
/**
|
|
37
43
|
* Each step of a replayed flow with its caption, its result and its frame.
|
|
38
44
|
* The replay stops at the first step that breaks, so every step before it
|
|
@@ -44,11 +50,12 @@ export function journeySteps(steps, outcome, frames = []) {
|
|
|
44
50
|
return steps.map((step, i) => {
|
|
45
51
|
const n = i + 1;
|
|
46
52
|
const shot = broke === null || n <= broke ? shotFields(frames[i]) : {};
|
|
47
|
-
const
|
|
53
|
+
const expected = expectedOf(step);
|
|
54
|
+
const base = { n, caption: describeStep(step), ...(expected !== undefined ? { expected } : {}), ...shot };
|
|
48
55
|
if (broke === null || n < broke)
|
|
49
56
|
return { ...base, result: "passed" };
|
|
50
57
|
if (n > broke)
|
|
51
|
-
return { n, caption: base.caption, result: "not-run" };
|
|
58
|
+
return { n, caption: base.caption, ...(expected !== undefined ? { expected } : {}), result: "not-run" };
|
|
52
59
|
// `broke` is set only when the outcome is not "passed".
|
|
53
60
|
const failed = outcome;
|
|
54
61
|
return { ...base, result: failed.status, reason: failed.reason, path: failed.path };
|
|
@@ -59,6 +66,8 @@ export function journeyOf(flow, outcome, frames) {
|
|
|
59
66
|
return {
|
|
60
67
|
name: flow.name,
|
|
61
68
|
file: flow.file,
|
|
69
|
+
...(flow.id !== undefined ? { id: flow.id } : {}),
|
|
70
|
+
...(flow.requirements !== undefined ? { requirements: [...flow.requirements] } : {}),
|
|
62
71
|
status: outcome.status,
|
|
63
72
|
steps: journeySteps(flow.steps, outcome, frames),
|
|
64
73
|
...(outcome.status === "passed" ? {} : { firstFailing: outcome.step }),
|
|
@@ -119,6 +128,7 @@ export function redactReplay(replay) {
|
|
|
119
128
|
steps: j.steps.map((s) => ({
|
|
120
129
|
...s,
|
|
121
130
|
caption: redactRoute(s.caption),
|
|
131
|
+
...(s.expected !== undefined ? { expected: redactRoute(s.expected) } : {}),
|
|
122
132
|
...(s.reason !== undefined ? { reason: redactRoute(s.reason) } : {}),
|
|
123
133
|
...(s.path !== undefined ? { path: redactRoute(s.path) } : {}),
|
|
124
134
|
})),
|
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The template-driven test report of a recorded `scenescout check`
|
|
3
|
+
* (`--record --template <file.json>`): the same run the replay page shows, laid
|
|
4
|
+
* out as a document a reviewer signs off on paper or in their own system.
|
|
5
|
+
*
|
|
6
|
+
* The template decides the layout only: the title block, the document ID, the
|
|
7
|
+
* order of the sections, the columns of the test table, the sign-off roles,
|
|
8
|
+
* free text and every word the page uses. The data is the run's: each saved
|
|
9
|
+
* flow is a test, each of its steps a row, and a step's result comes from the
|
|
10
|
+
* replay's assertions alone, never from a model. A step that failed or was
|
|
11
|
+
* refused is a deviation. The manifest lists the SHA-256 of every evidence
|
|
12
|
+
* file, so a reader can tell the frames and videos beside the report are the
|
|
13
|
+
* ones the run wrote.
|
|
14
|
+
*
|
|
15
|
+
* Everything here but readReportTemplate and evidenceIndex is pure: the same
|
|
16
|
+
* result, template and evidence render the same bytes (a golden file in
|
|
17
|
+
* scripts/report-test.ts holds that). check-run.ts reads the files and writes the page.
|
|
18
|
+
*/
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
import fs from "node:fs";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import { z } from "zod";
|
|
23
|
+
import { REPLAY_FILE, replayFrames, replayVideos } from "./check-replay.js";
|
|
24
|
+
import { summarise } from "./check.js";
|
|
25
|
+
import { parseJsonFile } from "./flow.js";
|
|
26
|
+
import { escapeHtml } from "./replay.js";
|
|
27
|
+
/** The page's generator mark: an earlier run's report is removed or replaced only when it carries it. */
|
|
28
|
+
export const REPORT_GENERATOR = "scenescout-check-report";
|
|
29
|
+
/** The file the report is written to when the template names none. */
|
|
30
|
+
export const DEFAULT_REPORT_FILE = "report.html";
|
|
31
|
+
/** What the cell of a missing value shows. */
|
|
32
|
+
export const MISSING = "—";
|
|
33
|
+
/** A report's file name: plain, in the output folder itself, ending .html. */
|
|
34
|
+
export const REPORT_FILE_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}\.html$/;
|
|
35
|
+
/** A name Windows reserves for a device whatever its extension (CON.html opens the console): never a report's file. */
|
|
36
|
+
export const WINDOWS_DEVICE_RE = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])\./i;
|
|
37
|
+
/** Whether a name may be a report's file: plain, ending .html, not replay.html and not a Windows device name. */
|
|
38
|
+
export function isReportFileName(name) {
|
|
39
|
+
return REPORT_FILE_RE.test(name) && !WINDOWS_DEVICE_RE.test(name) && name.toLowerCase() !== REPLAY_FILE;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The report file an earlier check.json records it wrote (its `testReport`),
|
|
43
|
+
* or null: anything that is not a plain report file name is ignored, so the
|
|
44
|
+
* value can never point outside the output folder or at the replay page.
|
|
45
|
+
*/
|
|
46
|
+
export function recordedReportFile(checkJson) {
|
|
47
|
+
if (typeof checkJson !== "object" || checkJson === null)
|
|
48
|
+
return null;
|
|
49
|
+
const file = checkJson.testReport;
|
|
50
|
+
return typeof file === "string" && isReportFileName(file) ? file : null;
|
|
51
|
+
}
|
|
52
|
+
/** Whether an HTML file is a report a check wrote. */
|
|
53
|
+
export function isGeneratedReport(html) {
|
|
54
|
+
return html.includes(`<meta name="generator" content="${REPORT_GENERATOR}">`);
|
|
55
|
+
}
|
|
56
|
+
// ── the template ────────────────────────────────────────────────────────────
|
|
57
|
+
/** The columns a test table may have, in any order. */
|
|
58
|
+
export const REPORT_COLUMNS = ["testId", "requirements", "step", "expected", "actual", "result", "evidence"];
|
|
59
|
+
/** The sections a template may order. Each data section appears at most once; `text` as often as wanted. */
|
|
60
|
+
export const REPORT_SECTIONS = ["summary", "tests", "deviations", "manifest", "signoff", "text"];
|
|
61
|
+
/** The tokens the title block may use, each replaced by a fact of the run. */
|
|
62
|
+
export const REPORT_TOKENS = {
|
|
63
|
+
date: "the day the run started, YYYY-MM-DD (UTC)",
|
|
64
|
+
time: "the time the run started, HH:MM:SS (UTC)",
|
|
65
|
+
run: "the run's start as YYYYMMDD-HHMMSS (UTC)",
|
|
66
|
+
version: "the SceneScout version",
|
|
67
|
+
commit: "the commit the run was of (GITHUB_SHA), shortened to 12 characters, or — when none",
|
|
68
|
+
};
|
|
69
|
+
/** Every word the page writes that is not data. A template may replace any of them, so it carries its own vocabulary. */
|
|
70
|
+
export const DEFAULT_LABELS = {
|
|
71
|
+
pass: "Pass",
|
|
72
|
+
fail: "Fail",
|
|
73
|
+
refused: "Could not run",
|
|
74
|
+
notRun: "Not run",
|
|
75
|
+
asExpected: "As expected",
|
|
76
|
+
summary: "Summary",
|
|
77
|
+
tests: "Test results",
|
|
78
|
+
deviations: "Deviations",
|
|
79
|
+
manifest: "Evidence manifest",
|
|
80
|
+
signoff: "Sign-off",
|
|
81
|
+
testId: "Test ID",
|
|
82
|
+
requirements: "Requirements",
|
|
83
|
+
step: "Step",
|
|
84
|
+
expected: "Expected result",
|
|
85
|
+
actual: "Actual result",
|
|
86
|
+
result: "Result",
|
|
87
|
+
evidence: "Evidence",
|
|
88
|
+
frame: "Frame after the step",
|
|
89
|
+
video: "Video",
|
|
90
|
+
test: "Test",
|
|
91
|
+
number: "#",
|
|
92
|
+
noDeviations: "No deviations.",
|
|
93
|
+
noTests: "No saved flow ran.",
|
|
94
|
+
overall: "Overall result",
|
|
95
|
+
target: "Target",
|
|
96
|
+
started: "Started",
|
|
97
|
+
ended: "Ended",
|
|
98
|
+
tool: "SceneScout version",
|
|
99
|
+
commit: "Commit",
|
|
100
|
+
testsRun: "Tests",
|
|
101
|
+
testsPassed: "Passed",
|
|
102
|
+
testsFailed: "Failed",
|
|
103
|
+
testsNotRun: "Could not run",
|
|
104
|
+
gateIssues: "Issues failing the check's gate",
|
|
105
|
+
file: "File",
|
|
106
|
+
bytes: "Bytes",
|
|
107
|
+
sha256: "SHA-256",
|
|
108
|
+
notFound: "not found",
|
|
109
|
+
name: "Name",
|
|
110
|
+
role: "Role",
|
|
111
|
+
signature: "Signature",
|
|
112
|
+
signedDate: "Date",
|
|
113
|
+
meaning: "Meaning of signature",
|
|
114
|
+
documentId: "Document ID",
|
|
115
|
+
};
|
|
116
|
+
const text = (what, max) => z.string().trim().min(1, `is ${what}`).max(max, `is at most ${max} characters`);
|
|
117
|
+
const TOKEN_RE = /\{([^{}]*)\}/g;
|
|
118
|
+
/** Text that may use the title block's tokens: every `{…}` must name one. */
|
|
119
|
+
const tokenText = (what, max) => text(what, max).superRefine((value, ctx) => {
|
|
120
|
+
for (const m of value.matchAll(TOKEN_RE)) {
|
|
121
|
+
if (!Object.hasOwn(REPORT_TOKENS, m[1]))
|
|
122
|
+
ctx.addIssue({
|
|
123
|
+
code: z.ZodIssueCode.custom,
|
|
124
|
+
message: `uses {${m[1]}}, which is not a token: use ${Object.keys(REPORT_TOKENS)
|
|
125
|
+
.map((t) => `{${t}}`)
|
|
126
|
+
.join(", ")}`,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
const paragraphs = z.array(text("a paragraph of text", 4000)).max(50, "holds at most 50 paragraphs");
|
|
131
|
+
const heading = text("the section's heading", 200);
|
|
132
|
+
const columnSchema = z.union([z.enum(REPORT_COLUMNS), z.object({ key: z.enum(REPORT_COLUMNS), heading }).strict()], {
|
|
133
|
+
errorMap: () => ({ message: `must be one of ${REPORT_COLUMNS.join(", ")}, or { "key": one of those, "heading": "…" }` }),
|
|
134
|
+
});
|
|
135
|
+
const signoffRole = z.union([text("the role that signs", 200), z.object({ role: text("the role that signs", 200), meaning: text("what the signature means", 500).optional() }).strict()], {
|
|
136
|
+
errorMap: () => ({ message: 'must be the role that signs, e.g. "Reviewer", or { "role": "…", "meaning": "…" }' }),
|
|
137
|
+
});
|
|
138
|
+
const sectionSchema = z.discriminatedUnion("type", [
|
|
139
|
+
z.object({ type: z.literal("summary"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
|
|
140
|
+
z
|
|
141
|
+
.object({
|
|
142
|
+
type: z.literal("tests"),
|
|
143
|
+
heading: heading.optional(),
|
|
144
|
+
paragraphs: paragraphs.optional(),
|
|
145
|
+
columns: z
|
|
146
|
+
.array(columnSchema)
|
|
147
|
+
.min(1, "needs at least one column")
|
|
148
|
+
.refine((cols) => new Set(cols.map((c) => (typeof c === "string" ? c : c.key))).size === cols.length, { message: "names a column twice" }),
|
|
149
|
+
})
|
|
150
|
+
.strict(),
|
|
151
|
+
z.object({ type: z.literal("deviations"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
|
|
152
|
+
z.object({ type: z.literal("manifest"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
|
|
153
|
+
z
|
|
154
|
+
.object({
|
|
155
|
+
type: z.literal("signoff"),
|
|
156
|
+
heading: heading.optional(),
|
|
157
|
+
paragraphs: paragraphs.optional(),
|
|
158
|
+
roles: z.array(signoffRole).min(1, "needs at least one role").max(20, "holds at most 20 roles"),
|
|
159
|
+
})
|
|
160
|
+
.strict(),
|
|
161
|
+
z.object({ type: z.literal("text"), heading: heading.optional(), paragraphs: paragraphs.min(1, "needs at least one paragraph") }).strict(),
|
|
162
|
+
]);
|
|
163
|
+
const labelsSchema = z
|
|
164
|
+
.object(Object.fromEntries(Object.keys(DEFAULT_LABELS).map((k) => [k, text("the word to use", 200).optional()])))
|
|
165
|
+
.strict();
|
|
166
|
+
const templateSchema = z
|
|
167
|
+
.object({
|
|
168
|
+
/** The file name the report is written to, beside replay.html. */
|
|
169
|
+
file: z
|
|
170
|
+
.string()
|
|
171
|
+
.regex(REPORT_FILE_RE, "must be a plain file name ending .html, e.g. test-report.html")
|
|
172
|
+
.refine((f) => f.toLowerCase() !== REPLAY_FILE, { message: `must not be ${REPLAY_FILE}, which the replay page is written to` })
|
|
173
|
+
.refine((f) => !WINDOWS_DEVICE_RE.test(f), { message: "must not be a name Windows reserves for a device, such as CON.html or NUL.html" })
|
|
174
|
+
.optional(),
|
|
175
|
+
title: tokenText("the document's title", 300),
|
|
176
|
+
subtitle: tokenText("a line under the title", 300).optional(),
|
|
177
|
+
documentId: tokenText("the document ID pattern, e.g. TR-{run}", 200).optional(),
|
|
178
|
+
fields: z
|
|
179
|
+
.array(z.object({ label: text("the field's label", 200), value: tokenText("the field's value", 500) }).strict())
|
|
180
|
+
.max(20, "holds at most 20 fields")
|
|
181
|
+
.optional(),
|
|
182
|
+
labels: labelsSchema.optional(),
|
|
183
|
+
sections: z
|
|
184
|
+
.array(sectionSchema)
|
|
185
|
+
.min(1, "needs at least one section")
|
|
186
|
+
.max(50, "holds at most 50 sections")
|
|
187
|
+
.superRefine((sections, ctx) => {
|
|
188
|
+
const seen = new Set();
|
|
189
|
+
sections.forEach((s, i) => {
|
|
190
|
+
if (s.type !== "text" && seen.has(s.type))
|
|
191
|
+
ctx.addIssue({
|
|
192
|
+
code: z.ZodIssueCode.custom,
|
|
193
|
+
path: [i, "type"],
|
|
194
|
+
message: `"${s.type}" appears more than once: each section but text appears at most once`,
|
|
195
|
+
});
|
|
196
|
+
seen.add(s.type);
|
|
197
|
+
});
|
|
198
|
+
}),
|
|
199
|
+
})
|
|
200
|
+
.strict();
|
|
201
|
+
/** Validate a template's text. Every mistake names the file and the field, as a flow's do. */
|
|
202
|
+
export function parseReportTemplate(raw, file) {
|
|
203
|
+
const parsed = parseJsonFile(raw, file, templateSchema);
|
|
204
|
+
if (!parsed.ok) {
|
|
205
|
+
// A section's discriminator is "type", not a flow's "action": say so in its own words.
|
|
206
|
+
return { ok: false, error: parsed.error.replace(/must be one of navigate, [^;]*/g, `must be one of ${REPORT_SECTIONS.join(", ")}`) };
|
|
207
|
+
}
|
|
208
|
+
return { ok: true, template: parsed.data };
|
|
209
|
+
}
|
|
210
|
+
/** Read and validate a template file, or throw a sentence naming it. */
|
|
211
|
+
export function readReportTemplate(file) {
|
|
212
|
+
let raw;
|
|
213
|
+
try {
|
|
214
|
+
raw = fs.readFileSync(file, "utf8");
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
throw new Error(`--template: cannot read ${file} (${err instanceof Error ? err.message : String(err)})`);
|
|
218
|
+
}
|
|
219
|
+
const parsed = parseReportTemplate(raw, file);
|
|
220
|
+
if (!parsed.ok)
|
|
221
|
+
throw new Error(`--template: ${parsed.error}`);
|
|
222
|
+
return parsed.template;
|
|
223
|
+
}
|
|
224
|
+
/** The file the template's report is written to. */
|
|
225
|
+
export function reportFileOf(template) {
|
|
226
|
+
return template.file ?? DEFAULT_REPORT_FILE;
|
|
227
|
+
}
|
|
228
|
+
/** Why --template cannot be used with these options, or null: the report renders a recorded run, so recording must be on. */
|
|
229
|
+
export function templateRecordError(options) {
|
|
230
|
+
if (options.template === undefined || options.record === true)
|
|
231
|
+
return null;
|
|
232
|
+
return "--template renders the report from a recorded check's frames: add --record (or set SCENESCOUT_RECORD=on)";
|
|
233
|
+
}
|
|
234
|
+
/** The files a recorded check's report vouches for, in this order: check.json, the replay page, every frame and every video. */
|
|
235
|
+
export function evidenceFiles(result) {
|
|
236
|
+
if (!result.replay)
|
|
237
|
+
return ["check.json"];
|
|
238
|
+
return ["check.json", REPLAY_FILE, ...replayFrames(result.replay), ...replayVideos(result.replay)];
|
|
239
|
+
}
|
|
240
|
+
/** Hash each file beside the report. A file that is not there is listed as missing rather than left out. */
|
|
241
|
+
export function evidenceIndex(outDir, files) {
|
|
242
|
+
return files.map((rel) => {
|
|
243
|
+
try {
|
|
244
|
+
const data = fs.readFileSync(path.join(outDir, ...rel.split("/")));
|
|
245
|
+
return { path: rel, bytes: data.length, sha256: createHash("sha256").update(data).digest("hex") };
|
|
246
|
+
}
|
|
247
|
+
catch (err) {
|
|
248
|
+
if (err.code === "ENOENT")
|
|
249
|
+
return { path: rel, bytes: null, sha256: null };
|
|
250
|
+
throw err;
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/** Every test in the order the replay shows them: the check's own session first, then each role. */
|
|
255
|
+
function testsOf(result) {
|
|
256
|
+
return (result.replay?.roles ?? []).flatMap((r) => r.journeys.map((journey) => ({ journey, role: r.own ? null : r.role })));
|
|
257
|
+
}
|
|
258
|
+
function tokensOf(result, meta) {
|
|
259
|
+
const started = result.replay?.startedAt ?? result.generatedAt;
|
|
260
|
+
const date = started.slice(0, 10);
|
|
261
|
+
const time = started.slice(11, 19);
|
|
262
|
+
return {
|
|
263
|
+
date,
|
|
264
|
+
time,
|
|
265
|
+
run: `${date.replace(/-/g, "")}-${time.replace(/:/g, "")}`,
|
|
266
|
+
version: meta.version,
|
|
267
|
+
commit: meta.commit ? meta.commit.slice(0, 12) : MISSING,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
function fill(value, tokens) {
|
|
271
|
+
return value.replace(TOKEN_RE, (whole, name) => (Object.hasOwn(tokens, name) ? tokens[name] : whole));
|
|
272
|
+
}
|
|
273
|
+
/** A UTC time as the page shows it. */
|
|
274
|
+
function stamp(iso) {
|
|
275
|
+
return iso.length >= 19 ? `${iso.slice(0, 10)} ${iso.slice(11, 19)} UTC` : iso;
|
|
276
|
+
}
|
|
277
|
+
/** Escaped text, or the missing mark. */
|
|
278
|
+
function cell(value) {
|
|
279
|
+
return value === undefined || value === null || value === "" ? MISSING : escapeHtml(value);
|
|
280
|
+
}
|
|
281
|
+
function resultWord(result, L) {
|
|
282
|
+
return result === "passed" ? L.pass : result === "failed" ? L.fail : result === "refused" ? L.refused : L.notRun;
|
|
283
|
+
}
|
|
284
|
+
/** What a step did, in words: as expected when it passed, why when it did not, nothing when it never ran. */
|
|
285
|
+
function actualOf(step, L) {
|
|
286
|
+
if (step.result === "passed")
|
|
287
|
+
return L.asExpected;
|
|
288
|
+
if (step.result === "not-run")
|
|
289
|
+
return undefined;
|
|
290
|
+
return [step.reason, step.path ? `on ${step.path}` : ""].filter(Boolean).join(" ") || undefined;
|
|
291
|
+
}
|
|
292
|
+
function paragraphsHtml(list) {
|
|
293
|
+
return (list ?? []).map((p) => `<p>${escapeHtml(p)}</p>`).join("");
|
|
294
|
+
}
|
|
295
|
+
function sectionHead(section, fallback) {
|
|
296
|
+
return `<h2>${escapeHtml(section.heading ?? fallback)}</h2>${paragraphsHtml(section.paragraphs)}`;
|
|
297
|
+
}
|
|
298
|
+
function frameCellHtml(step, L) {
|
|
299
|
+
if (!step.frame)
|
|
300
|
+
return MISSING;
|
|
301
|
+
const src = escapeHtml(step.frame);
|
|
302
|
+
return `<a class="frame" href="${src}" target="_blank" rel="noreferrer" data-testid="report-frame-open"><img loading="lazy" src="${src}" alt="${escapeHtml(`${L.frame}: ${step.caption}`)}"><span>${escapeHtml(step.frame)}</span></a>`;
|
|
303
|
+
}
|
|
304
|
+
function summaryHtml(section, result, meta, L) {
|
|
305
|
+
const tests = testsOf(result);
|
|
306
|
+
const { passed, couldNotRun, failing } = summarise(result);
|
|
307
|
+
const count = (status) => tests.filter((t) => t.journey.status === status).length;
|
|
308
|
+
// A test that failed fails the report even when the gate let it through (--fail-on never): a pass here is a pass of every test.
|
|
309
|
+
const green = couldNotRun === 0 && passed && count("failed") === 0;
|
|
310
|
+
const overall = couldNotRun > 0 ? L.refused : green ? L.pass : L.fail;
|
|
311
|
+
const row = (term, value, cls = "") => `<tr${cls ? ` class="${cls}"` : ""}><th scope="row">${escapeHtml(term)}</th><td>${value}</td></tr>`;
|
|
312
|
+
const overallClass = green ? "pass" : "fail";
|
|
313
|
+
return (`<section class="summary" data-section="summary">${sectionHead(section, L.summary)}<table class="facts">` +
|
|
314
|
+
row(L.overall, `<span class="verdict ${overallClass}" data-testid="report-verdict">${escapeHtml(overall)}</span>`) +
|
|
315
|
+
row(L.target, cell(result.url)) +
|
|
316
|
+
row(L.started, cell(stamp(result.replay?.startedAt ?? result.generatedAt))) +
|
|
317
|
+
row(L.ended, cell(stamp(result.generatedAt))) +
|
|
318
|
+
row(L.tool, cell(meta.version)) +
|
|
319
|
+
row(L.commit, cell(meta.commit)) +
|
|
320
|
+
row(L.testsRun, String(tests.length)) +
|
|
321
|
+
row(L.testsPassed, String(count("passed"))) +
|
|
322
|
+
row(L.testsFailed, String(count("failed"))) +
|
|
323
|
+
row(L.testsNotRun, String(count("refused"))) +
|
|
324
|
+
row(L.deviations, String(deviationsOf(result).length)) +
|
|
325
|
+
row(L.gateIssues, String(failing)) +
|
|
326
|
+
`</table></section>`);
|
|
327
|
+
}
|
|
328
|
+
function columnsOf(section, L) {
|
|
329
|
+
return section.columns.map((c) => (typeof c === "string" ? { key: c, heading: L[c] } : { key: c.key, heading: c.heading }));
|
|
330
|
+
}
|
|
331
|
+
function testsHtml(section, result, L) {
|
|
332
|
+
const tests = testsOf(result);
|
|
333
|
+
const columns = columnsOf(section, L);
|
|
334
|
+
const head = `<thead><tr>${columns.map((c) => `<th scope="col" data-column="${c.key}">${escapeHtml(c.heading)}</th>`).join("")}</tr></thead>`;
|
|
335
|
+
const bodies = tests.map(({ journey: j, role }, ti) => {
|
|
336
|
+
const verdictClass = j.status === "passed" ? "pass" : "fail";
|
|
337
|
+
const video = j.video
|
|
338
|
+
? ` <a class="video" href="${escapeHtml(j.video)}" target="_blank" rel="noreferrer" data-testid="report-video-open">${escapeHtml(L.video)}: ${escapeHtml(j.video)}</a>`
|
|
339
|
+
: "";
|
|
340
|
+
const caption = `<tr class="test-head ${verdictClass}"><th colspan="${columns.length}" scope="rowgroup">` +
|
|
341
|
+
`<span class="verdict ${verdictClass}">${escapeHtml(resultWord(j.status, L))}</span> ` +
|
|
342
|
+
`<b>${cell(j.id)}</b> ${escapeHtml(j.name)} <span class="file">${escapeHtml(j.file)}${role === null ? "" : ` · ${escapeHtml(L.role)}: ${escapeHtml(role)}`}</span>${video}</th></tr>`;
|
|
343
|
+
const rows = j.steps.map((s, si) => {
|
|
344
|
+
const cells = columns.map(({ key }) => {
|
|
345
|
+
switch (key) {
|
|
346
|
+
case "testId":
|
|
347
|
+
case "requirements":
|
|
348
|
+
// One cell for the whole test, spanning its steps.
|
|
349
|
+
if (si > 0)
|
|
350
|
+
return "";
|
|
351
|
+
return `<td rowspan="${j.steps.length}" class="span">${key === "testId" ? cell(j.id) : j.requirements && j.requirements.length > 0 ? j.requirements.map(escapeHtml).join(", ") : MISSING}</td>`;
|
|
352
|
+
case "step":
|
|
353
|
+
return `<td class="step"><span class="n">${s.n}.</span> ${escapeHtml(s.caption)}</td>`;
|
|
354
|
+
case "expected":
|
|
355
|
+
return `<td>${cell(s.expected)}</td>`;
|
|
356
|
+
case "actual":
|
|
357
|
+
return `<td>${cell(actualOf(s, L))}</td>`;
|
|
358
|
+
case "result":
|
|
359
|
+
return `<td class="result r-${s.result}" data-result="${s.result}">${escapeHtml(resultWord(s.result, L))}</td>`;
|
|
360
|
+
case "evidence":
|
|
361
|
+
return `<td class="evidence">${frameCellHtml(s, L)}</td>`;
|
|
362
|
+
}
|
|
363
|
+
});
|
|
364
|
+
return `<tr class="${s.result}">${cells.join("")}</tr>`;
|
|
365
|
+
});
|
|
366
|
+
return `<tbody id="test-${ti + 1}" data-status="${j.status}">${caption}${rows.join("")}</tbody>`;
|
|
367
|
+
});
|
|
368
|
+
return (`<section class="tests" data-section="tests">${sectionHead(section, L.tests)}` +
|
|
369
|
+
(tests.length === 0 ? `<p class="none">${escapeHtml(L.noTests)}</p>` : `<table class="grid">${head}${bodies.join("")}</table>`) +
|
|
370
|
+
`</section>`);
|
|
371
|
+
}
|
|
372
|
+
/** Every step that failed or was refused, in the order of the tests. */
|
|
373
|
+
export function deviationsOf(result, labels = {}) {
|
|
374
|
+
const L = { ...DEFAULT_LABELS, ...labels };
|
|
375
|
+
return testsOf(result).flatMap(({ journey: j }) => j.steps.flatMap((s) => s.result === "failed" || s.result === "refused"
|
|
376
|
+
? [
|
|
377
|
+
{
|
|
378
|
+
test: j.name,
|
|
379
|
+
...(j.id !== undefined ? { testId: j.id } : {}),
|
|
380
|
+
step: s.n,
|
|
381
|
+
caption: s.caption,
|
|
382
|
+
...(s.expected !== undefined ? { expected: s.expected } : {}),
|
|
383
|
+
...(actualOf(s, L) !== undefined ? { actual: actualOf(s, L) } : {}),
|
|
384
|
+
result: s.result,
|
|
385
|
+
},
|
|
386
|
+
]
|
|
387
|
+
: []));
|
|
388
|
+
}
|
|
389
|
+
function deviationsHtml(section, result, L) {
|
|
390
|
+
const list = deviationsOf(result, L);
|
|
391
|
+
const body = list.length === 0
|
|
392
|
+
? `<p class="none" data-testid="report-no-deviations">${escapeHtml(L.noDeviations)}</p>`
|
|
393
|
+
: `<table class="grid"><thead><tr><th scope="col">${escapeHtml(L.number)}</th><th scope="col">${escapeHtml(L.testId)}</th><th scope="col">${escapeHtml(L.test)}</th><th scope="col">${escapeHtml(L.step)}</th><th scope="col">${escapeHtml(L.expected)}</th><th scope="col">${escapeHtml(L.actual)}</th><th scope="col">${escapeHtml(L.result)}</th></tr></thead><tbody>` +
|
|
394
|
+
list
|
|
395
|
+
.map((d, i) => `<tr class="deviation" data-testid="report-deviation"><td>${i + 1}</td><td>${cell(d.testId)}</td><td>${escapeHtml(d.test)}</td><td><span class="n">${d.step}.</span> ${escapeHtml(d.caption)}</td><td>${cell(d.expected)}</td><td>${cell(d.actual)}</td><td>${escapeHtml(resultWord(d.result, L))}</td></tr>`)
|
|
396
|
+
.join("") +
|
|
397
|
+
`</tbody></table>`;
|
|
398
|
+
return `<section class="deviations" data-section="deviations">${sectionHead(section, L.deviations)}${body}</section>`;
|
|
399
|
+
}
|
|
400
|
+
function manifestHtml(section, result, meta, evidence, L) {
|
|
401
|
+
const facts = `<table class="facts">` +
|
|
402
|
+
`<tr><th scope="row">${escapeHtml(L.tool)}</th><td>${cell(meta.version)}</td></tr>` +
|
|
403
|
+
`<tr><th scope="row">${escapeHtml(L.target)}</th><td>${cell(result.url)}</td></tr>` +
|
|
404
|
+
`<tr><th scope="row">${escapeHtml(L.started)}</th><td>${cell(stamp(result.replay?.startedAt ?? result.generatedAt))}</td></tr>` +
|
|
405
|
+
`<tr><th scope="row">${escapeHtml(L.ended)}</th><td>${cell(stamp(result.generatedAt))}</td></tr>` +
|
|
406
|
+
`</table>`;
|
|
407
|
+
const rows = evidence
|
|
408
|
+
.map((e) => `<tr data-testid="report-manifest-entry"><td class="path">${escapeHtml(e.path)}</td><td>${e.bytes === null ? escapeHtml(L.notFound) : e.bytes}</td><td class="hash">${e.sha256 === null ? escapeHtml(L.notFound) : escapeHtml(e.sha256)}</td></tr>`)
|
|
409
|
+
.join("");
|
|
410
|
+
return (`<section class="manifest" data-section="manifest">${sectionHead(section, L.manifest)}${facts}` +
|
|
411
|
+
`<table class="grid"><thead><tr><th scope="col">${escapeHtml(L.file)}</th><th scope="col">${escapeHtml(L.bytes)}</th><th scope="col">${escapeHtml(L.sha256)}</th></tr></thead><tbody>${rows}</tbody></table></section>`);
|
|
412
|
+
}
|
|
413
|
+
function signoffHtml(section, L) {
|
|
414
|
+
const rows = section.roles
|
|
415
|
+
.map((r) => {
|
|
416
|
+
const role = typeof r === "string" ? r : r.role;
|
|
417
|
+
const meaning = typeof r === "string" ? undefined : r.meaning;
|
|
418
|
+
return `<tr class="signature"><td>${escapeHtml(role)}</td><td class="blank"></td><td class="blank"></td><td class="blank"></td><td>${meaning === undefined ? "" : escapeHtml(meaning)}</td></tr>`;
|
|
419
|
+
})
|
|
420
|
+
.join("");
|
|
421
|
+
return (`<section class="signoff" data-section="signoff">${sectionHead(section, L.signoff)}` +
|
|
422
|
+
`<table class="grid sign"><thead><tr><th scope="col">${escapeHtml(L.role)}</th><th scope="col">${escapeHtml(L.name)}</th><th scope="col">${escapeHtml(L.signature)}</th><th scope="col">${escapeHtml(L.signedDate)}</th><th scope="col">${escapeHtml(L.meaning)}</th></tr></thead><tbody>${rows}</tbody></table></section>`);
|
|
423
|
+
}
|
|
424
|
+
const STYLE = `
|
|
425
|
+
:root { color-scheme: light; --line:#c9ced6; --text:#15181d; --muted:#5d6673; --pass:#047857; --pass-bg:#d1fae5; --fail:#b91c1c; --fail-bg:#fee2e2; --head:#eef0f3; }
|
|
426
|
+
* { box-sizing:border-box; }
|
|
427
|
+
body { margin:0; background:#fff; color:var(--text); font:14px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif; }
|
|
428
|
+
main { max-width:1100px; margin:0 auto; padding:24px 16px 64px; }
|
|
429
|
+
header.title { border-bottom:2px solid var(--text); padding-bottom:12px; margin-bottom:8px; }
|
|
430
|
+
header.title h1 { font-size:22px; margin:0 0 4px; }
|
|
431
|
+
header.title .subtitle { margin:0 0 8px; color:var(--muted); }
|
|
432
|
+
h2 { font-size:17px; margin:28px 0 8px; border-bottom:1px solid var(--line); padding-bottom:4px; }
|
|
433
|
+
table { border-collapse:collapse; width:100%; margin:8px 0; }
|
|
434
|
+
th, td { border:1px solid var(--line); padding:5px 8px; text-align:left; vertical-align:top; overflow-wrap:anywhere; }
|
|
435
|
+
thead th, table.facts th { background:var(--head); }
|
|
436
|
+
table.facts { width:auto; min-width:50%; }
|
|
437
|
+
tr.test-head th { background:var(--head); font-weight:400; }
|
|
438
|
+
.verdict { display:inline-block; padding:0 8px; border-radius:4px; font-weight:700; }
|
|
439
|
+
.verdict.pass { color:var(--pass); background:var(--pass-bg); } .verdict.fail { color:var(--fail); background:var(--fail-bg); }
|
|
440
|
+
.file { color:var(--muted); font-size:12px; }
|
|
441
|
+
.n { color:var(--muted); font-weight:700; }
|
|
442
|
+
td.step { font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; }
|
|
443
|
+
td.r-passed { color:var(--pass); font-weight:700; } td.r-failed, td.r-refused { color:var(--fail); font-weight:700; } td.r-not-run { color:var(--muted); }
|
|
444
|
+
tr.failed td, tr.refused td { background:var(--fail-bg); }
|
|
445
|
+
a.frame { display:block; color:var(--muted); font-size:11px; }
|
|
446
|
+
a.frame img { display:block; max-width:220px; max-height:150px; object-fit:cover; object-position:top; border:1px solid var(--line); }
|
|
447
|
+
td.hash, td.path { font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; }
|
|
448
|
+
table.sign td.blank { height:2.6em; min-width:8em; }
|
|
449
|
+
.none { color:var(--muted); font-style:italic; }
|
|
450
|
+
@media print { main { max-width:none; padding:0; } a { color:inherit; text-decoration:none; } tbody, tr { break-inside:avoid; } }
|
|
451
|
+
`;
|
|
452
|
+
/**
|
|
453
|
+
* The whole report: one HTML file with no scripts, no event handlers and no
|
|
454
|
+
* external assets. The frames and videos it links sit beside it, as they do
|
|
455
|
+
* for the replay page. Every word from the template and every value from the
|
|
456
|
+
* app is escaped.
|
|
457
|
+
*/
|
|
458
|
+
export function buildTemplateReport(result, template, evidence, meta) {
|
|
459
|
+
const L = { ...DEFAULT_LABELS, ...(template.labels ?? {}) };
|
|
460
|
+
const tokens = tokensOf(result, meta);
|
|
461
|
+
const title = fill(template.title, tokens);
|
|
462
|
+
const fields = [
|
|
463
|
+
...(template.documentId !== undefined ? [{ label: L.documentId, value: fill(template.documentId, tokens) }] : []),
|
|
464
|
+
...(template.fields ?? []).map((f) => ({ label: f.label, value: fill(f.value, tokens) })),
|
|
465
|
+
];
|
|
466
|
+
const sections = template.sections.map((s) => {
|
|
467
|
+
switch (s.type) {
|
|
468
|
+
case "summary":
|
|
469
|
+
return summaryHtml(s, result, meta, L);
|
|
470
|
+
case "tests":
|
|
471
|
+
return testsHtml(s, result, L);
|
|
472
|
+
case "deviations":
|
|
473
|
+
return deviationsHtml(s, result, L);
|
|
474
|
+
case "manifest":
|
|
475
|
+
return manifestHtml(s, result, meta, evidence, L);
|
|
476
|
+
case "signoff":
|
|
477
|
+
return signoffHtml(s, L);
|
|
478
|
+
case "text":
|
|
479
|
+
return `<section class="text" data-section="text">${s.heading !== undefined ? `<h2>${escapeHtml(s.heading)}</h2>` : ""}${paragraphsHtml(s.paragraphs)}</section>`;
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
return `<!doctype html>
|
|
483
|
+
<html lang="en">
|
|
484
|
+
<head>
|
|
485
|
+
<meta charset="utf-8">
|
|
486
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
487
|
+
<meta name="generator" content="${REPORT_GENERATOR}">
|
|
488
|
+
<title>${escapeHtml(title)}</title>
|
|
489
|
+
<style>${STYLE}</style>
|
|
490
|
+
</head>
|
|
491
|
+
<body>
|
|
492
|
+
<main>
|
|
493
|
+
<header class="title">
|
|
494
|
+
<h1 data-testid="report-title">${escapeHtml(title)}</h1>
|
|
495
|
+
${template.subtitle !== undefined ? `<p class="subtitle">${escapeHtml(fill(template.subtitle, tokens))}</p>\n` : ""}${fields.length > 0
|
|
496
|
+
? `<table class="facts">${fields.map((f) => `<tr><th scope="row">${escapeHtml(f.label)}</th><td>${escapeHtml(f.value)}</td></tr>`).join("")}</table>\n`
|
|
497
|
+
: ""}</header>
|
|
498
|
+
${sections.join("\n")}
|
|
499
|
+
</main>
|
|
500
|
+
</body>
|
|
501
|
+
</html>
|
|
502
|
+
`;
|
|
503
|
+
}
|
package/dist/engine/check.js
CHANGED
|
@@ -503,6 +503,7 @@ export const CHECK_OPTION_NAMES = [
|
|
|
503
503
|
"sarif-file-anchor",
|
|
504
504
|
"record",
|
|
505
505
|
"video",
|
|
506
|
+
"template",
|
|
506
507
|
];
|
|
507
508
|
/** Options that may be given alone, meaning on: `--record`, as well as `--record on` and `--record=off`. */
|
|
508
509
|
const SWITCH_OPTIONS = new Set(["record", "video"]);
|
|
@@ -685,6 +686,12 @@ export function parseCheckArgs(args, cwd) {
|
|
|
685
686
|
if (videoRaw !== undefined && !SWITCH_ON.includes(videoRaw) && !SWITCH_OFF.includes(videoRaw)) {
|
|
686
687
|
return { ok: false, error: "--video is on or off, or given alone for on" };
|
|
687
688
|
}
|
|
689
|
+
const template = flags.get("template");
|
|
690
|
+
if (template !== undefined && template.trim() === "")
|
|
691
|
+
return { ok: false, error: "--template needs the path of a report template (JSON)" };
|
|
692
|
+
if (template !== undefined && recordRaw !== undefined && SWITCH_OFF.includes(recordRaw)) {
|
|
693
|
+
return { ok: false, error: "--template renders the report from a recorded check's frames, so it cannot be used with --record off" };
|
|
694
|
+
}
|
|
688
695
|
const resolve = (p) => resolveArgPath(cwd, p);
|
|
689
696
|
return {
|
|
690
697
|
ok: true,
|
|
@@ -713,6 +720,7 @@ export function parseCheckArgs(args, cwd) {
|
|
|
713
720
|
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
714
721
|
...(recordRaw !== undefined ? { record: SWITCH_ON.includes(recordRaw) } : {}),
|
|
715
722
|
...(videoRaw !== undefined && SWITCH_ON.includes(videoRaw) ? { video: true } : {}),
|
|
723
|
+
...(template !== undefined ? { template: resolve(template) } : {}),
|
|
716
724
|
},
|
|
717
725
|
};
|
|
718
726
|
}
|
|
@@ -1178,5 +1186,6 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
1178
1186
|
worthALook: result.worthALook,
|
|
1179
1187
|
// Routes a check with no session was sent to sign-in from: a coverage gap. Absent when the check had a session.
|
|
1180
1188
|
...(result.needsSignIn ? { needsSignIn: result.needsSignIn } : {}),
|
|
1189
|
+
...(result.testReport ? { testReport: result.testReport } : {}),
|
|
1181
1190
|
};
|
|
1182
1191
|
}
|
package/dist/engine/flow.js
CHANGED
|
@@ -205,30 +205,41 @@ const target = z
|
|
|
205
205
|
});
|
|
206
206
|
}
|
|
207
207
|
});
|
|
208
|
+
/**
|
|
209
|
+
* What a step should bring about, in the words of the test it belongs to: the
|
|
210
|
+
* expected result a template-driven test report (`--template`) shows beside
|
|
211
|
+
* the step. Documentation only: it is never checked, so a step passes or
|
|
212
|
+
* fails on its action and its assertions alone.
|
|
213
|
+
*/
|
|
214
|
+
const expected = z.string().trim().min(1, "is the result the step should bring about, in words").max(500).optional();
|
|
208
215
|
/** Most times a repeat step may run its steps. A longer loop is a page that should be reached another way. */
|
|
209
216
|
export const MAX_REPEATS = 100;
|
|
210
217
|
const SINGLE_STEP_SCHEMAS = [
|
|
211
|
-
z.object({ action: z.literal("navigate"), target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
|
|
212
|
-
z.object({ action: z.literal("click"), target }).strict(),
|
|
213
|
-
z.object({ action: z.literal("type"), target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
|
|
214
|
-
z.object({ action: z.literal("select"), target, value: z.string() }).strict(),
|
|
215
|
-
z.object({ action: z.literal("press"), value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
|
|
218
|
+
z.object({ action: z.literal("navigate"), expected, target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
|
|
219
|
+
z.object({ action: z.literal("click"), expected, target }).strict(),
|
|
220
|
+
z.object({ action: z.literal("type"), expected, target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
|
|
221
|
+
z.object({ action: z.literal("select"), expected, target, value: z.string() }).strict(),
|
|
222
|
+
z.object({ action: z.literal("press"), expected, value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
|
|
216
223
|
// A small valid file generated for the input (its kind from `fixture`, else the input's accept attribute), as
|
|
217
224
|
// scout_upload attaches one. `target` is the file input or the control that opens its chooser; absent, the page's only file input.
|
|
218
225
|
z
|
|
219
226
|
.object({
|
|
220
227
|
action: z.literal("upload"),
|
|
228
|
+
expected,
|
|
221
229
|
target: target.optional(),
|
|
222
230
|
fixture: z.enum(FIXTURE_KINDS).optional(),
|
|
223
231
|
name: z.string().min(1).max(120).optional(),
|
|
224
232
|
})
|
|
225
233
|
.strict(),
|
|
226
|
-
z.object({ action: z.literal("expect-text"), text: z.string().min(1, "is the text that must be visible") }).strict(),
|
|
227
|
-
z.object({ action: z.literal("expect-element"), target, state: z.enum(ELEMENT_STATES) }).strict(),
|
|
228
|
-
z
|
|
234
|
+
z.object({ action: z.literal("expect-text"), expected, text: z.string().min(1, "is the text that must be visible") }).strict(),
|
|
235
|
+
z.object({ action: z.literal("expect-element"), expected, target, state: z.enum(ELEMENT_STATES) }).strict(),
|
|
236
|
+
z
|
|
237
|
+
.object({ action: z.literal("expect-url"), expected, pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) })
|
|
238
|
+
.strict(),
|
|
229
239
|
z
|
|
230
240
|
.object({
|
|
231
241
|
action: z.literal("expect-request"),
|
|
242
|
+
expected,
|
|
232
243
|
request: z.string().regex(REQUEST_RE, 'must be a method and a path, e.g. "GET /api/things"'),
|
|
233
244
|
status: z.union([z.number().int().min(100).max(599), z.string().regex(/^[1-5]xx$/, 'must be a status such as 200, or a class such as "2xx"')]),
|
|
234
245
|
})
|
|
@@ -246,6 +257,7 @@ const isRepeatCondition = (s) => s.action === "expect-text" || s.action === "exp
|
|
|
246
257
|
const repeatSchema = z
|
|
247
258
|
.object({
|
|
248
259
|
action: z.literal("repeat"),
|
|
260
|
+
expected,
|
|
249
261
|
steps: z
|
|
250
262
|
.array(singleStepSchema)
|
|
251
263
|
.min(1, "needs at least one step to repeat")
|
|
@@ -262,6 +274,10 @@ const flowSchema = z
|
|
|
262
274
|
.object({
|
|
263
275
|
name: z.string().min(1).max(100).optional(),
|
|
264
276
|
description: z.string().max(500).optional(),
|
|
277
|
+
/** The test's identifier in a template-driven test report (--template). Traceability only: the replay never reads it. */
|
|
278
|
+
id: z.string().trim().min(1, "is the test's identifier").max(100).optional(),
|
|
279
|
+
/** The requirement identifiers the flow covers, shown beside it in a template-driven test report. */
|
|
280
|
+
requirements: z.array(z.string().trim().min(1, "is a requirement identifier").max(100)).max(50, "holds at most 50 identifiers").optional(),
|
|
265
281
|
/**
|
|
266
282
|
* Who walks the flow: a role whose sign-in `scenescout login --role` saved in the project. Absent, the flow runs
|
|
267
283
|
* in the check's own session. Flows run in file-name order, so one role's flow can pick up what another's left.
|
|
@@ -330,6 +346,10 @@ export function parseFlow(text, file) {
|
|
|
330
346
|
const flow = { name: parsed.data.name ?? file.replace(/\.json$/i, ""), file, steps: parsed.data.steps };
|
|
331
347
|
if (parsed.data.role !== undefined)
|
|
332
348
|
flow.role = parsed.data.role;
|
|
349
|
+
if (parsed.data.id !== undefined)
|
|
350
|
+
flow.id = parsed.data.id;
|
|
351
|
+
if (parsed.data.requirements !== undefined)
|
|
352
|
+
flow.requirements = parsed.data.requirements;
|
|
333
353
|
return { ok: true, flow };
|
|
334
354
|
}
|
|
335
355
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.22.0",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
"mcp-check": "npm run build && npm run mcp-check:run",
|
|
74
74
|
"mcp-check:run": "tsx scripts/mcp-check.ts",
|
|
75
75
|
"test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
|
|
76
|
-
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run ci-test && npm run qa-test && npm run export-test && npm run contract-test && npm run pace-test && npm run limits-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run tickets-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run profiles-test && npm run refresh-test && npm run scripted-login-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test && npm run guide-test",
|
|
76
|
+
"test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run report-test && npm run ci-test && npm run qa-test && npm run export-test && npm run contract-test && npm run pace-test && npm run limits-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run tickets-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run profiles-test && npm run refresh-test && npm run scripted-login-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test && npm run guide-test",
|
|
77
77
|
"scan-test": "tsx scripts/scan-test.ts",
|
|
78
78
|
"oracle-test": "tsx --test scripts/oracle-test.ts",
|
|
79
79
|
"policy-test": "tsx --test scripts/policy-test.ts",
|
|
@@ -81,6 +81,7 @@
|
|
|
81
81
|
"dispatch-test": "tsx --test scripts/dispatch-test.ts",
|
|
82
82
|
"design-test": "tsx --test scripts/design-test.ts",
|
|
83
83
|
"check-test": "tsx --test scripts/check-test.ts",
|
|
84
|
+
"report-test": "tsx --test scripts/report-test.ts",
|
|
84
85
|
"ci-test": "tsx --test scripts/ci-test.ts",
|
|
85
86
|
"qa-test": "tsx --test scripts/qa-test.ts",
|
|
86
87
|
"export-test": "tsx --test scripts/export-test.ts",
|