scenescout 3.21.3 → 3.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # scenescout
2
2
 
3
+ ## 3.22.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 072b68b: `scenescout check --record --template <file.json>` (the action's `template` input) also writes the recorded run up as a test report laid out by the template: each saved flow a test and each step a row with its expected and actual result and its frame, failed steps as deviations, a SHA-256 manifest of the evidence and blank sign-off rows. Flows gain optional `id`, `requirements` and a step's `expected` for traceability; an example template is in `examples/report-template.json`.
8
+
3
9
  ## 3.21.3
4
10
 
5
11
  ### Patch Changes
package/README.md CHANGED
@@ -279,7 +279,10 @@ Use SceneScout to test http://localhost:3000, record the run
279
279
  or, on the tool directly, `scout_attach {record: true}`. `SCENESCOUT_RECORD=on` in
280
280
  the server's environment records every run. A CI gate records too:
281
281
  `scenescout check --record` writes `replay.html`, every journey step by step with
282
- its frames ([recording a check](docs/ci.md#recording-a-check)).
282
+ its frames ([recording a check](docs/ci.md#recording-a-check)), and
283
+ `--template <file.json>` also writes it up as a test report laid out as a template
284
+ says, with expected and actual results, deviations, blank sign-off rows and a
285
+ SHA-256 manifest of the evidence ([test reports](docs/ci.md#a-test-report-from-a-template)).
283
286
 
284
287
  Then `scout_report` writes two files side by side in `.scenescout/`:
285
288
  `report.md` as always, and `report.html` — the whole run as one self-contained
package/dist/check-run.js CHANGED
@@ -20,6 +20,7 @@ import { firstLineOf } from "./engine/limits.js";
20
20
  import { isNonPageResource } from "./engine/crawl.js";
21
21
  import { checkRetestPlan, retestResults, wellFormedFindings } from "./engine/verify.js";
22
22
  import { capFrames, isReplayFrameFile, journeyOf, journeyVideoPath, isGeneratedReplay, isJourneyVideoFile, redactReplay, REPLAY_VIDEOS_DIRNAME, replayVideos, REPLAY_FILE, REPLAY_FRAMES_DIRNAME, replayFramePath, replayFrames, replaySessionKey, visitOf, } from "./engine/check-replay.js";
23
+ import { buildTemplateReport, evidenceFiles, evidenceIndex, isGeneratedReport, readReportTemplate, recordedReportFile, reportFileOf, templateRecordError, } from "./engine/check-report.js";
23
24
  /** The baselines folder a check uses: the one --baselines names, else the project's own. */
24
25
  export function baselinesDirOf(options) {
25
26
  return options.baselinesDir ?? defaultBaselinesDir(options.projectDir);
@@ -470,7 +471,34 @@ async function takeBaselines(engine, options, mode, targets, log) {
470
471
  * check, recorded or not, so a page from an earlier run is never read, or
471
472
  * uploaded, as this one's; whatever else the folder holds stays.
472
473
  */
473
- export function clearReplayOutput(outDir) {
474
+ export function clearReplayOutput(outDir, reportFile) {
475
+ // The test report the earlier run's check.json says it wrote (--template), and the one this run's template names:
476
+ // each only when it still carries the mark. A copy renamed or kept under any other name is the project's own.
477
+ const reports = new Set();
478
+ try {
479
+ const earlier = recordedReportFile(JSON.parse(fs.readFileSync(path.join(outDir, "check.json"), "utf8")));
480
+ if (earlier)
481
+ reports.add(earlier);
482
+ }
483
+ catch {
484
+ // No earlier check.json, or one that cannot be read: it names no report to remove.
485
+ }
486
+ if (reportFile)
487
+ reports.add(reportFile);
488
+ for (const name of reports) {
489
+ const at = path.join(outDir, name);
490
+ let text;
491
+ try {
492
+ text = fs.readFileSync(at, "utf8");
493
+ }
494
+ catch (err) {
495
+ if (err.code === "ENOENT")
496
+ continue;
497
+ throw err;
498
+ }
499
+ if (isGeneratedReport(text))
500
+ fs.rmSync(at);
501
+ }
474
502
  // Only a page a check wrote: a file of the same name the project keeps there is its own.
475
503
  const page = path.join(outDir, REPLAY_FILE);
476
504
  let text = null;
@@ -512,6 +540,39 @@ export function clearReplayOutput(outDir) {
512
540
  if (fs.readdirSync(root).length === 0)
513
541
  fs.rmdirSync(root);
514
542
  }
543
+ /**
544
+ * The report template a check was given (--template), read and validated, or
545
+ * null for none. Throws, before the browser starts, when the check is not
546
+ * recorded, when the template is not valid, or when the file it would write is
547
+ * one the project keeps there and a check did not write.
548
+ */
549
+ export function prepareTemplateReport(options, outDir) {
550
+ if (options.template === undefined)
551
+ return null;
552
+ const needsRecord = templateRecordError(options);
553
+ if (needsRecord)
554
+ throw new Error(needsRecord);
555
+ const template = readReportTemplate(options.template);
556
+ const file = path.join(outDir, reportFileOf(template));
557
+ let text = null;
558
+ try {
559
+ text = fs.readFileSync(file, "utf8");
560
+ }
561
+ catch (err) {
562
+ if (err.code !== "ENOENT")
563
+ throw err;
564
+ }
565
+ if (text !== null && !isGeneratedReport(text))
566
+ throw new Error(`${file} is not a report SceneScout wrote, so a check will not overwrite it. Move or rename it, name another file in the template's "file", or pass --out to write the check somewhere else`);
567
+ return template;
568
+ }
569
+ /** Write the template's report beside replay.html, after check.json and the replay page, so the manifest hashes the files as written. Returns its file name. */
570
+ export function writeTemplateReport(outDir, result, template, meta) {
571
+ const file = reportFileOf(template);
572
+ const evidence = evidenceIndex(outDir, evidenceFiles(result));
573
+ fs.writeFileSync(path.join(outDir, file), buildTemplateReport(result, template, evidence, meta));
574
+ return file;
575
+ }
515
576
  /**
516
577
  * Why a recorded check (--record or --video) must not start, or null: a
517
578
  * replay.html in the output folder that a check did not write is the
package/dist/cli.js CHANGED
@@ -21,7 +21,8 @@ import { APPROX_DISK_MB, defaultAttachNote, defaultEngine, launchTarget, parseBr
21
21
  import { CLIENT_LABELS, firstMessageHint, manualFor, parseClients, registerWithClient, vscodeBinary } from "./clients.js";
22
22
  import { CLAUDE_CODE_NOT_NEEDED, CLI_NAME, desktopExtensionRoots, diagnose, doctorAllGood, findDesktopExtension, installClosing, ensureCommand, findOnUserPath, installSkill, isEphemeralRoot, launchCommand, manualRegisterCommand, planCommand, registerMcp, resolveClaudeDir, spawnRunner, } from "./installer.js";
23
23
  import { downloadBrowsers, presentBrowsers } from "./installer.js";
24
- import { baselinesDirOf, clearReplayOutput, replayPageConflict, defaultCheckDir, readCheckInputs, runCheck } from "./check-run.js";
24
+ import { baselinesDirOf, clearReplayOutput, replayPageConflict, defaultCheckDir, prepareTemplateReport, readCheckInputs, runCheck, writeTemplateReport, } from "./check-run.js";
25
+ import { reportFileOf } from "./engine/check-report.js";
25
26
  import { buildCheckReplayHtml, commitOf, REPLAY_FILE, replayFrames, replayVideos } from "./engine/check-replay.js";
26
27
  import { recordChoice } from "./engine/capture.js";
27
28
  import { httpClient, httpJudgeAsk, runCi } from "./ci-run.js";
@@ -112,7 +113,10 @@ Usage:
112
113
  and write replay.html beside the report, role → journey → step (default:
113
114
  SCENESCOUT_RECORD, else off; the frames go in replay-frames/);
114
115
  --video [on|off]: record a WebM of each saved flow, and only of the flows,
115
- into replay-videos/, played on replay.html beside its steps (default off))
116
+ into replay-videos/, played on replay.html beside its steps (default off);
117
+ --template file.json: on a recorded check, also write a test report laid out
118
+ as the template says (report.html unless it names a file), with the frames,
119
+ deviations and a SHA-256 manifest of the evidence; needs --record)
116
120
  Exit code: 0 passed, 1 failed the gate, 2 could not run.
117
121
  scenescout ci <url> An exploratory run with no person present: a model reached through its API
118
122
  drives the tools by the SceneScout method and the run ends in the report.
@@ -580,15 +584,18 @@ async function check(args) {
580
584
  let options = parsed.options;
581
585
  const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
582
586
  let inputs;
587
+ let template = null;
583
588
  try {
584
589
  // --record, else SCENESCOUT_RECORD, else off.
585
590
  options = { ...options, record: recordChoice(options.record, process.env) };
591
+ // --template: read and checked before the browser starts, so a bad template never costs a run.
592
+ template = prepareTemplateReport(options, outDir);
586
593
  // A replay.html the project keeps there is its own: a recorded check refuses to start rather than overwrite it.
587
594
  const conflict = options.record || options.video ? replayPageConflict(outDir) : null;
588
595
  if (conflict)
589
596
  throw new Error(conflict);
590
597
  // A replay page an earlier run left must never be read, or uploaded, as this run's.
591
- clearReplayOutput(outDir);
598
+ clearReplayOutput(outDir, template ? reportFileOf(template) : undefined);
592
599
  inputs = readCheckInputs(options);
593
600
  }
594
601
  catch (err) {
@@ -615,7 +622,11 @@ async function check(args) {
615
622
  console.error(`scenescout check: could not run a saved flow: ${refused}`);
616
623
  process.exit(EXIT.error);
617
624
  }
625
+ // Recorded in check.json, so the next check removes exactly this file and the action finds it.
626
+ if (template && result.replay)
627
+ result = { ...result, testReport: reportFileOf(template) };
618
628
  const markdown = formatCheck(result);
629
+ let reportFile = null;
619
630
  try {
620
631
  const version = packageVersion();
621
632
  if (!options.outDir)
@@ -645,6 +656,8 @@ async function check(args) {
645
656
  couldNotRun,
646
657
  });
647
658
  fs.writeFileSync(path.join(outDir, REPLAY_FILE), html);
659
+ if (template)
660
+ reportFile = writeTemplateReport(outDir, result, template, { version, commit: commitOf(process.env) });
648
661
  }
649
662
  // On GitHub Actions the verdict also goes on the run's summary page.
650
663
  if (process.env.GITHUB_STEP_SUMMARY)
@@ -659,6 +672,8 @@ async function check(args) {
659
672
  console.log(`Wrote report.md, check.sarif and check.json to ${outDir}`);
660
673
  if (result.replay)
661
674
  console.log(`Wrote ${REPLAY_FILE} to ${outDir}, with ${replayFrames(result.replay).length} frame(s) and ${replayVideos(result.replay).length} journey video(s) beside it: open it in a browser to see each step`);
675
+ if (reportFile)
676
+ console.log(`Wrote ${reportFile} to ${outDir}: the test report the template lays out, with a SHA-256 manifest of its evidence`);
662
677
  const pictured = result.baselines?.results.filter((r) => r.files).length ?? 0;
663
678
  if (pictured > 0)
664
679
  console.log(`Wrote the pictures of ${pictured} changed baseline(s) under ${path.join(outDir, VISUAL_DIRNAME)}`);
@@ -33,6 +33,12 @@ function shotFields(shot) {
33
33
  return { frame: shot.frame };
34
34
  return shot?.pastCap ? { pastCap: true } : {};
35
35
  }
36
+ /** What a step should bring about: its own `expected`, else an expect-* step's assertion, else nothing. */
37
+ export function expectedOf(step) {
38
+ if (step.expected !== undefined)
39
+ return step.expected;
40
+ return step.action.startsWith("expect-") ? describeStep(step) : undefined;
41
+ }
36
42
  /**
37
43
  * Each step of a replayed flow with its caption, its result and its frame.
38
44
  * The replay stops at the first step that breaks, so every step before it
@@ -44,11 +50,12 @@ export function journeySteps(steps, outcome, frames = []) {
44
50
  return steps.map((step, i) => {
45
51
  const n = i + 1;
46
52
  const shot = broke === null || n <= broke ? shotFields(frames[i]) : {};
47
- const base = { n, caption: describeStep(step), ...shot };
53
+ const expected = expectedOf(step);
54
+ const base = { n, caption: describeStep(step), ...(expected !== undefined ? { expected } : {}), ...shot };
48
55
  if (broke === null || n < broke)
49
56
  return { ...base, result: "passed" };
50
57
  if (n > broke)
51
- return { n, caption: base.caption, result: "not-run" };
58
+ return { n, caption: base.caption, ...(expected !== undefined ? { expected } : {}), result: "not-run" };
52
59
  // `broke` is set only when the outcome is not "passed".
53
60
  const failed = outcome;
54
61
  return { ...base, result: failed.status, reason: failed.reason, path: failed.path };
@@ -59,6 +66,8 @@ export function journeyOf(flow, outcome, frames) {
59
66
  return {
60
67
  name: flow.name,
61
68
  file: flow.file,
69
+ ...(flow.id !== undefined ? { id: flow.id } : {}),
70
+ ...(flow.requirements !== undefined ? { requirements: [...flow.requirements] } : {}),
62
71
  status: outcome.status,
63
72
  steps: journeySteps(flow.steps, outcome, frames),
64
73
  ...(outcome.status === "passed" ? {} : { firstFailing: outcome.step }),
@@ -119,6 +128,7 @@ export function redactReplay(replay) {
119
128
  steps: j.steps.map((s) => ({
120
129
  ...s,
121
130
  caption: redactRoute(s.caption),
131
+ ...(s.expected !== undefined ? { expected: redactRoute(s.expected) } : {}),
122
132
  ...(s.reason !== undefined ? { reason: redactRoute(s.reason) } : {}),
123
133
  ...(s.path !== undefined ? { path: redactRoute(s.path) } : {}),
124
134
  })),
@@ -0,0 +1,503 @@
1
+ /**
2
+ * The template-driven test report of a recorded `scenescout check`
3
+ * (`--record --template <file.json>`): the same run the replay page shows, laid
4
+ * out as a document a reviewer signs off on paper or in their own system.
5
+ *
6
+ * The template decides the layout only: the title block, the document ID, the
7
+ * order of the sections, the columns of the test table, the sign-off roles,
8
+ * free text and every word the page uses. The data is the run's: each saved
9
+ * flow is a test, each of its steps a row, and a step's result comes from the
10
+ * replay's assertions alone, never from a model. A step that failed or was
11
+ * refused is a deviation. The manifest lists the SHA-256 of every evidence
12
+ * file, so a reader can tell the frames and videos beside the report are the
13
+ * ones the run wrote.
14
+ *
15
+ * Everything here but readReportTemplate and evidenceIndex is pure: the same
16
+ * result, template and evidence render the same bytes (a golden file in
17
+ * scripts/report-test.ts holds that). check-run.ts reads the files and writes the page.
18
+ */
19
+ import { createHash } from "node:crypto";
20
+ import fs from "node:fs";
21
+ import path from "node:path";
22
+ import { z } from "zod";
23
+ import { REPLAY_FILE, replayFrames, replayVideos } from "./check-replay.js";
24
+ import { summarise } from "./check.js";
25
+ import { parseJsonFile } from "./flow.js";
26
+ import { escapeHtml } from "./replay.js";
27
+ /** The page's generator mark: an earlier run's report is removed or replaced only when it carries it. */
28
+ export const REPORT_GENERATOR = "scenescout-check-report";
29
+ /** The file the report is written to when the template names none. */
30
+ export const DEFAULT_REPORT_FILE = "report.html";
31
+ /** What the cell of a missing value shows. */
32
+ export const MISSING = "—";
33
+ /** A report's file name: plain, in the output folder itself, ending .html. */
34
+ export const REPORT_FILE_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}\.html$/;
35
+ /** A name Windows reserves for a device whatever its extension (CON.html opens the console): never a report's file. */
36
+ export const WINDOWS_DEVICE_RE = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])\./i;
37
+ /** Whether a name may be a report's file: plain, ending .html, not replay.html and not a Windows device name. */
38
+ export function isReportFileName(name) {
39
+ return REPORT_FILE_RE.test(name) && !WINDOWS_DEVICE_RE.test(name) && name.toLowerCase() !== REPLAY_FILE;
40
+ }
41
+ /**
42
+ * The report file an earlier check.json records it wrote (its `testReport`),
43
+ * or null: anything that is not a plain report file name is ignored, so the
44
+ * value can never point outside the output folder or at the replay page.
45
+ */
46
+ export function recordedReportFile(checkJson) {
47
+ if (typeof checkJson !== "object" || checkJson === null)
48
+ return null;
49
+ const file = checkJson.testReport;
50
+ return typeof file === "string" && isReportFileName(file) ? file : null;
51
+ }
52
+ /** Whether an HTML file is a report a check wrote. */
53
+ export function isGeneratedReport(html) {
54
+ return html.includes(`<meta name="generator" content="${REPORT_GENERATOR}">`);
55
+ }
56
+ // ── the template ────────────────────────────────────────────────────────────
57
+ /** The columns a test table may have, in any order. */
58
+ export const REPORT_COLUMNS = ["testId", "requirements", "step", "expected", "actual", "result", "evidence"];
59
+ /** The sections a template may order. Each data section appears at most once; `text` as often as wanted. */
60
+ export const REPORT_SECTIONS = ["summary", "tests", "deviations", "manifest", "signoff", "text"];
61
+ /** The tokens the title block may use, each replaced by a fact of the run. */
62
+ export const REPORT_TOKENS = {
63
+ date: "the day the run started, YYYY-MM-DD (UTC)",
64
+ time: "the time the run started, HH:MM:SS (UTC)",
65
+ run: "the run's start as YYYYMMDD-HHMMSS (UTC)",
66
+ version: "the SceneScout version",
67
+ commit: "the commit the run was of (GITHUB_SHA), shortened to 12 characters, or — when none",
68
+ };
69
+ /** Every word the page writes that is not data. A template may replace any of them, so it carries its own vocabulary. */
70
+ export const DEFAULT_LABELS = {
71
+ pass: "Pass",
72
+ fail: "Fail",
73
+ refused: "Could not run",
74
+ notRun: "Not run",
75
+ asExpected: "As expected",
76
+ summary: "Summary",
77
+ tests: "Test results",
78
+ deviations: "Deviations",
79
+ manifest: "Evidence manifest",
80
+ signoff: "Sign-off",
81
+ testId: "Test ID",
82
+ requirements: "Requirements",
83
+ step: "Step",
84
+ expected: "Expected result",
85
+ actual: "Actual result",
86
+ result: "Result",
87
+ evidence: "Evidence",
88
+ frame: "Frame after the step",
89
+ video: "Video",
90
+ test: "Test",
91
+ number: "#",
92
+ noDeviations: "No deviations.",
93
+ noTests: "No saved flow ran.",
94
+ overall: "Overall result",
95
+ target: "Target",
96
+ started: "Started",
97
+ ended: "Ended",
98
+ tool: "SceneScout version",
99
+ commit: "Commit",
100
+ testsRun: "Tests",
101
+ testsPassed: "Passed",
102
+ testsFailed: "Failed",
103
+ testsNotRun: "Could not run",
104
+ gateIssues: "Issues failing the check's gate",
105
+ file: "File",
106
+ bytes: "Bytes",
107
+ sha256: "SHA-256",
108
+ notFound: "not found",
109
+ name: "Name",
110
+ role: "Role",
111
+ signature: "Signature",
112
+ signedDate: "Date",
113
+ meaning: "Meaning of signature",
114
+ documentId: "Document ID",
115
+ };
116
+ const text = (what, max) => z.string().trim().min(1, `is ${what}`).max(max, `is at most ${max} characters`);
117
+ const TOKEN_RE = /\{([^{}]*)\}/g;
118
+ /** Text that may use the title block's tokens: every `{…}` must name one. */
119
+ const tokenText = (what, max) => text(what, max).superRefine((value, ctx) => {
120
+ for (const m of value.matchAll(TOKEN_RE)) {
121
+ if (!Object.hasOwn(REPORT_TOKENS, m[1]))
122
+ ctx.addIssue({
123
+ code: z.ZodIssueCode.custom,
124
+ message: `uses {${m[1]}}, which is not a token: use ${Object.keys(REPORT_TOKENS)
125
+ .map((t) => `{${t}}`)
126
+ .join(", ")}`,
127
+ });
128
+ }
129
+ });
130
+ const paragraphs = z.array(text("a paragraph of text", 4000)).max(50, "holds at most 50 paragraphs");
131
+ const heading = text("the section's heading", 200);
132
+ const columnSchema = z.union([z.enum(REPORT_COLUMNS), z.object({ key: z.enum(REPORT_COLUMNS), heading }).strict()], {
133
+ errorMap: () => ({ message: `must be one of ${REPORT_COLUMNS.join(", ")}, or { "key": one of those, "heading": "…" }` }),
134
+ });
135
+ const signoffRole = z.union([text("the role that signs", 200), z.object({ role: text("the role that signs", 200), meaning: text("what the signature means", 500).optional() }).strict()], {
136
+ errorMap: () => ({ message: 'must be the role that signs, e.g. "Reviewer", or { "role": "…", "meaning": "…" }' }),
137
+ });
138
+ const sectionSchema = z.discriminatedUnion("type", [
139
+ z.object({ type: z.literal("summary"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
140
+ z
141
+ .object({
142
+ type: z.literal("tests"),
143
+ heading: heading.optional(),
144
+ paragraphs: paragraphs.optional(),
145
+ columns: z
146
+ .array(columnSchema)
147
+ .min(1, "needs at least one column")
148
+ .refine((cols) => new Set(cols.map((c) => (typeof c === "string" ? c : c.key))).size === cols.length, { message: "names a column twice" }),
149
+ })
150
+ .strict(),
151
+ z.object({ type: z.literal("deviations"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
152
+ z.object({ type: z.literal("manifest"), heading: heading.optional(), paragraphs: paragraphs.optional() }).strict(),
153
+ z
154
+ .object({
155
+ type: z.literal("signoff"),
156
+ heading: heading.optional(),
157
+ paragraphs: paragraphs.optional(),
158
+ roles: z.array(signoffRole).min(1, "needs at least one role").max(20, "holds at most 20 roles"),
159
+ })
160
+ .strict(),
161
+ z.object({ type: z.literal("text"), heading: heading.optional(), paragraphs: paragraphs.min(1, "needs at least one paragraph") }).strict(),
162
+ ]);
163
+ const labelsSchema = z
164
+ .object(Object.fromEntries(Object.keys(DEFAULT_LABELS).map((k) => [k, text("the word to use", 200).optional()])))
165
+ .strict();
166
+ const templateSchema = z
167
+ .object({
168
+ /** The file name the report is written to, beside replay.html. */
169
+ file: z
170
+ .string()
171
+ .regex(REPORT_FILE_RE, "must be a plain file name ending .html, e.g. test-report.html")
172
+ .refine((f) => f.toLowerCase() !== REPLAY_FILE, { message: `must not be ${REPLAY_FILE}, which the replay page is written to` })
173
+ .refine((f) => !WINDOWS_DEVICE_RE.test(f), { message: "must not be a name Windows reserves for a device, such as CON.html or NUL.html" })
174
+ .optional(),
175
+ title: tokenText("the document's title", 300),
176
+ subtitle: tokenText("a line under the title", 300).optional(),
177
+ documentId: tokenText("the document ID pattern, e.g. TR-{run}", 200).optional(),
178
+ fields: z
179
+ .array(z.object({ label: text("the field's label", 200), value: tokenText("the field's value", 500) }).strict())
180
+ .max(20, "holds at most 20 fields")
181
+ .optional(),
182
+ labels: labelsSchema.optional(),
183
+ sections: z
184
+ .array(sectionSchema)
185
+ .min(1, "needs at least one section")
186
+ .max(50, "holds at most 50 sections")
187
+ .superRefine((sections, ctx) => {
188
+ const seen = new Set();
189
+ sections.forEach((s, i) => {
190
+ if (s.type !== "text" && seen.has(s.type))
191
+ ctx.addIssue({
192
+ code: z.ZodIssueCode.custom,
193
+ path: [i, "type"],
194
+ message: `"${s.type}" appears more than once: each section but text appears at most once`,
195
+ });
196
+ seen.add(s.type);
197
+ });
198
+ }),
199
+ })
200
+ .strict();
201
+ /** Validate a template's text. Every mistake names the file and the field, as a flow's do. */
202
+ export function parseReportTemplate(raw, file) {
203
+ const parsed = parseJsonFile(raw, file, templateSchema);
204
+ if (!parsed.ok) {
205
+ // A section's discriminator is "type", not a flow's "action": say so in its own words.
206
+ return { ok: false, error: parsed.error.replace(/must be one of navigate, [^;]*/g, `must be one of ${REPORT_SECTIONS.join(", ")}`) };
207
+ }
208
+ return { ok: true, template: parsed.data };
209
+ }
210
+ /** Read and validate a template file, or throw a sentence naming it. */
211
+ export function readReportTemplate(file) {
212
+ let raw;
213
+ try {
214
+ raw = fs.readFileSync(file, "utf8");
215
+ }
216
+ catch (err) {
217
+ throw new Error(`--template: cannot read ${file} (${err instanceof Error ? err.message : String(err)})`);
218
+ }
219
+ const parsed = parseReportTemplate(raw, file);
220
+ if (!parsed.ok)
221
+ throw new Error(`--template: ${parsed.error}`);
222
+ return parsed.template;
223
+ }
224
+ /** The file the template's report is written to. */
225
+ export function reportFileOf(template) {
226
+ return template.file ?? DEFAULT_REPORT_FILE;
227
+ }
228
+ /** Why --template cannot be used with these options, or null: the report renders a recorded run, so recording must be on. */
229
+ export function templateRecordError(options) {
230
+ if (options.template === undefined || options.record === true)
231
+ return null;
232
+ return "--template renders the report from a recorded check's frames: add --record (or set SCENESCOUT_RECORD=on)";
233
+ }
234
+ /** The files a recorded check's report vouches for, in this order: check.json, the replay page, every frame and every video. */
235
+ export function evidenceFiles(result) {
236
+ if (!result.replay)
237
+ return ["check.json"];
238
+ return ["check.json", REPLAY_FILE, ...replayFrames(result.replay), ...replayVideos(result.replay)];
239
+ }
240
+ /** Hash each file beside the report. A file that is not there is listed as missing rather than left out. */
241
+ export function evidenceIndex(outDir, files) {
242
+ return files.map((rel) => {
243
+ try {
244
+ const data = fs.readFileSync(path.join(outDir, ...rel.split("/")));
245
+ return { path: rel, bytes: data.length, sha256: createHash("sha256").update(data).digest("hex") };
246
+ }
247
+ catch (err) {
248
+ if (err.code === "ENOENT")
249
+ return { path: rel, bytes: null, sha256: null };
250
+ throw err;
251
+ }
252
+ });
253
+ }
254
+ /** Every test in the order the replay shows them: the check's own session first, then each role. */
255
+ function testsOf(result) {
256
+ return (result.replay?.roles ?? []).flatMap((r) => r.journeys.map((journey) => ({ journey, role: r.own ? null : r.role })));
257
+ }
258
+ function tokensOf(result, meta) {
259
+ const started = result.replay?.startedAt ?? result.generatedAt;
260
+ const date = started.slice(0, 10);
261
+ const time = started.slice(11, 19);
262
+ return {
263
+ date,
264
+ time,
265
+ run: `${date.replace(/-/g, "")}-${time.replace(/:/g, "")}`,
266
+ version: meta.version,
267
+ commit: meta.commit ? meta.commit.slice(0, 12) : MISSING,
268
+ };
269
+ }
270
+ function fill(value, tokens) {
271
+ return value.replace(TOKEN_RE, (whole, name) => (Object.hasOwn(tokens, name) ? tokens[name] : whole));
272
+ }
273
+ /** A UTC time as the page shows it. */
274
+ function stamp(iso) {
275
+ return iso.length >= 19 ? `${iso.slice(0, 10)} ${iso.slice(11, 19)} UTC` : iso;
276
+ }
277
+ /** Escaped text, or the missing mark. */
278
+ function cell(value) {
279
+ return value === undefined || value === null || value === "" ? MISSING : escapeHtml(value);
280
+ }
281
+ function resultWord(result, L) {
282
+ return result === "passed" ? L.pass : result === "failed" ? L.fail : result === "refused" ? L.refused : L.notRun;
283
+ }
284
+ /** What a step did, in words: as expected when it passed, why when it did not, nothing when it never ran. */
285
+ function actualOf(step, L) {
286
+ if (step.result === "passed")
287
+ return L.asExpected;
288
+ if (step.result === "not-run")
289
+ return undefined;
290
+ return [step.reason, step.path ? `on ${step.path}` : ""].filter(Boolean).join(" ") || undefined;
291
+ }
292
+ function paragraphsHtml(list) {
293
+ return (list ?? []).map((p) => `<p>${escapeHtml(p)}</p>`).join("");
294
+ }
295
+ function sectionHead(section, fallback) {
296
+ return `<h2>${escapeHtml(section.heading ?? fallback)}</h2>${paragraphsHtml(section.paragraphs)}`;
297
+ }
298
+ function frameCellHtml(step, L) {
299
+ if (!step.frame)
300
+ return MISSING;
301
+ const src = escapeHtml(step.frame);
302
+ return `<a class="frame" href="${src}" target="_blank" rel="noreferrer" data-testid="report-frame-open"><img loading="lazy" src="${src}" alt="${escapeHtml(`${L.frame}: ${step.caption}`)}"><span>${escapeHtml(step.frame)}</span></a>`;
303
+ }
304
+ function summaryHtml(section, result, meta, L) {
305
+ const tests = testsOf(result);
306
+ const { passed, couldNotRun, failing } = summarise(result);
307
+ const count = (status) => tests.filter((t) => t.journey.status === status).length;
308
+ // A test that failed fails the report even when the gate let it through (--fail-on never): a pass here is a pass of every test.
309
+ const green = couldNotRun === 0 && passed && count("failed") === 0;
310
+ const overall = couldNotRun > 0 ? L.refused : green ? L.pass : L.fail;
311
+ const row = (term, value, cls = "") => `<tr${cls ? ` class="${cls}"` : ""}><th scope="row">${escapeHtml(term)}</th><td>${value}</td></tr>`;
312
+ const overallClass = green ? "pass" : "fail";
313
+ return (`<section class="summary" data-section="summary">${sectionHead(section, L.summary)}<table class="facts">` +
314
+ row(L.overall, `<span class="verdict ${overallClass}" data-testid="report-verdict">${escapeHtml(overall)}</span>`) +
315
+ row(L.target, cell(result.url)) +
316
+ row(L.started, cell(stamp(result.replay?.startedAt ?? result.generatedAt))) +
317
+ row(L.ended, cell(stamp(result.generatedAt))) +
318
+ row(L.tool, cell(meta.version)) +
319
+ row(L.commit, cell(meta.commit)) +
320
+ row(L.testsRun, String(tests.length)) +
321
+ row(L.testsPassed, String(count("passed"))) +
322
+ row(L.testsFailed, String(count("failed"))) +
323
+ row(L.testsNotRun, String(count("refused"))) +
324
+ row(L.deviations, String(deviationsOf(result).length)) +
325
+ row(L.gateIssues, String(failing)) +
326
+ `</table></section>`);
327
+ }
328
+ function columnsOf(section, L) {
329
+ return section.columns.map((c) => (typeof c === "string" ? { key: c, heading: L[c] } : { key: c.key, heading: c.heading }));
330
+ }
331
+ function testsHtml(section, result, L) {
332
+ const tests = testsOf(result);
333
+ const columns = columnsOf(section, L);
334
+ const head = `<thead><tr>${columns.map((c) => `<th scope="col" data-column="${c.key}">${escapeHtml(c.heading)}</th>`).join("")}</tr></thead>`;
335
+ const bodies = tests.map(({ journey: j, role }, ti) => {
336
+ const verdictClass = j.status === "passed" ? "pass" : "fail";
337
+ const video = j.video
338
+ ? ` <a class="video" href="${escapeHtml(j.video)}" target="_blank" rel="noreferrer" data-testid="report-video-open">${escapeHtml(L.video)}: ${escapeHtml(j.video)}</a>`
339
+ : "";
340
+ const caption = `<tr class="test-head ${verdictClass}"><th colspan="${columns.length}" scope="rowgroup">` +
341
+ `<span class="verdict ${verdictClass}">${escapeHtml(resultWord(j.status, L))}</span> ` +
342
+ `<b>${cell(j.id)}</b> ${escapeHtml(j.name)} <span class="file">${escapeHtml(j.file)}${role === null ? "" : ` · ${escapeHtml(L.role)}: ${escapeHtml(role)}`}</span>${video}</th></tr>`;
343
+ const rows = j.steps.map((s, si) => {
344
+ const cells = columns.map(({ key }) => {
345
+ switch (key) {
346
+ case "testId":
347
+ case "requirements":
348
+ // One cell for the whole test, spanning its steps.
349
+ if (si > 0)
350
+ return "";
351
+ return `<td rowspan="${j.steps.length}" class="span">${key === "testId" ? cell(j.id) : j.requirements && j.requirements.length > 0 ? j.requirements.map(escapeHtml).join(", ") : MISSING}</td>`;
352
+ case "step":
353
+ return `<td class="step"><span class="n">${s.n}.</span> ${escapeHtml(s.caption)}</td>`;
354
+ case "expected":
355
+ return `<td>${cell(s.expected)}</td>`;
356
+ case "actual":
357
+ return `<td>${cell(actualOf(s, L))}</td>`;
358
+ case "result":
359
+ return `<td class="result r-${s.result}" data-result="${s.result}">${escapeHtml(resultWord(s.result, L))}</td>`;
360
+ case "evidence":
361
+ return `<td class="evidence">${frameCellHtml(s, L)}</td>`;
362
+ }
363
+ });
364
+ return `<tr class="${s.result}">${cells.join("")}</tr>`;
365
+ });
366
+ return `<tbody id="test-${ti + 1}" data-status="${j.status}">${caption}${rows.join("")}</tbody>`;
367
+ });
368
+ return (`<section class="tests" data-section="tests">${sectionHead(section, L.tests)}` +
369
+ (tests.length === 0 ? `<p class="none">${escapeHtml(L.noTests)}</p>` : `<table class="grid">${head}${bodies.join("")}</table>`) +
370
+ `</section>`);
371
+ }
372
+ /** Every step that failed or was refused, in the order of the tests. */
373
+ export function deviationsOf(result, labels = {}) {
374
+ const L = { ...DEFAULT_LABELS, ...labels };
375
+ return testsOf(result).flatMap(({ journey: j }) => j.steps.flatMap((s) => s.result === "failed" || s.result === "refused"
376
+ ? [
377
+ {
378
+ test: j.name,
379
+ ...(j.id !== undefined ? { testId: j.id } : {}),
380
+ step: s.n,
381
+ caption: s.caption,
382
+ ...(s.expected !== undefined ? { expected: s.expected } : {}),
383
+ ...(actualOf(s, L) !== undefined ? { actual: actualOf(s, L) } : {}),
384
+ result: s.result,
385
+ },
386
+ ]
387
+ : []));
388
+ }
389
+ function deviationsHtml(section, result, L) {
390
+ const list = deviationsOf(result, L);
391
+ const body = list.length === 0
392
+ ? `<p class="none" data-testid="report-no-deviations">${escapeHtml(L.noDeviations)}</p>`
393
+ : `<table class="grid"><thead><tr><th scope="col">${escapeHtml(L.number)}</th><th scope="col">${escapeHtml(L.testId)}</th><th scope="col">${escapeHtml(L.test)}</th><th scope="col">${escapeHtml(L.step)}</th><th scope="col">${escapeHtml(L.expected)}</th><th scope="col">${escapeHtml(L.actual)}</th><th scope="col">${escapeHtml(L.result)}</th></tr></thead><tbody>` +
394
+ list
395
+ .map((d, i) => `<tr class="deviation" data-testid="report-deviation"><td>${i + 1}</td><td>${cell(d.testId)}</td><td>${escapeHtml(d.test)}</td><td><span class="n">${d.step}.</span> ${escapeHtml(d.caption)}</td><td>${cell(d.expected)}</td><td>${cell(d.actual)}</td><td>${escapeHtml(resultWord(d.result, L))}</td></tr>`)
396
+ .join("") +
397
+ `</tbody></table>`;
398
+ return `<section class="deviations" data-section="deviations">${sectionHead(section, L.deviations)}${body}</section>`;
399
+ }
400
+ function manifestHtml(section, result, meta, evidence, L) {
401
+ const facts = `<table class="facts">` +
402
+ `<tr><th scope="row">${escapeHtml(L.tool)}</th><td>${cell(meta.version)}</td></tr>` +
403
+ `<tr><th scope="row">${escapeHtml(L.target)}</th><td>${cell(result.url)}</td></tr>` +
404
+ `<tr><th scope="row">${escapeHtml(L.started)}</th><td>${cell(stamp(result.replay?.startedAt ?? result.generatedAt))}</td></tr>` +
405
+ `<tr><th scope="row">${escapeHtml(L.ended)}</th><td>${cell(stamp(result.generatedAt))}</td></tr>` +
406
+ `</table>`;
407
+ const rows = evidence
408
+ .map((e) => `<tr data-testid="report-manifest-entry"><td class="path">${escapeHtml(e.path)}</td><td>${e.bytes === null ? escapeHtml(L.notFound) : e.bytes}</td><td class="hash">${e.sha256 === null ? escapeHtml(L.notFound) : escapeHtml(e.sha256)}</td></tr>`)
409
+ .join("");
410
+ return (`<section class="manifest" data-section="manifest">${sectionHead(section, L.manifest)}${facts}` +
411
+ `<table class="grid"><thead><tr><th scope="col">${escapeHtml(L.file)}</th><th scope="col">${escapeHtml(L.bytes)}</th><th scope="col">${escapeHtml(L.sha256)}</th></tr></thead><tbody>${rows}</tbody></table></section>`);
412
+ }
413
+ function signoffHtml(section, L) {
414
+ const rows = section.roles
415
+ .map((r) => {
416
+ const role = typeof r === "string" ? r : r.role;
417
+ const meaning = typeof r === "string" ? undefined : r.meaning;
418
+ return `<tr class="signature"><td>${escapeHtml(role)}</td><td class="blank"></td><td class="blank"></td><td class="blank"></td><td>${meaning === undefined ? "" : escapeHtml(meaning)}</td></tr>`;
419
+ })
420
+ .join("");
421
+ return (`<section class="signoff" data-section="signoff">${sectionHead(section, L.signoff)}` +
422
+ `<table class="grid sign"><thead><tr><th scope="col">${escapeHtml(L.role)}</th><th scope="col">${escapeHtml(L.name)}</th><th scope="col">${escapeHtml(L.signature)}</th><th scope="col">${escapeHtml(L.signedDate)}</th><th scope="col">${escapeHtml(L.meaning)}</th></tr></thead><tbody>${rows}</tbody></table></section>`);
423
+ }
424
+ const STYLE = `
425
+ :root { color-scheme: light; --line:#c9ced6; --text:#15181d; --muted:#5d6673; --pass:#047857; --pass-bg:#d1fae5; --fail:#b91c1c; --fail-bg:#fee2e2; --head:#eef0f3; }
426
+ * { box-sizing:border-box; }
427
+ body { margin:0; background:#fff; color:var(--text); font:14px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif; }
428
+ main { max-width:1100px; margin:0 auto; padding:24px 16px 64px; }
429
+ header.title { border-bottom:2px solid var(--text); padding-bottom:12px; margin-bottom:8px; }
430
+ header.title h1 { font-size:22px; margin:0 0 4px; }
431
+ header.title .subtitle { margin:0 0 8px; color:var(--muted); }
432
+ h2 { font-size:17px; margin:28px 0 8px; border-bottom:1px solid var(--line); padding-bottom:4px; }
433
+ table { border-collapse:collapse; width:100%; margin:8px 0; }
434
+ th, td { border:1px solid var(--line); padding:5px 8px; text-align:left; vertical-align:top; overflow-wrap:anywhere; }
435
+ thead th, table.facts th { background:var(--head); }
436
+ table.facts { width:auto; min-width:50%; }
437
+ tr.test-head th { background:var(--head); font-weight:400; }
438
+ .verdict { display:inline-block; padding:0 8px; border-radius:4px; font-weight:700; }
439
+ .verdict.pass { color:var(--pass); background:var(--pass-bg); } .verdict.fail { color:var(--fail); background:var(--fail-bg); }
440
+ .file { color:var(--muted); font-size:12px; }
441
+ .n { color:var(--muted); font-weight:700; }
442
+ td.step { font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; }
443
+ td.r-passed { color:var(--pass); font-weight:700; } td.r-failed, td.r-refused { color:var(--fail); font-weight:700; } td.r-not-run { color:var(--muted); }
444
+ tr.failed td, tr.refused td { background:var(--fail-bg); }
445
+ a.frame { display:block; color:var(--muted); font-size:11px; }
446
+ a.frame img { display:block; max-width:220px; max-height:150px; object-fit:cover; object-position:top; border:1px solid var(--line); }
447
+ td.hash, td.path { font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; }
448
+ table.sign td.blank { height:2.6em; min-width:8em; }
449
+ .none { color:var(--muted); font-style:italic; }
450
+ @media print { main { max-width:none; padding:0; } a { color:inherit; text-decoration:none; } tbody, tr { break-inside:avoid; } }
451
+ `;
452
+ /**
453
+ * The whole report: one HTML file with no scripts, no event handlers and no
454
+ * external assets. The frames and videos it links sit beside it, as they do
455
+ * for the replay page. Every word from the template and every value from the
456
+ * app is escaped.
457
+ */
458
+ export function buildTemplateReport(result, template, evidence, meta) {
459
+ const L = { ...DEFAULT_LABELS, ...(template.labels ?? {}) };
460
+ const tokens = tokensOf(result, meta);
461
+ const title = fill(template.title, tokens);
462
+ const fields = [
463
+ ...(template.documentId !== undefined ? [{ label: L.documentId, value: fill(template.documentId, tokens) }] : []),
464
+ ...(template.fields ?? []).map((f) => ({ label: f.label, value: fill(f.value, tokens) })),
465
+ ];
466
+ const sections = template.sections.map((s) => {
467
+ switch (s.type) {
468
+ case "summary":
469
+ return summaryHtml(s, result, meta, L);
470
+ case "tests":
471
+ return testsHtml(s, result, L);
472
+ case "deviations":
473
+ return deviationsHtml(s, result, L);
474
+ case "manifest":
475
+ return manifestHtml(s, result, meta, evidence, L);
476
+ case "signoff":
477
+ return signoffHtml(s, L);
478
+ case "text":
479
+ return `<section class="text" data-section="text">${s.heading !== undefined ? `<h2>${escapeHtml(s.heading)}</h2>` : ""}${paragraphsHtml(s.paragraphs)}</section>`;
480
+ }
481
+ });
482
+ return `<!doctype html>
483
+ <html lang="en">
484
+ <head>
485
+ <meta charset="utf-8">
486
+ <meta name="viewport" content="width=device-width, initial-scale=1">
487
+ <meta name="generator" content="${REPORT_GENERATOR}">
488
+ <title>${escapeHtml(title)}</title>
489
+ <style>${STYLE}</style>
490
+ </head>
491
+ <body>
492
+ <main>
493
+ <header class="title">
494
+ <h1 data-testid="report-title">${escapeHtml(title)}</h1>
495
+ ${template.subtitle !== undefined ? `<p class="subtitle">${escapeHtml(fill(template.subtitle, tokens))}</p>\n` : ""}${fields.length > 0
496
+ ? `<table class="facts">${fields.map((f) => `<tr><th scope="row">${escapeHtml(f.label)}</th><td>${escapeHtml(f.value)}</td></tr>`).join("")}</table>\n`
497
+ : ""}</header>
498
+ ${sections.join("\n")}
499
+ </main>
500
+ </body>
501
+ </html>
502
+ `;
503
+ }
@@ -503,6 +503,7 @@ export const CHECK_OPTION_NAMES = [
503
503
  "sarif-file-anchor",
504
504
  "record",
505
505
  "video",
506
+ "template",
506
507
  ];
507
508
  /** Options that may be given alone, meaning on: `--record`, as well as `--record on` and `--record=off`. */
508
509
  const SWITCH_OPTIONS = new Set(["record", "video"]);
@@ -685,6 +686,12 @@ export function parseCheckArgs(args, cwd) {
685
686
  if (videoRaw !== undefined && !SWITCH_ON.includes(videoRaw) && !SWITCH_OFF.includes(videoRaw)) {
686
687
  return { ok: false, error: "--video is on or off, or given alone for on" };
687
688
  }
689
+ const template = flags.get("template");
690
+ if (template !== undefined && template.trim() === "")
691
+ return { ok: false, error: "--template needs the path of a report template (JSON)" };
692
+ if (template !== undefined && recordRaw !== undefined && SWITCH_OFF.includes(recordRaw)) {
693
+ return { ok: false, error: "--template renders the report from a recorded check's frames, so it cannot be used with --record off" };
694
+ }
688
695
  const resolve = (p) => resolveArgPath(cwd, p);
689
696
  return {
690
697
  ok: true,
@@ -713,6 +720,7 @@ export function parseCheckArgs(args, cwd) {
713
720
  ...(anchor ? { sarifFileAnchor: anchor.value } : {}),
714
721
  ...(recordRaw !== undefined ? { record: SWITCH_ON.includes(recordRaw) } : {}),
715
722
  ...(videoRaw !== undefined && SWITCH_ON.includes(videoRaw) ? { video: true } : {}),
723
+ ...(template !== undefined ? { template: resolve(template) } : {}),
716
724
  },
717
725
  };
718
726
  }
@@ -1178,5 +1186,6 @@ export function toSummaryJson(result, toolVersion) {
1178
1186
  worthALook: result.worthALook,
1179
1187
  // Routes a check with no session was sent to sign-in from: a coverage gap. Absent when the check had a session.
1180
1188
  ...(result.needsSignIn ? { needsSignIn: result.needsSignIn } : {}),
1189
+ ...(result.testReport ? { testReport: result.testReport } : {}),
1181
1190
  };
1182
1191
  }
@@ -205,30 +205,41 @@ const target = z
205
205
  });
206
206
  }
207
207
  });
208
+ /**
209
+ * What a step should bring about, in the words of the test it belongs to: the
210
+ * expected result a template-driven test report (`--template`) shows beside
211
+ * the step. Documentation only: it is never checked, so a step passes or
212
+ * fails on its action and its assertions alone.
213
+ */
214
+ const expected = z.string().trim().min(1, "is the result the step should bring about, in words").max(500).optional();
208
215
  /** Most times a repeat step may run its steps. A longer loop is a page that should be reached another way. */
209
216
  export const MAX_REPEATS = 100;
210
217
  const SINGLE_STEP_SCHEMAS = [
211
- z.object({ action: z.literal("navigate"), target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
212
- z.object({ action: z.literal("click"), target }).strict(),
213
- z.object({ action: z.literal("type"), target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
214
- z.object({ action: z.literal("select"), target, value: z.string() }).strict(),
215
- z.object({ action: z.literal("press"), value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
218
+ z.object({ action: z.literal("navigate"), expected, target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
219
+ z.object({ action: z.literal("click"), expected, target }).strict(),
220
+ z.object({ action: z.literal("type"), expected, target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
221
+ z.object({ action: z.literal("select"), expected, target, value: z.string() }).strict(),
222
+ z.object({ action: z.literal("press"), expected, value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
216
223
  // A small valid file generated for the input (its kind from `fixture`, else the input's accept attribute), as
217
224
  // scout_upload attaches one. `target` is the file input or the control that opens its chooser; absent, the page's only file input.
218
225
  z
219
226
  .object({
220
227
  action: z.literal("upload"),
228
+ expected,
221
229
  target: target.optional(),
222
230
  fixture: z.enum(FIXTURE_KINDS).optional(),
223
231
  name: z.string().min(1).max(120).optional(),
224
232
  })
225
233
  .strict(),
226
- z.object({ action: z.literal("expect-text"), text: z.string().min(1, "is the text that must be visible") }).strict(),
227
- z.object({ action: z.literal("expect-element"), target, state: z.enum(ELEMENT_STATES) }).strict(),
228
- z.object({ action: z.literal("expect-url"), pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) }).strict(),
234
+ z.object({ action: z.literal("expect-text"), expected, text: z.string().min(1, "is the text that must be visible") }).strict(),
235
+ z.object({ action: z.literal("expect-element"), expected, target, state: z.enum(ELEMENT_STATES) }).strict(),
236
+ z
237
+ .object({ action: z.literal("expect-url"), expected, pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) })
238
+ .strict(),
229
239
  z
230
240
  .object({
231
241
  action: z.literal("expect-request"),
242
+ expected,
232
243
  request: z.string().regex(REQUEST_RE, 'must be a method and a path, e.g. "GET /api/things"'),
233
244
  status: z.union([z.number().int().min(100).max(599), z.string().regex(/^[1-5]xx$/, 'must be a status such as 200, or a class such as "2xx"')]),
234
245
  })
@@ -246,6 +257,7 @@ const isRepeatCondition = (s) => s.action === "expect-text" || s.action === "exp
246
257
  const repeatSchema = z
247
258
  .object({
248
259
  action: z.literal("repeat"),
260
+ expected,
249
261
  steps: z
250
262
  .array(singleStepSchema)
251
263
  .min(1, "needs at least one step to repeat")
@@ -262,6 +274,10 @@ const flowSchema = z
262
274
  .object({
263
275
  name: z.string().min(1).max(100).optional(),
264
276
  description: z.string().max(500).optional(),
277
+ /** The test's identifier in a template-driven test report (--template). Traceability only: the replay never reads it. */
278
+ id: z.string().trim().min(1, "is the test's identifier").max(100).optional(),
279
+ /** The requirement identifiers the flow covers, shown beside it in a template-driven test report. */
280
+ requirements: z.array(z.string().trim().min(1, "is a requirement identifier").max(100)).max(50, "holds at most 50 identifiers").optional(),
265
281
  /**
266
282
  * Who walks the flow: a role whose sign-in `scenescout login --role` saved in the project. Absent, the flow runs
267
283
  * in the check's own session. Flows run in file-name order, so one role's flow can pick up what another's left.
@@ -330,6 +346,10 @@ export function parseFlow(text, file) {
330
346
  const flow = { name: parsed.data.name ?? file.replace(/\.json$/i, ""), file, steps: parsed.data.steps };
331
347
  if (parsed.data.role !== undefined)
332
348
  flow.role = parsed.data.role;
349
+ if (parsed.data.id !== undefined)
350
+ flow.id = parsed.data.id;
351
+ if (parsed.data.requirements !== undefined)
352
+ flow.requirements = parsed.data.requirements;
333
353
  return { ok: true, flow };
334
354
  }
335
355
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scenescout",
3
- "version": "3.21.3",
3
+ "version": "3.22.0",
4
4
  "description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
5
5
  "license": "MIT",
6
6
  "author": "brunoboto96",
@@ -73,7 +73,7 @@
73
73
  "mcp-check": "npm run build && npm run mcp-check:run",
74
74
  "mcp-check:run": "tsx scripts/mcp-check.ts",
75
75
  "test": "npm run build && npm run test:unit && npm run smoke:run && npm run mcp-check:run",
76
- "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run ci-test && npm run qa-test && npm run export-test && npm run contract-test && npm run pace-test && npm run limits-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run tickets-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run profiles-test && npm run refresh-test && npm run scripted-login-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test && npm run guide-test",
76
+ "test:unit": "npm run scan-test && npm run oracle-test && npm run policy-test && npm run fixture-test && npm run dispatch-test && npm run design-test && npm run check-test && npm run report-test && npm run ci-test && npm run qa-test && npm run export-test && npm run contract-test && npm run pace-test && npm run limits-test && npm run request-test && npm run settle-test && npm run claims-test && npm run verify-test && npm run brief-test && npm run tickets-test && npm run lane-test && npm run calibration-test && npm run bench-test && npm run memory-test && npm run install-test && npm run profiles-test && npm run refresh-test && npm run scripted-login-test && npm run live-test && npm run demo-test && npm run holdout-test && npm run hygiene-test && npm run guide-test",
77
77
  "scan-test": "tsx scripts/scan-test.ts",
78
78
  "oracle-test": "tsx --test scripts/oracle-test.ts",
79
79
  "policy-test": "tsx --test scripts/policy-test.ts",
@@ -81,6 +81,7 @@
81
81
  "dispatch-test": "tsx --test scripts/dispatch-test.ts",
82
82
  "design-test": "tsx --test scripts/design-test.ts",
83
83
  "check-test": "tsx --test scripts/check-test.ts",
84
+ "report-test": "tsx --test scripts/report-test.ts",
84
85
  "ci-test": "tsx --test scripts/ci-test.ts",
85
86
  "qa-test": "tsx --test scripts/qa-test.ts",
86
87
  "export-test": "tsx --test scripts/export-test.ts",