scenescout 3.15.0 → 3.17.0

package/dist/check-run.js

@@ -1,34 +1,65 @@
 /**
  * Runs `scenescout check`: attach, crawl every route the engine can find,
- * measure each one, replay the saved flows, re-test the open findings a page
- * load can reproduce, and write the verdict. The rules live in
- * engine/check.ts, engine/flow.ts and engine/verify.ts; this file only drives
- * the browser and the files.
+ * measure each one, picture the targets of any visual baselines, replay the
+ * saved flows, re-test the open findings a page load can reproduce, and write
+ * the verdict. The rules live in engine/check.ts, engine/flow.ts,
+ * engine/verify.ts and engine/baseline.ts; this file only drives the browser
+ * and the files.
  */
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { BASELINE_CAPTURE, baselineFiles, baselineMeta, defaultBaselinesDir, elementTarget, isVisualPicture, judgeBaseline, missingTargetsMessage, parseBaselineTargets, readStoredBaseline, TARGETS_FILE, VISUAL_DIRNAME, visualFiles, } from "./engine/baseline.js";
 import { BrowserEngine } from "./engine/browser.js";
-import { checkFindings, MAX_DISCOVERY_ROUNDS, redactFlowRuns, redactRoute, redactRoutes, settingsOf, withoutOwnResponse, } from "./engine/check.js";
+import { checkFindings, redactBaselineRun, MAX_DISCOVERY_ROUNDS, redactFlowRuns, redactRoute, redactRoutes, settingsOf, unmeasuredReason, withoutOwnResponse, } from "./engine/check.js";
 import { loadFlows, resolveFlowsDir } from "./engine/flow.js";
-import { MemoryStore, MEMORY_DIRNAME } from "./engine/memory.js";
+import { MemoryStore, MEMORY_DIRNAME, writeSelfIgnore } from "./engine/memory.js";
+import { decodePng, encodePng } from "./engine/png.js";
+import { firstLineOf } from "./engine/limits.js";
 import { checkRetestPlan, retestResults, wellFormedFindings } from "./engine/verify.js";
+/** The baselines folder a check uses: the one --baselines names, else the project's own. */
+export function baselinesDirOf(options) {
+    return options.baselinesDir ?? defaultBaselinesDir(options.projectDir);
+}
+/** Read and validate targets.json. A missing or invalid one stops the check before it starts: baselines that silently compare nothing pass by never running. */
+function readBaselineTargets(options) {
+    const file = path.join(baselinesDirOf(options), TARGETS_FILE);
+    let text;
+    try {
+        text = fs.readFileSync(file, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            throw new Error(`--baseline ${options.baseline}: ${missingTargetsMessage(file)}`);
+        throw new Error(`could not read ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const parsed = parseBaselineTargets(text, file);
+    if (!parsed.ok)
+        throw new Error(`baseline targets: ${parsed.error}`);
+    return parsed.targets;
+}
 /**
- * Read the flows and the findings, before any browser starts. Throws with a
- * sentence naming the file and the field on anything it cannot read: a flow
- * that is silently skipped is a flow that silently passes.
+ * Read the flows, the findings and the baseline targets, before any browser
+ * starts. Throws with a sentence naming the file and the field on anything it
+ * cannot read: a flow that is silently skipped is a flow that silently passes.
  */
 export function readCheckInputs(options) {
     const where = resolveFlowsDir(options.flows, options.projectDir, (p) => fs.existsSync(p) && fs.statSync(p).isDirectory());
     if ("error" in where)
         throw new Error(where.error);
-    const { flows, skipped: skippedFlows } = where.dir ? loadFlows(where.dir) : { flows: [], skipped: [] };
+    const loaded = where.dir ? loadFlows(where.dir) : { flows: [], skipped: [] };
+    const read = {
+        flows: loaded.flows,
+        skippedFlows: loaded.skipped,
+        flowsDir: where.dir,
+        ...(options.baseline !== "off" ? { baselineTargets: readBaselineTargets(options) } : {}),
+    };
     if (!options.retest)
-        return { flows, skippedFlows, findings: null };
+        return { ...read, findings: null };
     // Read, never written: a check leaves the project's memory as it found it.
     const memoryPath = path.join(options.projectDir, MEMORY_DIRNAME, "memory.json");
     if (!fs.existsSync(memoryPath))
-        return { flows, skippedFlows, findings: null };
+        return { ...read, findings: null };
     let findings;
     try {
         findings = JSON.parse(fs.readFileSync(memoryPath, "utf8")).findings;
@@ -38,7 +69,7 @@ export function readCheckInputs(options) {
     }
     if (findings !== undefined && !Array.isArray(findings))
         throw new Error(`${memoryPath}: "findings" is not a list. Pass --retest off to check without it`);
-    return { flows, skippedFlows, findings: wellFormedFindings(findings ?? []) };
+    return { ...read, findings: wellFormedFindings(findings ?? []) };
 }
 export async function runCheck(options, log = () => { }, inputs = { flows: [], findings: null }) {
     if (options.paths && options.timeBudgetMs !== undefined)
@@ -66,6 +97,8 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             ...(options.browser ? { browser: options.browser } : {}),
             actionTimeoutMs: options.actionTimeoutMs,
             navTimeoutMs: options.navTimeoutMs,
+            // Named rather than left to attach's defaults: a baseline is only comparable with a picture taken in the same window, at the same scale.
+            ...(options.baseline !== "off" ? { viewport: BASELINE_CAPTURE.viewport, deviceScaleFactor: BASELINE_CAPTURE.deviceScaleFactor } : {}),
             objective: "Deterministic check: visit every route and measure it",
             task: "Checking every route",
         });
@@ -107,6 +140,13 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
                 log(`  ${missing.length} page(s) of open findings loaded only to re-test them`);
             }
         }
+        // Before the flows, so a picture is of the page as a visit finds it, not as a flow left it. Not when the crawl
+        // reached only sign-in pages: the check has no verdict then, and an update would write sign-in pages as baselines.
+        if (options.baseline !== "off" && !inputs.baselineTargets)
+            throw new Error(`--baseline ${options.baseline} was given no targets: read them with readCheckInputs`);
+        const baselines = options.baseline !== "off" && inputs.baselineTargets && unmeasuredReason(routes, !options.paths) === null
+            ? await takeBaselines(engine, options, options.baseline, inputs.baselineTargets, log)
+            : null;
         const flowRuns = [];
         for (const flow of inputs.flows) {
             // --flow-writes never: observe's rule, whatever --mode lets the crawl do.
@@ -134,7 +174,8 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             : null;
         const measured = redactRoutes(routes.map(withoutOwnResponse));
         const flows = redactFlowRuns(flowRuns);
-        const { issues, worthALook } = checkFindings(measured, start.origin, options.ignore, flows);
+        const pictured = baselines ? redactBaselineRun(baselines) : null;
+        const { issues, worthALook } = checkFindings(measured, start.origin, options.ignore, flows, pictured, options.ignorePaths);
         return {
             url: redactRoute(options.url),
             generatedAt: new Date().toISOString(),
@@ -147,10 +188,12 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             unvisited: options.paths ? [] : engine.crawlableRoutes().map(redactRoute),
             ...(options.timeBudgetMs !== undefined ? { timeBudget: { ms: options.timeBudgetMs, reached: timeLimitReached } } : {}),
             ignored: options.ignore,
+            ignoredPaths: options.ignorePaths,
             flows,
             skippedFlows: inputs.skippedFlows ?? [],
             retest,
             settings: settingsOf(options),
+            baselines: pictured,
         };
     }
     finally {
@@ -163,3 +206,137 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
 export function defaultCheckDir(projectDir) {
     return path.join(projectDir, MEMORY_DIRNAME, "check");
 }
+/** A file's bytes, or null when there is no such file. Any other failure to read it is thrown. */
+function readIfThere(file) {
+    try {
+        return fs.readFileSync(file);
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return null;
+        throw err;
+    }
+}
+/**
+ * Remove the pictures an earlier run left under <out>/visual/, so none is
+ * kept or uploaded as this run's. Only files named as a check names them
+ * (isVisualPicture) are removed, then any folder that leaves empty: the
+ * output folder is one a project chose, and whatever else it holds stays.
+ */
+function clearVisualPictures(outDir) {
+    const root = path.join(outDir, VISUAL_DIRNAME);
+    let folders;
+    try {
+        folders = fs.readdirSync(root, { withFileTypes: true });
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return;
+        throw err;
+    }
+    for (const folder of folders.filter((f) => f.isDirectory())) {
+        const dir = path.join(root, folder.name);
+        for (const file of fs.readdirSync(dir, { withFileTypes: true })) {
+            if (file.isFile() && isVisualPicture(folder.name, file.name))
+                fs.rmSync(path.join(dir, file.name));
+        }
+        if (fs.readdirSync(dir).length === 0)
+            fs.rmdirSync(dir);
+    }
+    if (fs.readdirSync(root).length === 0)
+        fs.rmdirSync(root);
+}
+/** The baselines folder as the report names it: relative to the project when inside it, with forward slashes. */
+function shownDir(dir, projectDir) {
+    const rel = path.relative(projectDir, dir);
+    if (rel === "")
+        return ".";
+    return rel.startsWith("..") || path.isAbsolute(rel) ? dir : rel.split(path.sep).join("/");
+}
+/**
+ * --baseline compare|update: take each target's picture, judge it against its
+ * stored baseline (baseline.ts), and write what the judgement says to write: a
+ * new baseline under update, and under compare the baseline, the picture now
+ * and the changed pixels of a target that changed, beside the report. A page
+ * load per target, so no picture depends on the one taken before it.
+ */
+async function takeBaselines(engine, options, mode, targets, log) {
+    const dir = baselinesDirOf(options);
+    const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
+    const engineName = engine.browserEngine;
+    const at = (root, rel) => path.join(root, ...rel.split("/"));
+    // The default folders are inside .scenescout/, which ignores itself, and its .gitignore is written before anything goes
+    // there: a run that ends early must not leave pictures a `git add -A` would pick up. A folder the project named is its own.
+    if (!options.outDir || (!options.baselinesDir && mode === "update")) {
+        const scenescoutDir = path.join(options.projectDir, MEMORY_DIRNAME);
+        fs.mkdirSync(scenescoutDir, { recursive: true });
+        writeSelfIgnore(scenescoutDir);
+    }
+    clearVisualPictures(outDir);
+    const results = [];
+    try {
+        for (const target of targets) {
+            const files = baselineFiles(engineName, target);
+            const own = { path: target.path, element: target.element, baseline: files.png };
+            const shown = `${target.element} on ${redactRoute(target.path)}`;
+            const element = elementTarget(target.element);
+            // parseBaselineTargets refuses such an element; one reaching here must not be pictured as the whole page instead.
+            if (element === undefined)
+                throw new Error(`baseline target ${shown} names neither the page nor an element`);
+            let shot;
+            let image;
+            try {
+                shot = await engine.captureForBaseline(target.path, element, BASELINE_CAPTURE);
+                image = decodePng(shot.png);
+            }
+            catch (err) {
+                // A closed browser, or a mistake in this code, is the check's failure, never the app's: the check ends "could not run".
+                if (!engine.alive || err instanceof TypeError || err instanceof ReferenceError || err instanceof RangeError)
+                    throw err;
+                const why = firstLineOf(err);
+                results.push({ ...own, status: "not-captured", detail: why });
+                log(`  baseline ${shown}: not captured (${why})`);
+                continue;
+            }
+            const capture = { ...BASELINE_CAPTURE, viewport: shot.viewport, deviceScaleFactor: shot.deviceScaleFactor };
+            const storedPng = readIfThere(at(dir, files.png));
+            const storedJson = readIfThere(at(dir, files.json));
+            const stored = readStoredBaseline(target, engineName, storedPng, storedJson === null ? null : storedJson.toString("utf8"));
+            const { diffImage, ...verdict } = judgeBaseline({
+                mode,
+                threshold: options.baselineThreshold,
+                stored,
+                now: { capture, platform: process.platform, image },
+            });
+            const result = { ...own, ...verdict, ...(shot.cut ? { partial: shot.cut } : {}) };
+            if (verdict.status === "updated") {
+                fs.mkdirSync(path.dirname(at(dir, files.png)), { recursive: true });
+                fs.writeFileSync(at(dir, files.png), shot.png);
+                const meta = baselineMeta({
+                    target,
+                    engine: engineName,
+                    platform: process.platform,
+                    capture,
+                    size: { width: image.width, height: image.height },
+                    capturedAt: new Date().toISOString(),
+                });
+                fs.writeFileSync(at(dir, files.json), JSON.stringify(meta, null, 2) + "\n");
+            }
+            if (diffImage && storedPng) {
+                const pictures = visualFiles(target);
+                fs.mkdirSync(path.dirname(at(outDir, pictures.diff)), { recursive: true });
+                fs.writeFileSync(at(outDir, pictures.expected), storedPng);
+                fs.writeFileSync(at(outDir, pictures.actual), shot.png);
+                fs.writeFileSync(at(outDir, pictures.diff), encodePng(diffImage));
+                result.files = pictures;
+            }
+            results.push(result);
+            log(`  baseline ${shown}: ${result.status}${result.diff ? ` (${result.diff.percent}% changed)` : ""}`);
+        }
+    }
+    finally {
+        // Reported, never thrown: an error from the loop above must not be replaced by this one.
+        await engine.endBaselineCaptures().catch((err) => log(`  could not stop requesting reduced motion: ${firstLineOf(err)}`));
+    }
+    return { mode, engine: engineName, threshold: options.baselineThreshold, dir: shownDir(dir, options.projectDir), results };
+}