scenescout 3.14.1 → 3.16.0

package/dist/check-run.js

@@ -1,36 +1,65 @@
 /**
  * Runs `scenescout check`: attach, crawl every route the engine can find,
- * measure each one, replay the saved flows, re-test the open findings a page
- * load can reproduce, and write the verdict. The rules live in
- * engine/check.ts, engine/flow.ts and engine/verify.ts; this file only drives
- * the browser and the files.
+ * measure each one, picture the targets of any visual baselines, replay the
+ * saved flows, re-test the open findings a page load can reproduce, and write
+ * the verdict. The rules live in engine/check.ts, engine/flow.ts,
+ * engine/verify.ts and engine/baseline.ts; this file only drives the browser
+ * and the files.
  */
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { BASELINE_CAPTURE, baselineFiles, baselineMeta, defaultBaselinesDir, elementTarget, isVisualPicture, judgeBaseline, missingTargetsMessage, parseBaselineTargets, readStoredBaseline, TARGETS_FILE, VISUAL_DIRNAME, visualFiles, } from "./engine/baseline.js";
 import { BrowserEngine } from "./engine/browser.js";
-import { checkFindings, redactFlowRuns, redactRoute, redactRoutes, settingsOf, withoutOwnResponse, } from "./engine/check.js";
+import { checkFindings, redactBaselineRun, MAX_DISCOVERY_ROUNDS, redactFlowRuns, redactRoute, redactRoutes, settingsOf, unmeasuredReason, withoutOwnResponse, } from "./engine/check.js";
 import { loadFlows, resolveFlowsDir } from "./engine/flow.js";
-import { MemoryStore, MEMORY_DIRNAME } from "./engine/memory.js";
+import { MemoryStore, MEMORY_DIRNAME, writeSelfIgnore } from "./engine/memory.js";
+import { decodePng, encodePng } from "./engine/png.js";
+import { firstLineOf } from "./engine/limits.js";
 import { checkRetestPlan, retestResults, wellFormedFindings } from "./engine/verify.js";
-/** Link discovery rounds: each crawl reveals the routes its pages link to. Past a few, a site is paginating rather than revealing. */
-const MAX_ROUNDS = 6;
+/** The baselines folder a check uses: the one --baselines names, else the project's own. */
+export function baselinesDirOf(options) {
+    return options.baselinesDir ?? defaultBaselinesDir(options.projectDir);
+}
+/** Read and validate targets.json. A missing or invalid one stops the check before it starts: baselines that silently compare nothing pass by never running. */
+function readBaselineTargets(options) {
+    const file = path.join(baselinesDirOf(options), TARGETS_FILE);
+    let text;
+    try {
+        text = fs.readFileSync(file, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            throw new Error(`--baseline ${options.baseline}: ${missingTargetsMessage(file)}`);
+        throw new Error(`could not read ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const parsed = parseBaselineTargets(text, file);
+    if (!parsed.ok)
+        throw new Error(`baseline targets: ${parsed.error}`);
+    return parsed.targets;
+}
 /**
- * Read the flows and the findings, before any browser starts. Throws with a
- * sentence naming the file and the field on anything it cannot read: a flow
- * that is silently skipped is a flow that silently passes.
+ * Read the flows, the findings and the baseline targets, before any browser
+ * starts. Throws with a sentence naming the file and the field on anything it
+ * cannot read: a flow that is silently skipped is a flow that silently passes.
  */
 export function readCheckInputs(options) {
     const where = resolveFlowsDir(options.flows, options.projectDir, (p) => fs.existsSync(p) && fs.statSync(p).isDirectory());
     if ("error" in where)
         throw new Error(where.error);
-    const { flows, skipped: skippedFlows } = where.dir ? loadFlows(where.dir) : { flows: [], skipped: [] };
+    const loaded = where.dir ? loadFlows(where.dir) : { flows: [], skipped: [] };
+    const read = {
+        flows: loaded.flows,
+        skippedFlows: loaded.skipped,
+        flowsDir: where.dir,
+        ...(options.baseline !== "off" ? { baselineTargets: readBaselineTargets(options) } : {}),
+    };
     if (!options.retest)
-        return { flows, skippedFlows, findings: null };
+        return { ...read, findings: null };
     // Read, never written: a check leaves the project's memory as it found it.
     const memoryPath = path.join(options.projectDir, MEMORY_DIRNAME, "memory.json");
     if (!fs.existsSync(memoryPath))
-        return { flows, skippedFlows, findings: null };
+        return { ...read, findings: null };
     let findings;
     try {
         findings = JSON.parse(fs.readFileSync(memoryPath, "utf8")).findings;
@@ -40,14 +69,22 @@ export function readCheckInputs(options) {
     }
     if (findings !== undefined && !Array.isArray(findings))
         throw new Error(`${memoryPath}: "findings" is not a list. Pass --retest off to check without it`);
-    return { flows, skippedFlows, findings: wellFormedFindings(findings ?? []) };
+    return { ...read, findings: wellFormedFindings(findings ?? []) };
 }
 export async function runCheck(options, log = () => { }, inputs = { flows: [], findings: null }) {
+    if (options.paths && options.timeBudgetMs !== undefined)
+        throw new Error("a time budget applies to route discovery, not to a list of paths");
     // A throwaway memory: a check is one run, and a store shared with earlier
     // exploratory runs would count their visits as this check's and skip those routes.
     const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "scenescout-check-"));
+    // Also on an exit the finally below never reaches, such as Ctrl+C, which the browser's driver answers with process.exit.
+    const removeScratch = () => fs.rmSync(scratch, { recursive: true, force: true });
+    process.once("exit", removeScratch);
     const engine = new BrowserEngine();
     const start = new URL(options.url);
+    // From the start of the run, browser launch included: the budget is wall-clock time a person waits.
+    const deadline = options.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined;
+    const pastDeadline = () => deadline !== undefined && Date.now() >= deadline;
     try {
         // Attached at the origin: the engine joins every crawled path onto the URL
         // it attached to, so attaching to a start page with a path would double it.
@@ -60,6 +97,8 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             ...(options.browser ? { browser: options.browser } : {}),
             actionTimeoutMs: options.actionTimeoutMs,
             navTimeoutMs: options.navTimeoutMs,
+            // Named rather than left to attach's defaults: a baseline is only comparable with a picture taken in the same window, at the same scale.
+            ...(options.baseline !== "off" ? { viewport: BASELINE_CAPTURE.viewport, deviceScaleFactor: BASELINE_CAPTURE.deviceScaleFactor } : {}),
             objective: "Deterministic check: visit every route and measure it",
             task: "Checking every route",
         });
@@ -68,23 +107,26 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
         if (authFailed)
             throw new Error(authFailed.replace(/ Continuing now tests a logged-out app\.$/, "").replace(/re-attach/, "run the check again"));
         const routes = [];
-        const crawl = async (paths, limit) => {
-            await engine.crawl(paths, { inspect: true, limit });
+        const crawl = async (paths, limit, deadline) => {
+            await engine.crawl(paths, { inspect: true, limit, deadline });
             routes.push(...engine.lastCrawlHealth);
         };
         if (options.paths) {
             await crawl(options.paths.slice(0, options.maxRoutes), options.maxRoutes);
         }
         else {
-            // The start page first, whatever else is known: it is the one route the user named.
+            // The start page first, whatever else is known: it is the one route the user named. The
+            // time budget does not apply to it, so a run always measures at least the page it was given.
             await crawl([`${start.pathname}${start.search}${start.hash}`], 1);
-            for (let round = 0; round < MAX_ROUNDS && routes.length < options.maxRoutes; round++) {
-                if (engine.crawlableRoutes().length === 0)
+            for (let round = 0; round < MAX_DISCOVERY_ROUNDS && routes.length < options.maxRoutes; round++) {
+                if (engine.crawlableRoutes().length === 0 || pastDeadline())
                     break;
-                await crawl(undefined, options.maxRoutes - routes.length);
+                await crawl(undefined, options.maxRoutes - routes.length, deadline);
                 log(`  ${routes.length} route(s) checked`);
             }
         }
+        // Out of time with routes still to visit; a cap of routes reached first is --max-routes's to report.
+        const timeLimitReached = pastDeadline() && routes.length < options.maxRoutes && engine.crawlableRoutes().length > 0;
         // Pages of open findings the crawl did not load exactly: loaded now, so each re-test has its own measurement.
         // Kept apart from `routes`: they are measured only to re-test, never checked against the page rules, and do not count
         // towards --max-routes. With --paths the check stays on the paths it was given.
@@ -98,6 +140,13 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
                 log(`  ${missing.length} page(s) of open findings loaded only to re-test them`);
             }
         }
+        // Before the flows, so a picture is of the page as a visit finds it, not as a flow left it. Not when the crawl
+        // reached only sign-in pages: the check has no verdict then, and an update would write sign-in pages as baselines.
+        if (options.baseline !== "off" && !inputs.baselineTargets)
+            throw new Error(`--baseline ${options.baseline} was given no targets: read them with readCheckInputs`);
+        const baselines = options.baseline !== "off" && inputs.baselineTargets && unmeasuredReason(routes, !options.paths) === null
+            ? await takeBaselines(engine, options, options.baseline, inputs.baselineTargets, log)
+            : null;
         const flowRuns = [];
         for (const flow of inputs.flows) {
             // --flow-writes never: observe's rule, whatever --mode lets the crawl do.
@@ -125,7 +174,8 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             : null;
         const measured = redactRoutes(routes.map(withoutOwnResponse));
         const flows = redactFlowRuns(flowRuns);
-        const { issues, worthALook } = checkFindings(measured, start.origin, options.ignore, flows);
+        const pictured = baselines ? redactBaselineRun(baselines) : null;
+        const { issues, worthALook } = checkFindings(measured, start.origin, options.ignore, flows, pictured);
         return {
             url: redactRoute(options.url),
             generatedAt: new Date().toISOString(),
@@ -134,21 +184,158 @@ export async function runCheck(options, log = () => { }, inputs = { flows: [], f
             routes: measured,
             issues,
             worthALook,
-            // Routes that failed to load are issues already; "not visited" is only what --max-routes left out.
+            // Routes that failed to load are issues already; "not visited" is only what --max-routes (or the time budget) left out.
             unvisited: options.paths ? [] : engine.crawlableRoutes().map(redactRoute),
+            ...(options.timeBudgetMs !== undefined ? { timeBudget: { ms: options.timeBudgetMs, reached: timeLimitReached } } : {}),
             ignored: options.ignore,
             flows,
             skippedFlows: inputs.skippedFlows ?? [],
             retest,
             settings: settingsOf(options),
+            baselines: pictured,
         };
     }
     finally {
         await engine.close().catch((err) => log(`closing the browser failed: ${err instanceof Error ? err.message : String(err)}`));
-        fs.rmSync(scratch, { recursive: true, force: true });
+        process.off("exit", removeScratch);
+        removeScratch();
     }
 }
 /** Where a check writes when not told: beside the project's other SceneScout output, which is already ignored by git. */
 export function defaultCheckDir(projectDir) {
     return path.join(projectDir, MEMORY_DIRNAME, "check");
 }
+/** A file's bytes, or null when there is no such file. Any other failure to read it is thrown. */
+function readIfThere(file) {
+    try {
+        return fs.readFileSync(file);
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return null;
+        throw err;
+    }
+}
+/**
+ * Remove the pictures an earlier run left under <out>/visual/, so none is
+ * kept or uploaded as this run's. Only files named as a check names them
+ * (isVisualPicture) are removed, then any folder that leaves empty: the
+ * output folder is one a project chose, and whatever else it holds stays.
+ */
+function clearVisualPictures(outDir) {
+    const root = path.join(outDir, VISUAL_DIRNAME);
+    let folders;
+    try {
+        folders = fs.readdirSync(root, { withFileTypes: true });
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return;
+        throw err;
+    }
+    for (const folder of folders.filter((f) => f.isDirectory())) {
+        const dir = path.join(root, folder.name);
+        for (const file of fs.readdirSync(dir, { withFileTypes: true })) {
+            if (file.isFile() && isVisualPicture(folder.name, file.name))
+                fs.rmSync(path.join(dir, file.name));
+        }
+        if (fs.readdirSync(dir).length === 0)
+            fs.rmdirSync(dir);
+    }
+    if (fs.readdirSync(root).length === 0)
+        fs.rmdirSync(root);
+}
+/** The baselines folder as the report names it: relative to the project when inside it, with forward slashes. */
+function shownDir(dir, projectDir) {
+    const rel = path.relative(projectDir, dir);
+    if (rel === "")
+        return ".";
+    return rel.startsWith("..") || path.isAbsolute(rel) ? dir : rel.split(path.sep).join("/");
+}
+/**
+ * --baseline compare|update: take each target's picture, judge it against its
+ * stored baseline (baseline.ts), and write what the judgement says to write: a
+ * new baseline under update, and under compare the baseline, the picture now
+ * and the changed pixels of a target that changed, beside the report. A page
+ * load per target, so no picture depends on the one taken before it.
+ */
+async function takeBaselines(engine, options, mode, targets, log) {
+    const dir = baselinesDirOf(options);
+    const outDir = options.outDir ?? defaultCheckDir(options.projectDir);
+    const engineName = engine.browserEngine;
+    const at = (root, rel) => path.join(root, ...rel.split("/"));
+    // The default folders are inside .scenescout/, which ignores itself, and its .gitignore is written before anything goes
+    // there: a run that ends early must not leave pictures a `git add -A` would pick up. A folder the project named is its own.
+    if (!options.outDir || (!options.baselinesDir && mode === "update")) {
+        const scenescoutDir = path.join(options.projectDir, MEMORY_DIRNAME);
+        fs.mkdirSync(scenescoutDir, { recursive: true });
+        writeSelfIgnore(scenescoutDir);
+    }
+    clearVisualPictures(outDir);
+    const results = [];
+    try {
+        for (const target of targets) {
+            const files = baselineFiles(engineName, target);
+            const own = { path: target.path, element: target.element, baseline: files.png };
+            const shown = `${target.element} on ${redactRoute(target.path)}`;
+            const element = elementTarget(target.element);
+            // parseBaselineTargets refuses such an element; one reaching here must not be pictured as the whole page instead.
+            if (element === undefined)
+                throw new Error(`baseline target ${shown} names neither the page nor an element`);
+            let shot;
+            let image;
+            try {
+                shot = await engine.captureForBaseline(target.path, element, BASELINE_CAPTURE);
+                image = decodePng(shot.png);
+            }
+            catch (err) {
+                // A closed browser, or a mistake in this code, is the check's failure, never the app's: the check ends "could not run".
+                if (!engine.alive || err instanceof TypeError || err instanceof ReferenceError || err instanceof RangeError)
+                    throw err;
+                const why = firstLineOf(err);
+                results.push({ ...own, status: "not-captured", detail: why });
+                log(`  baseline ${shown}: not captured (${why})`);
+                continue;
+            }
+            const capture = { ...BASELINE_CAPTURE, viewport: shot.viewport, deviceScaleFactor: shot.deviceScaleFactor };
+            const storedPng = readIfThere(at(dir, files.png));
+            const storedJson = readIfThere(at(dir, files.json));
+            const stored = readStoredBaseline(target, engineName, storedPng, storedJson === null ? null : storedJson.toString("utf8"));
+            const { diffImage, ...verdict } = judgeBaseline({
+                mode,
+                threshold: options.baselineThreshold,
+                stored,
+                now: { capture, platform: process.platform, image },
+            });
+            const result = { ...own, ...verdict, ...(shot.cut ? { partial: shot.cut } : {}) };
+            if (verdict.status === "updated") {
+                fs.mkdirSync(path.dirname(at(dir, files.png)), { recursive: true });
+                fs.writeFileSync(at(dir, files.png), shot.png);
+                const meta = baselineMeta({
+                    target,
+                    engine: engineName,
+                    platform: process.platform,
+                    capture,
+                    size: { width: image.width, height: image.height },
+                    capturedAt: new Date().toISOString(),
+                });
+                fs.writeFileSync(at(dir, files.json), JSON.stringify(meta, null, 2) + "\n");
+            }
+            if (diffImage && storedPng) {
+                const pictures = visualFiles(target);
+                fs.mkdirSync(path.dirname(at(outDir, pictures.diff)), { recursive: true });
+                fs.writeFileSync(at(outDir, pictures.expected), storedPng);
+                fs.writeFileSync(at(outDir, pictures.actual), shot.png);
+                fs.writeFileSync(at(outDir, pictures.diff), encodePng(diffImage));
+                result.files = pictures;
+            }
+            results.push(result);
+            log(`  baseline ${shown}: ${result.status}${result.diff ? ` (${result.diff.percent}% changed)` : ""}`);
+        }
+    }
+    finally {
+        // Reported, never thrown: an error from the loop above must not be replaced by this one.
+        await engine.endBaselineCaptures().catch((err) => log(`  could not stop requesting reduced motion: ${firstLineOf(err)}`));
+    }
+    return { mode, engine: engineName, threshold: options.baselineThreshold, dir: shownDir(dir, options.projectDir), results };
+}