staysfixed 0.1.0

package/src/picture/capture.js

@@ -0,0 +1,276 @@
+/**
+ * Taking one picture of one screen.
+ *
+ * The order of operations here is the whole reason picture checks can be
+ * trusted: freeze the world FIRST (clock, fonts, randomness, network), then
+ * drive the app, then hold still until two frames in a row are identical, and
+ * only then press the shutter. Do any of those out of order and the tool starts
+ * crying wolf, which is worse than having no tool at all.
+ */
+
+import { PNG } from 'pngjs';
+import { applyFreeze, prepareForShutter } from '../freeze/index.js';
+import { settle } from '../freeze/settle.js';
+import { resolveMasks, paintMasks } from '../freeze/mask.js';
+import { StaysFixedError, isExpected, messageOf } from '../core/errors.js';
+
+/**
+ * Every instruction a declarative step is allowed to give, in the order they run
+ * when a single step carries more than one. `goto` first, `wait` last — that way
+ * `{ waitFor: '#list', click: '#list li', wait: 100 }` reads and behaves the same way.
+ * @type {readonly (keyof import('../types.js').Step)[]}
+ */
+const ACTION_ORDER = [
+  'goto',
+  'waitFor',
+  'scrollTo',
+  'hover',
+  'click',
+  'type',
+  'press',
+  'evaluate',
+  'waitForGone',
+  'wait',
+];
+
+/** Keys allowed on a step that are not actions: `text` feeds `type`, `note` is for humans. */
+const KNOWN_KEYS = new Set([...ACTION_ORDER, 'text', 'note']);
+
+/**
+ * Photograph one screen.
+ *
+ * @param {import('../types.js').PageHandle} page
+ * @param {import('../types.js').ScreenConfig} screen
+ * @param {{
+ *   viewport: import('../types.js').ViewportConfig,
+ *   tolerance: import('../types.js').ToleranceConfig,
+ *   freeze: import('../types.js').FreezeConfig,
+ *   masks: import('../types.js').Mask[],
+ * }} settings
+ * @param {{fixturesDir: string, record?: boolean, timeoutMs?: number}} ctx
+ * @returns {Promise<import('../types.js').CaptureReport & {masks: import('../types.js').MaskRect[]}>}
+ *   The standard report plus the mask rectangles that were painted, so the
+ *   comparison can paint the exact same rectangles onto the approved picture.
+ */
+export async function captureScreen(page, screen, settings, ctx) {
+  const deviceScaleFactor = settings.viewport.deviceScaleFactor ?? 2;
+  const settleConfig = settings.freeze.settle ?? {};
+  const timeoutMs = ctx.timeoutMs ?? settleConfig.timeoutMs ?? 10_000;
+
+  await page.setViewport(settings.viewport);
+  page.clearConsole();
+
+  const frozen = await applyFreeze(page, settings.freeze, {
+    fixturesDir: ctx.fixturesDir,
+    screenName: screen.name,
+    record: ctx.record ?? false,
+    deviceScaleFactor,
+  });
+
+  try {
+    if (typeof screen.do === 'function') {
+      await screen.do(page);
+    } else {
+      await runSteps(page, screen.steps ?? []);
+    }
+
+    // Scrolling back to the top is the deterministic default, but a screen that
+    // deliberately scrolled somewhere must be photographed where it landed.
+    const scrolledOnPurpose =
+      typeof screen.do === 'function' || (screen.steps ?? []).some((s) => s.scrollTo !== undefined);
+
+    await prepareForShutter(page, {
+      fonts: settings.freeze.fonts !== false,
+      timeoutMs,
+      keepScroll: scrolledOnPurpose,
+    });
+
+    /** @type {import('../types.js').CaptureOptions} */
+    const shotOptions = {};
+    if (screen.fullPage) shotOptions.fullPage = true;
+    if (screen.clip) shotOptions.clip = screen.clip;
+
+    const held = await settle(page, {
+      frames: settleConfig.frames ?? 2,
+      intervalMs: settleConfig.intervalMs ?? 250,
+      timeoutMs: settleConfig.timeoutMs ?? 10_000,
+      maxDriftPixels: settleConfig.maxDriftPixels ?? 0,
+      capture: () => page.shoot(shotOptions),
+    });
+
+    const rects = await resolveMasks(page, settings.masks ?? [], { deviceScaleFactor });
+    const png = rects.length > 0 ? paintInto(held.png, rects) : held.png;
+    const size = pngSize(png);
+
+    // The picture is taken; now put the app back if this screen asked us to.
+    //
+    // A reload between screens restores what the PAGE was holding, but not what the app
+    // has written to disk. A screen that collapses a sidebar, switches a theme or turns
+    // a setting off leaves that behind for everything after it, and the failure then
+    // lands on some innocent check further down the list. `after` is where a screen
+    // cleans up after itself. It runs even though the shutter has already fired, and a
+    // failure here is reported rather than swallowed — a cleanup that quietly stopped
+    // working would poison every run after it.
+    if (screen.after && screen.after.length > 0) {
+      await runSteps(page, screen.after);
+    }
+
+    return {
+      png,
+      width: size.width,
+      height: size.height,
+      settle: held.report,
+      consoleErrors: page.consoleErrors(),
+      freeze: frozen.stats(),
+      masks: rects,
+    };
+  } finally {
+    // Releasing must never be the thing that hides a real failure.
+    try {
+      await frozen.release();
+    } catch {
+      // ignored on purpose
+    }
+  }
+}
+
+/**
+ * Paint the masks into a screenshot and re-encode it.
+ * @param {Buffer} buffer
+ * @param {import('../types.js').MaskRect[]} rects
+ * @returns {Buffer}
+ */
+function paintInto(buffer, rects) {
+  const image = PNG.sync.read(buffer);
+  paintMasks(image, rects);
+  return PNG.sync.write(image);
+}
+
+/**
+ * Run the declarative steps from a JSON config, in order.
+ *
+ * @param {import('../types.js').PageApi} page
+ * @param {import('../types.js').Step[]} steps
+ * @returns {Promise<void>}
+ */
+export async function runSteps(page, steps) {
+  for (let i = 0; i < steps.length; i++) {
+    const step = steps[i];
+    const n = i + 1;
+    if (!step || typeof step !== 'object') {
+      throw new StaysFixedError(`Step ${n} is not an instruction.`, {
+        hint: 'Each step is an object, for example { click: "#save" }.',
+      });
+    }
+    for (const key of Object.keys(step)) {
+      if (!KNOWN_KEYS.has(/** @type {keyof import('../types.js').Step} */ (key))) {
+        throw new StaysFixedError(`Step ${n} says \`${key}\` — I do not know that instruction.`, {
+          hint: `The instructions I know are: ${ACTION_ORDER.join(', ')}, text, note.`,
+        });
+      }
+    }
+    try {
+      await runStep(page, step, n);
+    } catch (cause) {
+      if (isExpected(cause)) throw cause;
+      throw new StaysFixedError(`${describeStep(step, n)} did not work.`, {
+        hint: messageOf(cause),
+        cause,
+      });
+    }
+  }
+}
+
+/**
+ * @param {import('../types.js').PageApi} page
+ * @param {import('../types.js').Step} step
+ * @param {number} n
+ * @returns {Promise<void>}
+ */
+async function runStep(page, step, n) {
+  let did = false;
+  if (step.goto !== undefined) {
+    await page.goto(step.goto);
+    did = true;
+  }
+  if (step.waitFor !== undefined) {
+    await page.waitFor(step.waitFor);
+    did = true;
+  }
+  if (step.scrollTo !== undefined) {
+    await page.scrollTo(step.scrollTo);
+    did = true;
+  }
+  if (step.hover !== undefined) {
+    await page.hover(step.hover);
+    did = true;
+  }
+  if (step.click !== undefined) {
+    await page.click(step.click);
+    did = true;
+  }
+  if (step.type !== undefined) {
+    if (typeof step.text !== 'string') {
+      throw new StaysFixedError(`Step ${n} says to type into \`${step.type}\` but never says what to type.`, {
+        hint: 'Add `text` to that step, for example { type: "#search", text: "hello" }.',
+      });
+    }
+    await page.type(step.type, step.text);
+    did = true;
+  }
+  if (step.press !== undefined) {
+    await page.press(step.press);
+    did = true;
+  }
+  if (step.evaluate !== undefined) {
+    await page.evaluate(step.evaluate);
+    did = true;
+  }
+  if (step.waitForGone !== undefined) {
+    await page.waitForGone(step.waitForGone);
+    did = true;
+  }
+  if (step.wait !== undefined) {
+    await page.wait(step.wait);
+    did = true;
+  }
+  if (!did && step.note === undefined) {
+    throw new StaysFixedError(`Step ${n} does not say what to do.`, {
+      hint: `Give it one of: ${ACTION_ORDER.join(', ')}.`,
+    });
+  }
+}
+
+/**
+ * A step named the way a human would say it out loud.
+ * @param {import('../types.js').Step} step
+ * @param {number} n
+ * @returns {string}
+ */
+function describeStep(step, n) {
+  for (const key of ACTION_ORDER) {
+    const value = step[key];
+    if (value !== undefined) return `Step ${n} (${key} ${JSON.stringify(value)})`;
+  }
+  return `Step ${n}`;
+}
+
+/**
+ * Width and height straight out of the PNG header.
+ *
+ * Settle compares a frame every quarter second; decoding a whole retina
+ * screenshot just to learn its size would dominate the run. The IHDR chunk always
+ * starts at byte 8, so width and height sit at bytes 16 and 20, big-endian.
+ *
+ * @param {Buffer} buffer
+ * @returns {{width: number, height: number}}
+ */
+export function pngSize(buffer) {
+  if (!buffer || buffer.length < 24) {
+    throw new StaysFixedError('That screenshot came back empty — there is no picture to measure.');
+  }
+  if (buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) {
+    throw new StaysFixedError('That screenshot is not a PNG picture.');
+  }
+  return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
+}

package/src/picture/compare.js

@@ -0,0 +1,103 @@
+/**
+ * Comparing a fresh picture with the approved one.
+ *
+ * Two things here decide whether the tool is believed or ignored:
+ * the masks are painted onto BOTH pictures (so hiding a clock hides it on the
+ * old picture too, and adding a mask never forces a re-approval), and the
+ * verdict is a pixel count against an explicit allowance rather than a vague
+ * "looks close enough".
+ */
+
+import { PNG } from 'pngjs';
+import pixelmatch from 'pixelmatch';
+import { paintMasks } from '../freeze/mask.js';
+import { DEFAULT_TOLERANCE } from '../core/config.js';
+import { StaysFixedError } from '../core/errors.js';
+
+/**
+ * @param {Buffer} approvedBuf
+ * @param {Buffer} actualBuf
+ * @param {import('../types.js').ToleranceConfig} tolerance
+ * @param {import('../types.js').MaskRect[]} [maskRects]
+ * @returns {import('../types.js').CompareReport}
+ */
+export function comparePng(approvedBuf, actualBuf, tolerance, maskRects = []) {
+  const approved = decode(approvedBuf, 'the approved picture');
+  const actual = decode(actualBuf, 'the new picture');
+
+  const approvedSize = { width: approved.width, height: approved.height };
+  const size = { width: actual.width, height: actual.height };
+
+  if (approvedSize.width !== size.width || approvedSize.height !== size.height) {
+    // Different sizes cannot be compared pixel by pixel at all. The caller
+    // explains it in words instead of showing a meaningless diff.
+    return { equal: false, diffPixels: 0, diffRatio: 0, diffPng: null, sizeMismatch: true, size, approvedSize };
+  }
+
+  if (maskRects.length > 0) {
+    paintMasks(approved, maskRects);
+    paintMasks(actual, maskRects);
+  }
+
+  const { width, height } = size;
+  const diff = new PNG({ width, height });
+  const diffPixels = pixelmatch(approved.data, actual.data, diff.data, width, height, {
+    threshold: tolerance.threshold ?? DEFAULT_TOLERANCE.threshold,
+    // Read this one carefully: includeAA:true INCLUDES anti-aliased pixels in the
+    // count, which is stricter. So "ignore anti-aliasing noise" means includeAA:false.
+    includeAA: !(tolerance.antialiasing ?? DEFAULT_TOLERANCE.antialiasing),
+    alpha: 0.3,
+    diffColor: [255, 0, 0],
+    diffColorAlt: [255, 140, 0],
+  });
+
+  const total = width * height;
+  const allowed = tolerance.maxPixels ?? Math.floor(total * (tolerance.pixels ?? DEFAULT_TOLERANCE.pixels));
+  const equal = diffPixels <= allowed;
+
+  return {
+    equal,
+    diffPixels,
+    diffRatio: total > 0 ? diffPixels / total : 0,
+    diffPng: equal ? null : PNG.sync.write(diff),
+    sizeMismatch: false,
+    size,
+    approvedSize,
+  };
+}
+
+/**
+ * @param {Buffer} buffer
+ * @param {string} what
+ * @returns {PNG}
+ */
+function decode(buffer, what) {
+  try {
+    return PNG.sync.read(buffer);
+  } catch (cause) {
+    throw new StaysFixedError(`I could not open ${what} — the file is damaged or is not a PNG.`, {
+      hint: 'Delete it and take the picture again.',
+      cause,
+    });
+  }
+}
+
+/**
+ * One sentence a non-technical person can act on.
+ * @param {import('../types.js').CompareReport} report
+ * @param {string} name
+ * @returns {string}
+ */
+export function describeDifference(report, name) {
+  if (report.sizeMismatch) {
+    const was = `${report.approvedSize.width}x${report.approvedSize.height}`;
+    const now = `${report.size.width}x${report.size.height}`;
+    return `${name} changed size — it was ${was}, now it is ${now}.`;
+  }
+  if (report.equal) {
+    return `${name} looks exactly like the approved picture.`;
+  }
+  const pixels = report.diffPixels.toLocaleString('en-US');
+  const percent = (report.diffRatio * 100).toFixed(2);
+  return `${name} looks different — ${pixels} pixels changed (${percent}% of the picture).`;
+}

package/src/picture/run.js

@@ -0,0 +1,284 @@
+/**
+ * The picture net: photograph every screen, compare each against its approved
+ * picture, and report in plain language.
+ *
+ * Two decisions in here are deliberate and load-bearing.
+ * First, a screen that looks different is photographed again before the tool
+ * believes it — one bad frame must never be reported as a regression, and a
+ * screen that only matched on the second try is flagged so the flake register
+ * catches it and a human eventually deletes or fixes it.
+ * Second, nothing is ever approved automatically. A screen with no approved
+ * picture is reported as new and waits for a person to look at it.
+ */
+
+import { captureScreen } from './capture.js';
+import { comparePng, describeDifference } from './compare.js';
+import { readApproved, writeResult, writeDiff, approveFromResult } from './store.js';
+import { settingsForScreen } from '../core/config.js';
+import { approvedPicture, resultPicture } from '../core/paths.js';
+import { platformTag } from '../drive/find.js';
+import { resetWindow } from '../drive/launch.js';
+import { gitInfo } from '../core/git.js';
+import { messageOf } from '../core/errors.js';
+import { detail } from '../core/log.js';
+
+/**
+ * A picture result plus the one extra fact the flake register needs: whether it
+ * only agreed with the approved picture after being photographed again.
+ * @typedef {import('../types.js').PictureResult & {retriedToPass?: boolean}} PictureRunResult
+ */
+
+/**
+ * @param {import('../types.js').Project} project
+ * @param {import('../types.js').LaunchedApp} app
+ * @param {{
+ *   only?: string[],
+ *   record?: boolean,
+ *   updateNew?: boolean,
+ *   onResult?: (r: PictureRunResult) => void,
+ *   retries?: number,
+ *   tool?: string,
+ *   signal?: AbortSignal,
+ * }} [opts]
+ * @returns {Promise<PictureRunResult[]>}
+ */
+export async function runPictures(project, app, opts = {}) {
+  const { config } = project;
+  // The launcher hands back the public page surface; the capture loop needs the
+  // plumbing underneath it (init scripts, console buffer) that the same object carries.
+  const page = /** @type {import('../types.js').PageHandle} */ (app.page);
+  const only = opts.only && opts.only.length > 0 ? new Set(opts.only) : null;
+  const retries = Math.max(0, opts.retries ?? config.retries);
+  const here = platformTag();
+  // A desktop app has no url to go back to between screens, so it gets a reload instead.
+  // A web app does not need one: every screen starts with a `goto`.
+  const reset = config.app.kind === 'electron' ? () => resetWindow(app) : undefined;
+
+  /** @type {PictureRunResult[]} */
+  const results = [];
+
+  for (const screen of config.screens) {
+    if (opts.signal?.aborted) break;
+    if (only && !only.has(screen.name)) continue;
+
+    if (screen.skip) {
+      results.push(
+        finish(opts, {
+          name: screen.name,
+          describe: screen.describe,
+          status: 'skipped',
+          message: `${screen.name} is switched off in the config.`,
+          durationMs: 0,
+        }),
+      );
+      continue;
+    }
+
+    results.push(
+      await runOneScreen(project, page, screen, {
+        record: opts.record,
+        updateNew: opts.updateNew,
+        tool: opts.tool,
+        onResult: opts.onResult,
+        retries,
+        here,
+        reset,
+      }),
+    );
+  }
+
+  return results;
+}
+
+/**
+ * @param {import('../types.js').Project} project
+ * @param {import('../types.js').PageHandle} page
+ * @param {import('../types.js').ScreenConfig} screen
+ * @param {{record?: boolean, updateNew?: boolean, tool?: string, onResult?: (r: PictureRunResult) => void, retries: number, here: string, reset?: () => Promise<void>}} ctx
+ * @returns {Promise<PictureRunResult>}
+ */
+async function runOneScreen(project, page, screen, ctx) {
+  const { config, paths } = project;
+  const started = Date.now();
+  const settings = settingsForScreen(config, screen);
+  const approvedPaths = approvedPicture(paths, screen.name);
+  const resultPaths = resultPicture(paths, screen.name);
+
+  const approved = await readApproved(paths, screen.name);
+  const platformNote = platformWarning(approved?.meta?.platform, ctx.here);
+
+  /** @type {string[]} */
+  let consoleErrors = [];
+  /** @type {{width: number, height: number}|undefined} */
+  let size;
+  /** @type {import('../types.js').CompareReport|null} */
+  let compare = null;
+  let attempts = 0;
+
+  try {
+    // Photograph, and if it disagrees with the approved picture, photograph again
+    // before believing it. One flickering frame is not a regression.
+    for (let attempt = 1; attempt <= ctx.retries + 1; attempt++) {
+      attempts = attempt;
+      // Every screen starts from the same place.
+      //
+      // A web screen begins with a `goto`, so it is naturally isolated. A desktop app
+      // has no front door, and without this the ORDER of the screens quietly decides
+      // the result — the screen that collapses a sidebar left it collapsed for every
+      // screen and every guard after it. (A walk deliberately does NOT do this: a walk
+      // is one journey through the app, in order.)
+      if (ctx.reset) await ctx.reset();
+      const shot = await captureScreen(page, screen, settings, {
+        fixturesDir: paths.fixtures,
+        record: ctx.record ?? false,
+        timeoutMs: settings.freeze.settle?.timeoutMs,
+      });
+      consoleErrors = shot.consoleErrors;
+      size = { width: shot.width, height: shot.height };
+
+      await writeResult(paths, screen.name, shot.png, {
+        deviceScaleFactor: settings.viewport.deviceScaleFactor,
+        describe: screen.describe,
+      });
+
+      if (!approved) break;
+
+      compare = comparePng(approved.png, shot.png, settings.tolerance, shot.masks);
+      if (compare.equal) break;
+      if (attempt <= ctx.retries) {
+        detail(`${screen.name} looked different on attempt ${attempt} — taking it again.`);
+      }
+    }
+  } catch (error) {
+    return finish(ctx, {
+      name: screen.name,
+      describe: screen.describe,
+      status: 'failed',
+      message: join(`${screen.name} could not be photographed. ${messageOf(error)}`, platformNote),
+      durationMs: Date.now() - started,
+      attempts,
+      consoleErrors: consoleErrors.length > 0 ? consoleErrors : undefined,
+    });
+  }
+
+  /** @type {PictureRunResult} */
+  const base = {
+    name: screen.name,
+    describe: screen.describe,
+    status: 'passed',
+    durationMs: Date.now() - started,
+    attempts,
+    actualPath: resultPaths.png,
+    size,
+    consoleErrors: consoleErrors.length > 0 ? consoleErrors : undefined,
+  };
+
+  if (!approved) {
+    if (ctx.updateNew) {
+      // Only ever reached from a flag a person typed, and only for a screen that
+      // has never been approved. A picture that already exists always needs eyes.
+      const git = await gitInfo(paths.root);
+      const meta = await approveFromResult(paths, screen.name, {
+        git,
+        tool: ctx.tool,
+        describe: screen.describe,
+        deviceScaleFactor: settings.viewport.deviceScaleFactor,
+      });
+      return finish(ctx, {
+        ...base,
+        status: 'new',
+        approvedPath: approvedPaths.png,
+        approvedSize: { width: meta.width, height: meta.height },
+        message: join(`${screen.name} had no approved picture — this one was saved as the first.`, platformNote),
+      });
+    }
+    return finish(ctx, {
+      ...base,
+      status: 'new',
+      message: join(
+        `${screen.name} has no approved picture yet — look at it and run \`staysfixed approve ${screen.name}\` if it is right.`,
+        platformNote,
+      ),
+    });
+  }
+
+  if (!compare) {
+    // Cannot happen: with an approved picture every attempt compares. Kept so a
+    // future edit that breaks that assumption fails loudly instead of silently passing.
+    return finish(ctx, {
+      ...base,
+      status: 'failed',
+      message: join(`${screen.name} was photographed but never compared.`, platformNote),
+    });
+  }
+
+  const common = {
+    ...base,
+    approvedPath: approvedPaths.png,
+    approvedSize: compare.approvedSize,
+    diffPixels: compare.diffPixels,
+    diffRatio: compare.diffRatio,
+  };
+
+  if (compare.equal) {
+    const retriedToPass = attempts > 1;
+    return finish(ctx, {
+      ...common,
+      status: 'passed',
+      retriedToPass,
+      message: retriedToPass
+        ? join(
+            `${screen.name} matched, but only when it was photographed again — it may be unreliable.`,
+            platformNote,
+          )
+        : platformNote || undefined,
+    });
+  }
+
+  /** @type {string|undefined} */
+  let diffPath;
+  if (compare.diffPng) diffPath = await writeDiff(paths, screen.name, compare.diffPng);
+
+  const what = describeDifference(compare, screen.name);
+  const next = compare.sizeMismatch
+    ? 'There is no difference picture for a size change — open the new picture and look at it.'
+    : `Open the difference picture, and if the new look is right run \`staysfixed approve ${screen.name}\`.`;
+
+  return finish(ctx, {
+    ...common,
+    status: 'changed',
+    diffPath,
+    message: join(what, next, platformNote),
+  });
+}
+
+/**
+ * Font rendering differs between operating systems, so a picture approved on one
+ * and checked on another is the single most common false alarm. Say it; never
+ * fail on it alone.
+ * @param {string|undefined} approvedOn
+ * @param {string} here
+ * @returns {string}
+ */
+function platformWarning(approvedOn, here) {
+  if (!approvedOn || approvedOn === here) return '';
+  return `Careful: this picture was approved on ${approvedOn} and checked on ${here}. Text is drawn differently on each, so a small difference here may mean nothing.`;
+}
+
+/**
+ * @param {{onResult?: (r: PictureRunResult) => void}} ctx
+ * @param {PictureRunResult} result
+ * @returns {PictureRunResult}
+ */
+function finish(ctx, result) {
+  if (ctx.onResult) ctx.onResult(result);
+  return result;
+}
+
+/**
+ * @param {...(string|undefined)} parts
+ * @returns {string}
+ */
+function join(...parts) {
+  return parts.filter(Boolean).join(' ');
+}