tuffgal 0.1.0-alpha.0

package/src/schema/result.ts

@@ -0,0 +1,83 @@
+/**
+ * Outcome model. The runner emits a `RunResult` per invocation; the reporter
+ * consumes it to render the HTML report. Status values map onto the three
+ * outcomes the framework distinguishes:
+ *
+ * - `pass`  — action succeeded and screenshot matched baseline (or no baseline).
+ * - `changed` — action succeeded but screenshot drifted past the threshold.
+ *   The story does not fail. The user reviews and either approves the new
+ *   baseline or files a bug.
+ * - `failed` — a step threw. The story fails fast and skips any later actions.
+ */
+export type ActionStatus = 'pass' | 'changed' | 'failed' | 'skipped' | 'new';
+
+export interface ActionResult {
+  action: string;
+  parameters?: Record<string, string>;
+  status: ActionStatus;
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  /**
+   * Number of the step (0-indexed) that failed. Undefined if the action
+   * succeeded or was skipped without running.
+   */
+  failedStepIndex?: number;
+  failureMessage?: string;
+  baselinePath?: string;
+  actualPath?: string;
+  diffPath?: string;
+  diffPixels?: number;
+  diffRatio?: number;
+  /** Mean SSIM score of baseline vs actual — see screenshots/diff.ts. */
+  ssimScore?: number;
+  /**
+   * `true` when the captured page accessibility tree differs from the
+   * baseline tree. Informational only — does not gate pass/changed by
+   * itself; pixel/SSIM still drives the status.
+   */
+  a11yChanged?: boolean;
+}
+
+export type StoryStatus = 'pass' | 'changed' | 'failed';
+
+export interface StoryResult {
+  story: string;
+  file: string;
+  status: StoryStatus;
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  actions: ActionResult[];
+  /** Absolute path to the Playwright trace zip when the story failed. */
+  tracePath?: string;
+}
+
+export interface CoverageMetric {
+  total: number;
+  covered: number;
+  ratio: number;
+  missing: string[];
+}
+
+export interface RunResult {
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  totals: {
+    stories: number;
+    passed: number;
+    changed: number;
+    failed: number;
+  };
+  /**
+   * Custom coverage metrics layered on top of V8 line coverage:
+   * `screens` = baselined visit-* actions / declared screens,
+   * `flows` = stories with `flow` tag / journeys in `flowInventory`.
+   */
+  customCoverage: {
+    screens: CoverageMetric;
+    flows: CoverageMetric;
+  };
+  stories: StoryResult[];
+}

package/src/schema/story.ts

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+
+export const storyStepSchema = z.object({
+  action: z.string().min(1),
+  parameters: z.record(z.string(), z.string()).optional(),
+});
+
+export type StoryStep = z.infer<typeof storyStepSchema>;
+
+/**
+ * Stories declare prerequisite **labels** through `needs` and emit
+ * **labels** through `produces`. The harness uses these to build a
+ * dependency DAG, validate uniqueness, detect cycles, and inherit storage
+ * state from a producer onto its consumers. A produced label's storage
+ * state lives at `.auth/<label>.json` and is loaded as the initial state
+ * for every consumer.
+ *
+ * `storageState: "logged-in"` is retained as syntactic sugar — equivalent
+ * to `needs: ["logged-in"]`. New stories should prefer the explicit form.
+ */
+export const storySchema = z.object({
+  story: z.string().min(1),
+  storageState: z.enum(['fresh', 'logged-in']).optional(),
+  needs: z.array(z.string().min(1)).optional(),
+  produces: z.array(z.string().min(1)).optional(),
+  /**
+   * Named fixtures (see `runner/fixtures/`) applied to the test database
+   * BEFORE the story's browser context launches. Useful for stories that
+   * need preloaded state — e.g. a "mark all as read" flow needs links to
+   * exist first. Fixtures apply sequentially in declaration order. Note
+   * that the test DB is shared across parallel stories, so two stories
+   * applying conflicting fixtures should be serialised via `needs`/
+   * `produces` labels.
+   */
+  fixtures: z.array(z.string().min(1)).optional(),
+  /**
+   * Tag declaring which user-journey row in the configured
+   * `flowInventory` markdown table this story covers. Used to compute
+   * flow-coverage in the report. Match is case- and
+   * whitespace-insensitive.
+   */
+  flow: z.string().min(1).optional(),
+  actions: z.array(storyStepSchema).min(1),
+});
+
+export type Story = z.infer<typeof storySchema>;
+
+/**
+ * Normalises `storageState` into the canonical `needs` array so the
+ * scheduler only has one input format to reason about.
+ */
+export function normalisedNeeds(story: Story): string[] {
+  const explicit = story.needs ?? [];
+  if (story.storageState === 'logged-in' && !explicit.includes('logged-in')) {
+    return [...explicit, 'logged-in'];
+  }
+  return explicit;
+}

package/src/screenshots/baselineStore.ts

@@ -0,0 +1,114 @@
+import {
+  access,
+  copyFile,
+  mkdir,
+  readFile,
+  rm,
+  writeFile,
+} from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+
+export interface BaselinePaths {
+  baseline: string;
+  actual: string;
+  diff: string;
+  a11yBaseline: string;
+  a11yActual: string;
+}
+
+export interface StoreOptions {
+  baselinesDir: string;
+  reportDir: string;
+  storyFile: string;
+  actionName: string;
+}
+
+/**
+ * Computes deterministic paths for the baseline (committed), actual
+ * (regenerated each run), and diff (regenerated when a baseline existed and
+ * the diff was non-zero) PNGs. Centralised so the runner and the CLI's
+ * `approve` command agree on layout.
+ */
+export function pathsFor(options: StoreOptions): BaselinePaths {
+  const storySlug = options.storyFile.replace(/\.json$/i, '');
+  return {
+    baseline: join(options.baselinesDir, options.actionName, '0.png'),
+    actual: join(
+      options.reportDir,
+      'screenshots',
+      storySlug,
+      `${options.actionName}.actual.png`,
+    ),
+    diff: join(
+      options.reportDir,
+      'screenshots',
+      storySlug,
+      `${options.actionName}.diff.png`,
+    ),
+    a11yBaseline: join(options.baselinesDir, options.actionName, 'a11y.yaml'),
+    a11yActual: join(
+      options.reportDir,
+      'screenshots',
+      storySlug,
+      `${options.actionName}.a11y.yaml`,
+    ),
+  };
+}
+
+export async function readBaseline(path: string): Promise<Buffer | undefined> {
+  try {
+    await access(path);
+  } catch {
+    return undefined;
+  }
+  return readFile(path);
+}
+
+export async function readJsonBaseline(
+  path: string,
+): Promise<string | undefined> {
+  try {
+    await access(path);
+  } catch {
+    return undefined;
+  }
+  return readFile(path, 'utf8');
+}
+
+export async function writeText(path: string, content: string): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, content, 'utf8');
+}
+
+export async function writePng(path: string, png: Buffer): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, png);
+}
+
+export async function deleteIfExists(path: string): Promise<void> {
+  try {
+    await rm(path);
+  } catch {
+    // file was not there; nothing to do
+  }
+}
+
+export async function copyToBaseline(
+  actualPath: string,
+  baselinePath: string,
+): Promise<void> {
+  await mkdir(dirname(baselinePath), { recursive: true });
+  await copyFile(actualPath, baselinePath);
+}
+
+/**
+ * Turns an absolute screenshot path into a path that is relative to the
+ * generated report directory so the HTML report can reference it with a
+ * portable `src` attribute.
+ */
+export function pathRelativeToReport(
+  reportDir: string,
+  absolute: string,
+): string {
+  return relative(reportDir, absolute);
+}

package/src/screenshots/capture.ts

@@ -0,0 +1,19 @@
+import type { Locator, Page } from 'playwright';
+
+/**
+ * Full-page screenshot with animations disabled, caret hidden, and any masks
+ * applied so the same UI renders to bit-identical pixels across runs. Masks
+ * are Playwright locators whose bounding boxes are blacked out before the
+ * image is encoded — use them to neutralise randomised or time-based regions.
+ */
+export async function capturePage(
+  page: Page,
+  masks: Locator[] = [],
+): Promise<Buffer> {
+  return page.screenshot({
+    fullPage: true,
+    animations: 'disabled',
+    caret: 'hide',
+    mask: masks,
+  });
+}

package/src/screenshots/diff.ts

@@ -0,0 +1,101 @@
+import pixelmatch from 'pixelmatch';
+import { PNG } from 'pngjs';
+import { ssim as computeSsim } from 'ssim.js';
+
+export interface DiffOutcome {
+  diffPng: Buffer;
+  diffPixels: number;
+  totalPixels: number;
+  diffRatio: number;
+  /**
+   * Mean Structural Similarity score for the two images. 1.0 = identical;
+   * 0.99 = very close (sub-pixel layout shifts, font rendering); 0.95 =
+   * noticeable change; under 0.9 = obvious change. SSIM is more
+   * perceptually accurate than pixel-by-pixel diffing because it weights
+   * pixels by their structural context.
+   */
+  ssimScore: number;
+}
+
+/**
+ * Compares two PNG buffers. Computes SSIM (the perceptual gate) and a
+ * pixelmatch overlay (the visualisation). Both are returned so the
+ * runner can use SSIM for pass/changed decisions while the reporter
+ * still has a red-highlight diff image to show the human.
+ *
+ * Dimension mismatch is a regression on its own — fail loudly.
+ */
+export function diffPngs(
+  baseline: Buffer,
+  actual: Buffer,
+  pixelThreshold: number,
+): DiffOutcome {
+  const baselinePng = PNG.sync.read(baseline);
+  const actualPng = PNG.sync.read(actual);
+  if (
+    baselinePng.width !== actualPng.width ||
+    baselinePng.height !== actualPng.height
+  ) {
+    throw new ScreenshotSizeMismatchError(
+      { width: baselinePng.width, height: baselinePng.height },
+      { width: actualPng.width, height: actualPng.height },
+    );
+  }
+  const diffPng = new PNG({
+    width: baselinePng.width,
+    height: baselinePng.height,
+  });
+  const diffPixels = pixelmatch(
+    baselinePng.data,
+    actualPng.data,
+    diffPng.data,
+    baselinePng.width,
+    baselinePng.height,
+    { threshold: pixelThreshold },
+  );
+  const totalPixels = baselinePng.width * baselinePng.height;
+  const ssimResult = computeSsim(
+    {
+      data: new Uint8ClampedArray(
+        baselinePng.data.buffer,
+        baselinePng.data.byteOffset,
+        baselinePng.data.byteLength,
+      ),
+      width: baselinePng.width,
+      height: baselinePng.height,
+    },
+    {
+      data: new Uint8ClampedArray(
+        actualPng.data.buffer,
+        actualPng.data.byteOffset,
+        actualPng.data.byteLength,
+      ),
+      width: actualPng.width,
+      height: actualPng.height,
+    },
+    { ssim: 'bezkrovny' },
+  );
+  return {
+    diffPng: PNG.sync.write(diffPng),
+    diffPixels,
+    totalPixels,
+    diffRatio: diffPixels / totalPixels,
+    ssimScore: ssimResult.mssim,
+  };
+}
+
+export class ScreenshotSizeMismatchError extends Error {
+  readonly baseline: { width: number; height: number };
+  readonly actual: { width: number; height: number };
+  constructor(
+    baseline: { width: number; height: number },
+    actual: { width: number; height: number },
+  ) {
+    super(
+      `Screenshot dimensions changed: baseline ${baseline.width}x${baseline.height}, actual ${actual.width}x${actual.height}`,
+    );
+    this.name = 'ScreenshotSizeMismatchError';
+    this.baseline = baseline;
+    this.actual = actual;
+  }
+}