tuffgal 0.1.0-alpha.0

package/src/commands/supervise.ts

@@ -0,0 +1,267 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { createWriteStream, mkdirSync, statSync } from 'node:fs';
+import { createConnection } from 'node:net';
+import { join, resolve } from 'node:path';
+import type { ResolvedConfig, DevServerBridge } from '../config.ts';
+
+const HEARTBEAT_FILE = '.heartbeat';
+const SIGTERM_GRACE_MS = 5_000;
+const HEALTHCHECK_PROBE_TIMEOUT_MS = 2_000;
+
+const DEFAULT_HEALTHCHECK_INTERVAL_MS = 30_000;
+const DEFAULT_IDLE_LIMIT_MS = 10 * 60_000;
+const DEFAULT_MAX_RUNTIME_MS = 60 * 60_000;
+const DEFAULT_MAX_RESPAWNS = 3;
+
+export interface SuperviseOptions {
+  /** Milliseconds between port + heartbeat probes. Default 30_000. */
+  healthcheckIntervalMs?: number;
+  /**
+   * Window with no `tuffgal run` heartbeat before the supervisor self-
+   * terminates. Default 10 minutes. The heartbeat file lives at
+   * `config.paths.report/.heartbeat` and is touched by every `tuffgal
+   * run` invocation.
+   */
+  idleLimitMs?: number;
+  /** Wall-clock cap, end-to-end. Default 60 minutes. */
+  maxRuntimeMs?: number;
+  /**
+   * How many times the supervisor may respawn `devServers.command` after
+   * an unhealthy probe or unexpected exit before giving up. Default 3.
+   */
+  maxRespawns?: number;
+}
+
+/**
+ * Long-running supervisor around `config.devServers.command`. Solves
+ * three problems observed during heavy harness iteration:
+ *
+ *   1. Hot-reload rot — after many file-watch restarts the framework
+ *      drifts into a broken state. Supervisor probes every declared
+ *      healthcheck URL and restarts the whole tree on failure.
+ *   2. Forgotten dev servers — supervisor self-terminates after a
+ *      wall-clock cap and after an idle window with no heartbeat from
+ *      `tuffgal run`.
+ *   3. Manual signal management — SIGINT/SIGTERM teardown kills the
+ *      whole detached process group, not just the shell wrapper.
+ *
+ * Each invocation of `tuffgal run` writes
+ * `config.paths.report/.heartbeat` (ISO timestamp). Supervisor reads
+ * its mtime every healthcheck. If `Date.now() - mtime > idleLimitMs`,
+ * supervisor shuts down. Missing heartbeat file is treated as "no runs
+ * yet", not stale, so a fresh shell does not self-terminate immediately.
+ */
+export async function supervise(
+  config: ResolvedConfig,
+  options: SuperviseOptions = {},
+): Promise<void> {
+  if (!config.devServers) {
+    throw new Error(
+      'tuffgal supervise requires a `devServers` block in tuffgal.config.ts. ' +
+        'Declare `devServers: { command, healthCheck: [...] }` and try again.',
+    );
+  }
+
+  const healthcheckIntervalMs =
+    options.healthcheckIntervalMs ??
+    numberFromEnv(
+      'TUFFGAL_HEALTHCHECK_INTERVAL_MS',
+      DEFAULT_HEALTHCHECK_INTERVAL_MS,
+    );
+  const idleLimitMs =
+    options.idleLimitMs ??
+    numberFromEnv('TUFFGAL_IDLE_LIMIT_MS', DEFAULT_IDLE_LIMIT_MS);
+  const maxRuntimeMs =
+    options.maxRuntimeMs ??
+    numberFromEnv('TUFFGAL_MAX_RUNTIME_MS', DEFAULT_MAX_RUNTIME_MS);
+  const maxRespawns =
+    options.maxRespawns ??
+    numberFromEnv('TUFFGAL_MAX_RESPAWNS', DEFAULT_MAX_RESPAWNS);
+
+  mkdirSync(config.paths.report, { recursive: true });
+  const logPath = join(config.paths.report, 'dev-servers.log');
+  const heartbeatPath = join(config.paths.report, HEARTBEAT_FILE);
+
+  process.stdout.write(
+    [
+      'Starting tuffgal supervisor.',
+      `  Command:      ${config.devServers.command}`,
+      `  Healthcheck:  ${config.devServers.healthCheck.map((entry) => entry.url).join(', ')}`,
+      `  Log:          ${logPath}`,
+      `  Heartbeat:    ${heartbeatPath}`,
+      `  Interval:     ${healthcheckIntervalMs}ms`,
+      `  Idle limit:   ${idleLimitMs}ms`,
+      `  Max runtime:  ${maxRuntimeMs}ms`,
+      `  Max respawns: ${maxRespawns}`,
+      '',
+    ].join('\n'),
+  );
+
+  const startedAt = Date.now();
+  let respawns = 0;
+  let child = spawnDevServers(config.devServers, config.rootDir, logPath);
+  let stopped = false;
+
+  const teardown = async (reason: string): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    process.stdout.write(`Supervisor stopping: ${reason}\n`);
+    await stopChild(child, config.devServers);
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => void teardown('SIGINT received'));
+  process.on('SIGTERM', () => void teardown('SIGTERM received'));
+
+  while (!stopped) {
+    await sleep(healthcheckIntervalMs);
+    if (stopped) break;
+
+    if (Date.now() - startedAt > maxRuntimeMs) {
+      await teardown('wall-clock cap reached');
+      return;
+    }
+    if (heartbeatIsStale(heartbeatPath, idleLimitMs)) {
+      await teardown(`no tuffgal run activity in ${idleLimitMs}ms`);
+      return;
+    }
+
+    if (child.exitCode !== null) {
+      respawns += 1;
+      if (respawns > maxRespawns) {
+        await teardown(
+          `dev servers exited and respawn budget exhausted (${maxRespawns})`,
+        );
+        return;
+      }
+      process.stdout.write(
+        `Dev servers exited; respawning (${respawns}/${maxRespawns}).\n`,
+      );
+      child = spawnDevServers(config.devServers, config.rootDir, logPath);
+      continue;
+    }
+
+    const healthy = await probeAllHealthchecks(config.devServers);
+    if (!healthy) {
+      respawns += 1;
+      if (respawns > maxRespawns) {
+        await teardown(
+          `unhealthy ports and respawn budget exhausted (${maxRespawns})`,
+        );
+        return;
+      }
+      process.stdout.write(
+        `Healthcheck failed; killing + respawning (${respawns}/${maxRespawns}).\n`,
+      );
+      await stopChild(child, config.devServers);
+      child = spawnDevServers(config.devServers, config.rootDir, logPath);
+    }
+  }
+}
+
+function spawnDevServers(
+  devServers: DevServerBridge,
+  rootDir: string,
+  logPath: string,
+): ChildProcess {
+  const stream = createWriteStream(logPath, { flags: 'a' });
+  stream.write(`\n--- dev servers spawned @ ${new Date().toISOString()} ---\n`);
+  const cwd = devServers.cwd ? resolve(rootDir, devServers.cwd) : rootDir;
+  const child = spawn('sh', ['-c', devServers.command], {
+    cwd,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    detached: true,
+  });
+  if (child.stdout) child.stdout.pipe(stream);
+  if (child.stderr) child.stderr.pipe(stream);
+  return child;
+}
+
+async function probeAllHealthchecks(
+  devServers: DevServerBridge,
+): Promise<boolean> {
+  const results = await Promise.all(
+    devServers.healthCheck.map((entry) => probeUrl(entry.url)),
+  );
+  return results.every((result) => result);
+}
+
+function probeUrl(url: string): Promise<boolean> {
+  const parsed = new URL(url);
+  const port = parsed.port
+    ? Number(parsed.port)
+    : parsed.protocol === 'https:'
+      ? 443
+      : 80;
+  const host = parsed.hostname;
+  return new Promise((resolveProbe) => {
+    const socket = createConnection({ port, host });
+    const cleanup = (result: boolean): void => {
+      socket.removeAllListeners();
+      socket.destroy();
+      resolveProbe(result);
+    };
+    socket.once('connect', () => cleanup(true));
+    socket.once('error', () => cleanup(false));
+    socket.once('timeout', () => cleanup(false));
+    socket.setTimeout(HEALTHCHECK_PROBE_TIMEOUT_MS);
+  });
+}
+
+function heartbeatIsStale(heartbeatPath: string, idleLimitMs: number): boolean {
+  try {
+    const stat = statSync(heartbeatPath);
+    return Date.now() - stat.mtimeMs > idleLimitMs;
+  } catch {
+    // No heartbeat yet — count grace period from supervisor start so a
+    // fresh shell does not immediately kill itself.
+    return false;
+  }
+}
+
+async function stopChild(
+  child: ChildProcess,
+  devServers: DevServerBridge | undefined,
+): Promise<void> {
+  if (!child.pid || child.exitCode !== null) return;
+  const pid = child.pid;
+  const signal = devServers?.shutdownSignal ?? 'SIGTERM';
+  const graceMs = devServers?.shutdownGraceMs ?? SIGTERM_GRACE_MS;
+  await new Promise<void>((resolveOuter) => {
+    let done = false;
+    const finish = (): void => {
+      if (done) return;
+      done = true;
+      resolveOuter();
+    };
+    child.once('exit', finish);
+    try {
+      process.kill(-pid, signal);
+    } catch {
+      finish();
+      return;
+    }
+    setTimeout(() => {
+      if (child.exitCode === null) {
+        try {
+          process.kill(-pid, 'SIGKILL');
+        } catch {
+          // Already gone.
+        }
+      }
+    }, graceMs);
+  });
+}
+
+function numberFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolveSleep) => {
+    setTimeout(resolveSleep, ms);
+  });
+}

package/src/config.ts

@@ -0,0 +1,222 @@
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+import { access } from 'node:fs/promises';
+
+/**
+ * Bridge supplied by the consumer for tearing down + reseeding their test
+ * database between runs. Both methods are optional: a static-site project
+ * with no backend can omit `database` entirely.
+ */
+export interface DatabaseBridge {
+  /**
+   * Wipes the test database and reseeds the deterministic test user. Called
+   * once before the scheduler dispatches the first story.
+   */
+  reset?: () => Promise<void>;
+  /**
+   * Map of named fixture functions. Each story declares
+   * `fixtures: ["name"]` and the runner invokes the matching entry before
+   * launching the browser. Fixtures must be idempotent (apply twice safely)
+   * because Tuffgal applies them per-story without per-story DB reset.
+   */
+  fixtures?: Record<string, () => Promise<void>>;
+}
+
+/**
+ * Bridge for the optional `--manage-servers` mode. When supplied, Tuffgal
+ * spawns the command in a detached process group, polls the healthcheck
+ * URLs, and tears the process tree down at end-of-run.
+ */
+export interface DevServerBridge {
+  /** Shell command. Run via `sh -c` so pipes and `&&` work. */
+  command: string;
+  /** Working directory, relative to the config file's location. */
+  cwd?: string;
+  /**
+   * URLs to poll before considering the dev server stack ready. Each entry
+   * is probed via TCP `connect` (not HTTP) so self-signed certificates and
+   * 404 responses do not block readiness.
+   */
+  healthCheck: Array<{ url: string; timeoutMs?: number }>;
+  /** Signal sent on shutdown. Defaults to `SIGTERM`. */
+  shutdownSignal?: NodeJS.Signals;
+  /** Grace period before `SIGKILL`. Defaults to 5000. */
+  shutdownGraceMs?: number;
+}
+
+/**
+ * Static paths Tuffgal reads + writes. All relative to the config file's
+ * location.
+ */
+export interface PathsConfig {
+  /** Action JSON files. Recurses into subdirectories. */
+  actions: string;
+  /** Story JSON files. Recurses into subdirectories. */
+  stories: string;
+  /** Committed PNG baselines + a11y snapshots. */
+  baselines: string;
+  /** Generated HTML report + traces. Gitignore this. */
+  report: string;
+  /** Storage state cache for `produces`/`needs` label inheritance. */
+  authState?: string;
+}
+
+/**
+ * CI-friendly result emitters. Optional. The default reporter (HTML at
+ * `paths.report/index.html`) always runs.
+ */
+export interface CiConfig {
+  /** Path to write SARIF results.json for GitHub code scanning. */
+  sarif?: string;
+  /** Paths to advertise as `actions/upload-artifact` candidates. */
+  artifactPaths?: string[];
+}
+
+/**
+ * Root config shape passed to `defineConfig()`.
+ */
+export interface TuffgalConfig {
+  /** Where actions, stories, baselines, and reports live. */
+  paths: PathsConfig;
+  /** Base URL of the running app. */
+  baseUrl: string;
+  /**
+   * Origin (scheme + host + port) of the consumer's API. Intercept
+   * patterns that begin with this host stay scoped to API traffic and
+   * avoid accidentally matching Vite source modules.
+   */
+  apiHost?: string;
+  /**
+   * localStorage keys to persist across stories. Cookie-based apps may
+   * leave this empty — cookies always persist via Playwright's storage
+   * state.
+   */
+  storageStatePins?: string[];
+  /** Browser viewport. Defaults to 1280x800. */
+  viewport?: { width: number; height: number };
+  /** Default Playwright locator + action timeout. Defaults to 10_000. */
+  defaultTimeoutMs?: number;
+  /** Default navigation timeout. Defaults to 15_000. */
+  navigationTimeoutMs?: number;
+  /**
+   * Frozen ISO timestamp passed to `page.clock.install`. Pins
+   * `Date.now()` for any relative-time UI ("3 minutes ago") so screenshot
+   * diffs do not flicker.
+   */
+  frozenTime?: string;
+  /** Worker pool size. Default: `min(cpus / 2, 4)`. */
+  workers?: number;
+  /** Consumer-provided database bridge. */
+  database?: DatabaseBridge;
+  /** Consumer-provided dev-server bridge (only used with `--manage-servers`). */
+  devServers?: DevServerBridge;
+  /**
+   * Path to a markdown file listing the consumer's user journeys (one per
+   * row in a single markdown table). Tuffgal counts how many stories
+   * declare a matching `flow:` field and exposes the ratio as
+   * `customCoverage.flows` in the report.
+   */
+  flowInventory?: string;
+  /** CI integration knobs. */
+  ci?: CiConfig;
+}
+
+/**
+ * Resolved config — every optional field replaced with a concrete value
+ * so downstream consumers can rely on the shape without per-field `??`.
+ * Returned by `loadConfig`. Not part of the public surface.
+ */
+export interface ResolvedConfig {
+  rootDir: string;
+  paths: Required<PathsConfig>;
+  baseUrl: string;
+  apiHost: string | undefined;
+  storageStatePins: string[];
+  viewport: { width: number; height: number };
+  defaultTimeoutMs: number;
+  navigationTimeoutMs: number;
+  frozenTime: string;
+  workers: number | undefined;
+  database: DatabaseBridge | undefined;
+  devServers: DevServerBridge | undefined;
+  flowInventory: string | undefined;
+  ci: CiConfig | undefined;
+}
+
+const DEFAULTS = {
+  viewport: { width: 1280, height: 800 },
+  defaultTimeoutMs: 10_000,
+  navigationTimeoutMs: 15_000,
+  frozenTime: '2026-01-15T12:00:00.000Z',
+  authStateRelative: '.auth',
+} as const;
+
+/**
+ * Helper that returns its argument unchanged. Use in a `tuffgal.config.ts`
+ * file to get full TypeScript type-checking against `TuffgalConfig`.
+ */
+export function defineConfig(config: TuffgalConfig): TuffgalConfig {
+  return config;
+}
+
+/**
+ * Locates + dynamically imports the consumer's `tuffgal.config.ts` (or
+ * `.js`) from `cwd`. Throws a descriptive error when no config is found
+ * so the user knows where to put the file.
+ */
+export async function loadConfig(cwd: string): Promise<ResolvedConfig> {
+  const candidates = ['tuffgal.config.ts', 'tuffgal.config.js'];
+  for (const candidate of candidates) {
+    const absolute = resolve(cwd, candidate);
+    try {
+      await access(absolute);
+    } catch {
+      continue;
+    }
+    const module = (await import(pathToFileURL(absolute).href)) as {
+      default?: TuffgalConfig;
+    };
+    if (!module.default) {
+      throw new Error(
+        `${candidate} found at ${absolute} but does not have a default export. ` +
+          `Did you forget \`export default defineConfig({ ... })\`?`,
+      );
+    }
+    return resolveConfig(module.default, resolve(cwd, '.'));
+  }
+  throw new Error(
+    `No tuffgal.config.ts or tuffgal.config.js found in ${cwd}. Run ` +
+      `\`npx tuffgal init\` to scaffold one.`,
+  );
+}
+
+function resolveConfig(input: TuffgalConfig, rootDir: string): ResolvedConfig {
+  return {
+    rootDir,
+    paths: {
+      actions: resolve(rootDir, input.paths.actions),
+      stories: resolve(rootDir, input.paths.stories),
+      baselines: resolve(rootDir, input.paths.baselines),
+      report: resolve(rootDir, input.paths.report),
+      authState: resolve(
+        rootDir,
+        input.paths.authState ?? DEFAULTS.authStateRelative,
+      ),
+    },
+    baseUrl: input.baseUrl,
+    apiHost: input.apiHost,
+    storageStatePins: input.storageStatePins ?? [],
+    viewport: input.viewport ?? DEFAULTS.viewport,
+    defaultTimeoutMs: input.defaultTimeoutMs ?? DEFAULTS.defaultTimeoutMs,
+    navigationTimeoutMs:
+      input.navigationTimeoutMs ?? DEFAULTS.navigationTimeoutMs,
+    frozenTime: input.frozenTime ?? DEFAULTS.frozenTime,
+    workers: input.workers,
+    database: input.database,
+    devServers: input.devServers,
+    flowInventory: input.flowInventory
+      ? resolve(rootDir, input.flowInventory)
+      : undefined,
+    ci: input.ci,
+  };
+}

package/src/coverage/flows.ts

@@ -0,0 +1,90 @@
+import { readFile } from 'node:fs/promises';
+import type { StoryFile } from '../schema/load.ts';
+
+export interface FlowCoverage {
+  total: number;
+  covered: number;
+  ratio: number;
+  missing: string[];
+}
+
+/**
+ * Compares the journeys catalogued in the consumer's `flowInventory`
+ * markdown table with the stories under the configured `stories/`
+ * directory. A story is counted as covering a journey when its `flow`
+ * field matches a journey row (case- and whitespace-insensitive).
+ *
+ * Returns 1.0 coverage when no inventory was configured or the file
+ * cannot be read, so a fresh checkout without an inventory does not
+ * break the report.
+ */
+export async function computeFlowCoverage(
+  inventoryPath: string | undefined,
+  stories: StoryFile[],
+): Promise<FlowCoverage> {
+  if (!inventoryPath) {
+    return { total: 0, covered: 0, ratio: 1, missing: [] };
+  }
+  let raw: string;
+  try {
+    raw = await readFile(inventoryPath, 'utf8');
+  } catch {
+    return { total: 0, covered: 0, ratio: 1, missing: [] };
+  }
+  const journeys = parseJourneyTable(raw);
+  const claimed = new Set(
+    stories
+      .map((entry) => entry.story.flow)
+      .filter((flow): flow is string => typeof flow === 'string')
+      .map((flow) => normalise(flow)),
+  );
+  const missing: string[] = [];
+  for (const journey of journeys) {
+    if (!claimed.has(normalise(journey))) {
+      missing.push(journey);
+    }
+  }
+  const total = journeys.length;
+  const covered = total - missing.length;
+  return {
+    total,
+    covered,
+    ratio: total === 0 ? 1 : covered / total,
+    missing,
+  };
+}
+
+/**
+ * Pulls the first column of every body row from the first markdown table
+ * in the document. Skips the header row and the dash separator row. The
+ * inventory contains one row per journey, so this is the journey list.
+ */
+function parseJourneyTable(markdown: string): string[] {
+  const lines = markdown.split(/\r?\n/);
+  const journeys: string[] = [];
+  let insideTable = false;
+  let skippedHeaderAndSeparator = 0;
+  for (const line of lines) {
+    if (!line.startsWith('|')) {
+      if (insideTable) break;
+      continue;
+    }
+    insideTable = true;
+    if (skippedHeaderAndSeparator < 2) {
+      skippedHeaderAndSeparator += 1;
+      continue;
+    }
+    const firstCell = line
+      .split('|')
+      .map((cell) => cell.trim())
+      .filter((cell) => cell.length > 0)[0];
+    if (firstCell) {
+      journeys.push(firstCell);
+    }
+  }
+  return journeys;
+}
+
+function normalise(text: string): string {
+  return text.toLowerCase().replaceAll(/\s+/g, ' ').trim();
+}

package/src/coverage/screens.ts

@@ -0,0 +1,52 @@
+import { access, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export interface ScreenCoverage {
+  total: number;
+  covered: number;
+  ratio: number;
+  missing: string[];
+}
+
+const SCREENS_SUBDIR = 'screens';
+
+/**
+ * Walks every `visit-*.json` action under `<actions>/screens/` and reports
+ * how many have a baseline screenshot committed at
+ * `<baselines>/<action-name>/0.png`. The action name is taken from the
+ * filename (minus the `.json` suffix), which by convention matches the
+ * action's declared `action` field.
+ */
+export async function computeScreenCoverage(
+  actionsDir: string,
+  baselinesDir: string,
+): Promise<ScreenCoverage> {
+  const screensRoot = join(actionsDir, SCREENS_SUBDIR);
+  let entries: string[] = [];
+  try {
+    entries = await readdir(screensRoot);
+  } catch {
+    return { total: 0, covered: 0, ratio: 1, missing: [] };
+  }
+  const screenNames = entries
+    .filter((name) => name.endsWith('.json'))
+    .map((name) => name.replace(/\.json$/i, ''))
+    .sort();
+  const missing: string[] = [];
+  for (const name of screenNames) {
+    const baseline = join(baselinesDir, name, '0.png');
+    try {
+      await access(baseline);
+    } catch {
+      missing.push(name);
+    }
+  }
+  const total = screenNames.length;
+  const covered = total - missing.length;
+  return {
+    total,
+    covered,
+    ratio: total === 0 ? 1 : covered / total,
+    missing,
+  };
+}

package/src/index.ts

@@ -0,0 +1,28 @@
+// Public API surface. Consumer projects import only what is re-exported here.
+// Anything not re-exported is internal and may break between minor releases.
+
+export {
+  defineConfig,
+  loadConfig,
+  type TuffgalConfig,
+  type ResolvedConfig,
+  type DatabaseBridge,
+  type DevServerBridge,
+  type PathsConfig,
+  type CiConfig,
+} from './config.ts';
+
+export { runAll, type RunCliOptions } from './runner/run.ts';
+export { approveAll, type ApproveOptions } from './runner/approve.ts';
+export { supervise, type SuperviseOptions } from './commands/supervise.ts';
+export { init, type InitOptions } from './commands/init.ts';
+
+export type { Action, Step, Hint } from './schema/action.ts';
+export type { Story } from './schema/story.ts';
+export type {
+  RunResult,
+  StoryResult,
+  ActionResult,
+  ActionStatus,
+  StoryStatus,
+} from './schema/result.ts';