stillhunter 0.0.1 → 0.2.1

package/dist/finish.mjs

@@ -0,0 +1,357 @@
+// finish: checks the hunter's claims, makes it account for every failed check
+// and for what it never covered, re-runs each claimed probe twice from scratch
+// and reports what reproduced.
+import { copyFileSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { claimProblems, parseFinishInput, unaccounted } from './accounting.mjs';
+import { atOrAbove, block, brief, flag, FOLDER, loadRun, minutesAndSeconds, readConfig, readStdin, remaining, severityOf } from './common.mjs';
+import { coverage, coverageLines } from './coverage.mjs';
+import { hiddenLine } from './flags.mjs';
+import { KILL_GRACE_MS, PROBE_TIMEOUT_MS, runProbe, sessionPath } from './probe-host.mjs';
+import { RESET_TIMEOUT_MS, resetAndSignIn, signedOut } from './reset.mjs';
+import { allResults, latestResults, outcome, probesDir, signInProbe } from './results.mjs';
+import { answers } from './target.mjs';
+export const COVERAGE_REFUSAL_SECONDS = 180;
+// The proof's own budget, and the most a finish call may take in all: the
+// hunter's shell gives up on a command after 600 s and would then have no
+// report at all. No claimed probe's proof starts after the budget; a proof
+// started before it, retries and sign-ins included, is held to the wall limit.
+export const PROOF_SECONDS = 420;
+export const FINISH_WALL_SECONDS = 540;
+// Lowers the wall limit, never raises it, so the tests can reach it in
+// seconds. The hunter's shell runs only the runner and sets no variable.
+export const WALL_ENV = 'STILLHUNTER_FINISH_WALL_SECONDS';
+export const NOT_RERUN = 'Not re-run in time, not counted';
+export const NOT_RERUNNABLE = 'Could not re-run, not counted';
+const INTERMITTENT = 'Intermittent (1 of 2), not counted';
+const RUN_MARGIN_MS = 10_000;
+const MIN_RUN_MS = 15_000;
+function readState(dir) {
+    try {
+        return { accountingRefused: false, coverageRefused: false, ...JSON.parse(readFileSync(path.join(dir, 'finish-state.json'), 'utf8')) };
+    }
+    catch {
+        return { accountingRefused: false, coverageRefused: false };
+    }
+}
+function refuse(ctx, lines) {
+    process.stdout.write(block([`stillhunter | finish refused | run ${ctx.runId}`, ...lines]));
+    process.exit(2);
+}
+// How long one proof run may be given so that, with the browser's start, the
+// kill grace and teardown, it ends before the wall deadline; null when no
+// useful run fits.
+export function runTimeout(now, wallDeadline) {
+    const left = wallDeadline - now - KILL_GRACE_MS - RUN_MARGIN_MS;
+    return left < MIN_RUN_MS ? null : Math.min(PROBE_TIMEOUT_MS, left);
+}
+export function wallSeconds(env = process.env) {
+    const value = env[WALL_ENV];
+    const n = Number(value);
+    return value !== undefined && /^\d+$/.test(value) && n >= 1 && n < FINISH_WALL_SECONDS ? n : FINISH_WALL_SECONDS;
+}
+// A file written whole or not at all: a finish stopped while writing leaves
+// the previous version, never a truncated one.
+export function writeWhole(file, text, rename = renameSync) {
+    const temp = `${file}.${process.pid}.tmp`;
+    writeFileSync(temp, text);
+    // On Windows a rename onto a file another process has open (a reader, or a
+    // virus scanner looking at it) fails for a moment with EPERM, EACCES or
+    // EBUSY, so it is tried again for up to about 2 s.
+    for (let wait = 10;; wait *= 2) {
+        try {
+            return rename(temp, file);
+        }
+        catch (e) {
+            const code = e.code;
+            if (wait > 1000 || !['EPERM', 'EACCES', 'EBUSY'].includes(code ?? ''))
+                throw e;
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, wait);
+        }
+    }
+}
+// Claimed probes in the order they are proved: by the most severe finding
+// each one carries, then in the order they were claimed.
+export function proofOrder(findings) {
+    const rank = new Map();
+    for (const f of findings)
+        rank.set(f.probe, Math.min(rank.get(f.probe) ?? 3, ORDER[severityOf(f.severity)]));
+    return [...rank.keys()].sort((a, b) => rank.get(a) - rank.get(b));
+}
+// One re-run from scratch, or null when the wall deadline leaves no room for
+// it (or cut it short), so the probe counts as not re-run in time.
+async function proofRun(p, probe, label) {
+    const proofDir = path.join(p.dir, 'proof');
+    const host = (name, timeoutMs) => runProbe({
+        runDir: p.dir, probe: name, probeFile: path.join(probesDir(p.dir), `${name}.mjs`), target: p.ctx.target, browser: p.ctx.browser,
+        signInVariables: p.ctx.signInVariables, shotsDir: path.join(proofDir, 'shots'), timeoutMs,
+    });
+    const save = (r, file) => { writeFileSync(path.join(proofDir, file), JSON.stringify(r, null, 2) + '\n'); return r; };
+    // App data and, with it, any session go back to the start, and the probe
+    // that signed in before the claimed probe's hunt run signs in again (none,
+    // if none did then), so the claimed probe starts as its hunt run did. A
+    // hunt run with no record of that falls back to the run's sign-in probe.
+    let before = null;
+    if (p.config.reset) {
+        const resetMs = Math.min(RESET_TIMEOUT_MS, p.wallDeadline - Date.now() - KILL_GRACE_MS - RUN_MARGIN_MS - MIN_RUN_MS);
+        if (resetMs <= 0)
+            return null;
+        const huntStart = p.latest.get(probe)?.before;
+        before = await resetAndSignIn({
+            root: p.root, config: p.config, probe, signIn: huntStart ? (huntStart.signIn?.probe ?? null) : p.signIn, sessionFile: sessionPath(p.dir), resetTimeoutMs: resetMs,
+            signInTimeoutMs: () => runTimeout(Date.now(), p.wallDeadline),
+            runSignIn: async (name, t) => save(await host(name, t), `${probe}.${label}.signin.result.json`),
+        });
+        if (!before)
+            return null;
+        const s = before.signIn;
+        if (before.resetError)
+            p.notes.push(`before ${probe} ${label}: ${before.resetError}`);
+        if (s?.error)
+            p.notes.push(`before ${probe} ${label}: ${s.savedSession ? `the sign-in probe "${s.probe}" saved a session, then failed` : `signing in again with "${s.probe}" failed`}: ${s.error}`);
+    }
+    const t = runTimeout(Date.now(), p.wallDeadline);
+    if (t === null)
+        return null;
+    const r = await host(probe, t);
+    if (before)
+        r.before = before;
+    save(r, `${probe}.${label}.result.json`);
+    // A run the deadline shortened and that then timed out says nothing about the bug.
+    if (r.hostError === 'timeout' && t < PROBE_TIMEOUT_MS)
+        return null;
+    return r;
+}
+async function prove(p, probe) {
+    const runs = [];
+    for (const k of [1, 2]) {
+        let r = await proofRun(p, probe, String(k));
+        if (!r)
+            return null;
+        if (r.hostError) {
+            if ((await answers(p.ctx.target)) === null) {
+                p.notes.push(`${probe} re-run ${k}: ${r.error}; the app no longer answers at ${p.ctx.target}, so it was not retried`);
+            }
+            else {
+                p.notes.push(`${probe} re-run ${k}: ${r.error}; retried once`);
+                r = await proofRun(p, probe, `${k}.retry`);
+                if (!r)
+                    return null;
+            }
+        }
+        runs.push(r);
+    }
+    return runs;
+}
+// Confirmed when the check failed in both re-runs, intermittent when in one.
+// Otherwise it did not reproduce only if the check ran and held both times. A
+// re-run says nothing about the bug when it never got to the check (the probe
+// threw or timed out first, or the check is not in its result), or when it
+// did not start as the hunt run did (its reset or its sign-in failed, and the
+// hunt run's had not): then the claim could not be re-run.
+export function judge(f, runs, hunt) {
+    const severity = severityOf(f.severity);
+    const reads = runs.map((r, i) => readRun(r, i + 1, f.check, hunt));
+    const failed = reads.flatMap((o) => (o.state === 'failed' ? [o.detail] : []));
+    if (failed.length === runs.length)
+        return { kind: 'confirmed', f: { ...f, severity, runs: failed } };
+    const why = reads.map((o, i) => (o.state === 'none' ? o.why : `${o.state} on re-run ${i + 1}`)).join('; ');
+    const kind = failed.length ? 'intermittent' : reads.every((o) => o.state === 'held') ? 'dropped' : 'notRerunnable';
+    return { kind, f: { ...f, severity, why } };
+}
+function readRun(r, k, check, hunt) {
+    if (r.before?.resetError && !hunt?.before?.resetError)
+        return { state: 'none', why: `re-run ${k} does not count: the reset before it failed, as it had not before the hunt run` };
+    if (signedOut(r.before) && !signedOut(hunt?.before))
+        return { state: 'none', why: `re-run ${k} does not count: it ran signed out, as the sign-in after the reset failed, which it had not before the hunt run` };
+    const o = outcome(r, check);
+    return o.state === 'missing' ? { state: 'none', why: notReached(r, k) } : o;
+}
+function notReached(r, k) {
+    const setup = [
+        ...(r.before?.resetError ? ['the reset before it failed'] : []),
+        ...(signedOut(r.before) ? ['it ran signed out: the sign-in after the reset failed'] : []),
+    ];
+    return `re-run ${k} never reached the check${setup.length ? ` (${setup.join('; ')})` : ''}: ${r.error ? brief(r.error) : 'the check is not in its result'}`;
+}
+// Said first among the claims that could not be re-run: when the sign-in
+// after a reset failed with no session saved, the re-runs it came before ran
+// signed out. Only the re-runs of the probes behind the listed claims count,
+// so a list cut by severity, or a claim found signed out and re-run so, says
+// nothing about it.
+export function signInLine(byProbe, probes = byProbe.keys()) {
+    const runs = [...new Set(probes)].flatMap((p) => byProbe.get(p) ?? []).filter((r) => r.before);
+    const failed = runs.flatMap((r) => (signedOut(r.before) ? [r.before.signIn] : []));
+    const first = failed[0];
+    if (first) {
+        return failed.length === runs.length
+            ? `After reset, the sign-in probe "${first.probe}" failed: ${brief(first.error)}. Each claimed probe then ran signed out.`
+            : `After reset, the sign-in probe "${first.probe}" failed before ${failed.length} of ${runs.length} re-runs: ${brief(first.error)}. Those re-runs ran signed out.`;
+    }
+    return null;
+}
+const ORDER = { high: 0, medium: 1, low: 2 };
+// --proof-seconds lowers the proof budget, never raises it.
+export function proofSeconds(args) {
+    const value = flag(args, '--proof-seconds');
+    if (value === null)
+        return args.includes('--proof-seconds') ? null : PROOF_SECONDS;
+    const n = Number(value);
+    return /^\d+$/.test(value) && n >= 1 && n <= PROOF_SECONDS ? n : null;
+}
+export async function finish(args) {
+    const startedAt = Date.now();
+    const { ctx, dir, root } = loadRun(args);
+    const budget = proofSeconds(args);
+    if (budget === null)
+        refuse(ctx, [`--proof-seconds is a whole number from 1 to ${PROOF_SECONDS}.`]);
+    const parsed = parseFinishInput(readStdin());
+    if (!parsed.ok)
+        refuse(ctx, [parsed.message, 'Fix it and call finish again.']);
+    const input = parsed.input;
+    const latest = latestResults(dir);
+    const problems = claimProblems(input, latest);
+    if (problems.length)
+        refuse(ctx, [...problems.map((p) => `- ${p}`), 'Fix these and call finish again. Nothing was re-run.']);
+    const state = readState(dir);
+    const missing = unaccounted(input, latest);
+    const signIn = signInProbe(dir);
+    const cov = coverage(allResults(dir), ctx.target);
+    const left = remaining(ctx);
+    const refusal = [];
+    if (missing.length && !state.accountingRefused) {
+        state.accountingRefused = true;
+        refusal.push(`${missing.length} failed check${missing.length === 1 ? ' is' : 's are'} in neither findings nor dismissed:`, ...missing.map((m) => `- ${m.probe} / ${m.check}: ${m.detail}`), 'Claim each as a finding, or dismiss it with {"probe", "check", "reason"}: probe-wrong, duplicate (with "duplicateOf": the finding\'s title), intended, not-this-change or resilience-gap.');
+    }
+    if (left > COVERAGE_REFUSAL_SECONDS && (cov.neverFaulted.length || cov.neverNarrow.length) && !state.coverageRefused) {
+        state.coverageRefused = true;
+        refusal.push(`${left} s of the budget remain and the coverage has gaps:`, ...coverageLines(cov).slice(1), `Probe what belongs to ${ctx.mode === 'feature' ? 'this feature' : 'this change'}. What is outside it can stay: calling finish again accepts.`);
+    }
+    if (refusal.length) {
+        writeFileSync(path.join(dir, 'finish-state.json'), JSON.stringify(state, null, 2) + '\n');
+        refuse(ctx, [...refusal, 'This is said once; the next finish accepts and records what is still open.']);
+    }
+    writeFileSync(path.join(dir, 'leads.json'), JSON.stringify(input.leads, null, 2) + '\n');
+    const config = readConfig(root);
+    mkdirSync(path.join(dir, 'proof'), { recursive: true });
+    const proofDeadline = startedAt + budget * 1000;
+    const prover = { dir, root, ctx, config, latest, signIn, notes: [], wallDeadline: startedAt + wallSeconds() * 1000 };
+    const order = proofOrder(input.findings);
+    const byProbe = new Map();
+    const settle = (complete) => writeReport({ ctx, dir, input, missing, cov, latest, byProbe, notes: prover.notes, startedAt, budget, wall: wallSeconds(), order, complete });
+    // Most severe first, and the report is written before the proof and again
+    // after every proved probe, so a finish that is stopped part way still
+    // leaves its findings.
+    settle(false);
+    for (const probe of order) {
+        if (Date.now() >= proofDeadline)
+            break;
+        // Snapshot the exact script being proved, so keep saves the probe that was
+        // confirmed even if it is later re-run under the same name.
+        copyFileSync(path.join(probesDir(dir), `${probe}.mjs`), path.join(dir, 'proof', `${probe}.mjs`));
+        const runs = await prove(prover, probe);
+        if (runs)
+            byProbe.set(probe, runs);
+        settle(false);
+    }
+    process.stdout.write(block(settle(true)));
+}
+// Writes findings.json and report.txt from the probes proved so far and
+// returns the printed report. A claim whose probe has no proof yet is listed
+// as not re-run in time and not counted.
+function writeReport(r) {
+    const { ctx, dir, input, missing, cov } = r;
+    const confirmed = [];
+    const intermittent = [];
+    const dropped = [];
+    const notRerunnable = [];
+    const notRerun = [];
+    for (const f of input.findings) {
+        const runs = r.byProbe.get(f.probe);
+        if (!runs) {
+            notRerun.push({ ...f, severity: severityOf(f.severity), why: 'no time to re-run it before the proof budget or the wall limit ran out' });
+            continue;
+        }
+        const j = judge(f, runs, r.latest.get(f.probe));
+        if (j.kind === 'confirmed')
+            confirmed.push(j.f);
+        else if (j.kind === 'intermittent')
+            intermittent.push(j.f);
+        else if (j.kind === 'notRerunnable')
+            notRerunnable.push(j.f);
+        else
+            dropped.push(j.f);
+    }
+    const bySeverity = (a, b) => ORDER[a.severity] - ORDER[b.severity];
+    confirmed.sort(bySeverity);
+    notRerunnable.sort(bySeverity);
+    notRerun.sort(bySeverity);
+    const shown = confirmed.filter((f) => atOrAbove(f.severity, ctx.minSeverity));
+    const shownIntermittent = intermittent.filter((f) => atOrAbove(f.severity, ctx.minSeverity));
+    const shownNotRerunnable = notRerunnable.filter((f) => atOrAbove(f.severity, ctx.minSeverity));
+    const shownNotRerun = notRerun.filter((f) => atOrAbove(f.severity, ctx.minSeverity));
+    const erred = [...r.latest].filter(([, x]) => x.error).map(([name, x]) => `${name}: ${x.error}`);
+    const finishSeconds = Math.round((Date.now() - r.startedAt) / 1000);
+    const proof = { budgetSeconds: r.budget, wallLimitSeconds: r.wall, finishSeconds, complete: r.complete, order: r.order, proved: [...r.byProbe.keys()] };
+    writeWhole(path.join(dir, 'findings.json'), JSON.stringify({ confirmed, intermittent, dropped, notRerunnable, notRerun, dismissed: input.dismissed, unaccounted: missing, coverage: cov, notes: r.notes, proof }, null, 2) + '\n');
+    const reportPath = `${FOLDER}/runs/${ctx.runId}/report.txt`;
+    const elapsed = Math.round((Date.now() - ctx.startedAt) / 1000);
+    const header = [
+        `stillhunter | ${minutesAndSeconds(elapsed)} | run ${ctx.runId} | ${confirmed.length} confirmed, ${intermittent.length} intermittent, ${dropped.length} did not reproduce${notRerunnable.length ? `, ${notRerunnable.length} could not re-run` : ''}${notRerun.length ? `, ${notRerun.length} not re-run in time` : ''}`,
+        `target: ${ctx.target} | asked: ${ctx.focus ?? 'the change'}${ctx.minSeverity === 'low' ? '' : ` | showing ${ctx.minSeverity} severity and above`}`,
+    ];
+    const findingLines = (list) => list.flatMap((f, i) => [
+        `${i + 1}. ${f.severity.toUpperCase().padEnd(6)} ${f.title}`,
+        ...(f.where ? [`   where: ${f.where}`] : []),
+        ...(f.expected ? [`   expected: ${f.expected}`] : []),
+        `   saw: ${f.runs[0] ?? f.actual ?? ''}`,
+        `   repro: ${FOLDER}/runs/${ctx.runId}/probes/${f.probe}.mjs (expectation "${f.check}")`,
+    ]);
+    const listBlock = (title, list, why = false) => (list.length
+        ? ['', `${title}:`, ...list.map((f) => `- ${f.title} (probe ${f.probe}, expectation "${f.check}")${why ? `: ${f.why}` : ''}`)]
+        : []);
+    // The sign-in's failure comes first, as it is the likely reason for the rest.
+    const rerunBlock = (list) => {
+        if (!list.length)
+            return [];
+        const signInTrouble = signInLine(r.byProbe, list.map((f) => f.probe));
+        return [
+            '', `${NOT_RERUNNABLE}:`,
+            ...(signInTrouble ? [signInTrouble] : []),
+            ...list.map((f) => `- ${f.title} (probe ${f.probe}, expectation "${f.check}"): ${f.why}`),
+        ];
+    };
+    const coverageSection = ['', 'Coverage', ...coverageLines(cov)];
+    // Findings below the floor, of every kind, are hidden from the printed
+    // report; report.txt lists all of them.
+    const hidden = hiddenLine((confirmed.length - shown.length) + (intermittent.length - shownIntermittent.length) + (notRerunnable.length - shownNotRerunnable.length) + (notRerun.length - shownNotRerun.length), ctx.minSeverity, reportPath);
+    const printed = [
+        ...header,
+        ...(shown.length ? ['', 'Findings (each reproduced twice from scratch)', ...findingLines(shown)] : ['', confirmed.length ? 'No finding at this severity reproduced.' : 'No finding reproduced.']),
+        ...(hidden ? [hidden] : []),
+        ...listBlock(INTERMITTENT, shownIntermittent, true),
+        ...rerunBlock(shownNotRerunnable),
+        ...listBlock(NOT_RERUN, shownNotRerun),
+        ...coverageSection,
+    ];
+    const full = [
+        ...header,
+        ...(confirmed.length ? ['', 'Findings (each reproduced twice from scratch)', ...findingLines(confirmed)] : ['', 'No finding reproduced.']),
+        ...listBlock(INTERMITTENT, intermittent, true),
+        ...rerunBlock(notRerunnable),
+        ...listBlock(NOT_RERUN, notRerun),
+        ...(dropped.length ? ['', 'Did not reproduce:', ...dropped.map((f) => `- ${f.title}: ${f.why}`)] : []),
+        ...(input.dismissed.length ? ['', 'Dismissed:', ...input.dismissed.map(dismissalLine)] : []),
+        ...(missing.length ? ['', 'Unaccounted (failed, neither claimed nor dismissed):', ...missing.map(unaccountedLine)] : []),
+        ...(erred.length ? ['', 'Probes whose latest run ended in an error:', ...erred.map((e) => `- ${e}`)] : []),
+        ...(r.notes.length ? ['', 'Proof notes:', ...r.notes.map((n) => `- ${n}`)] : []),
+        ...coverageSection,
+        '',
+        `Proof: ${r.byProbe.size} of ${r.order.length} claimed probes re-run in ${finishSeconds} s (budget ${r.budget} s)${r.complete ? '' : '; finish had not ended when this was written'}`,
+        `Leads (not tested to proof): ${input.leads.length}, in ${FOLDER}/runs/${ctx.runId}/leads.json`,
+    ];
+    writeWhole(path.join(dir, 'report.txt'), full.join('\n') + '\n');
+    return printed;
+}
+const dismissalLine = (d) => `- ${d.probe} / ${d.check}: ${d.reason}${d.duplicateOf ? ` of "${d.duplicateOf}"` : ''}${d.why ? `: ${d.why}` : ''}`;
+const unaccountedLine = (u) => `- ${u.probe} / ${u.check}: ${u.detail}`;

package/dist/flags.mjs

@@ -0,0 +1,49 @@
+// The flags `start` takes. The hunter turns what the person asked for into
+// these; the runner then holds the hunt to them.
+import { SEVERITIES } from './common.mjs';
+export const DEPTHS = { quick: 4, default: 10, deep: 25 };
+const MAX_MINUTES = 120;
+export function parseStartFlags(args) {
+    let depth = 'default';
+    let minutes = null;
+    let minSeverity = 'low';
+    let focus = null;
+    for (let i = 0; i < args.length; i++) {
+        const name = args[i];
+        const value = args[i + 1];
+        if (name !== '--depth' && name !== '--minutes' && name !== '--min-severity' && name !== '--focus') {
+            return { ok: false, message: `Unknown start option ${name}. start takes --depth, --minutes, --min-severity and --focus.` };
+        }
+        if (value === undefined)
+            return { ok: false, message: `${name} needs a value.` };
+        i += 1;
+        if (name === '--depth') {
+            if (!(value in DEPTHS))
+                return { ok: false, message: `--depth is quick, default or deep, not "${value}".` };
+            depth = value;
+        }
+        else if (name === '--minutes') {
+            const n = Number(value);
+            if (!/^\d+$/.test(value) || n < 1 || n > MAX_MINUTES)
+                return { ok: false, message: `--minutes is a whole number from 1 to ${MAX_MINUTES}, not "${value}".` };
+            minutes = n;
+        }
+        else if (name === '--min-severity') {
+            if (!SEVERITIES.includes(value))
+                return { ok: false, message: `--min-severity is high, medium or low, not "${value}".` };
+            minSeverity = value;
+        }
+        else {
+            const text = value.trim().replace(/\s+/g, ' ');
+            focus = text === '' ? null : text.slice(0, 500);
+        }
+    }
+    return { ok: true, flags: { depth, minutes: minutes ?? DEPTHS[depth], minSeverity, focus } };
+}
+// The line that ends a report when the severity floor hid some findings.
+export function hiddenLine(hidden, minSeverity, reportPath) {
+    if (hidden === 0)
+        return null;
+    const below = SEVERITIES.slice(SEVERITIES.indexOf(minSeverity) + 1).join('/');
+    return `${hidden} ${below} finding${hidden === 1 ? '' : 's'} hidden: ${reportPath}`;
+}

package/dist/hunt.mjs

@@ -0,0 +1,193 @@
+#!/usr/bin/env node
+// Stillhunter's runner: the model reads the change, forms hypotheses and tests
+// them with probe scripts; the runner keeps the scripts, runs them in a
+// contained child process against a loopback-only browser and, at the end,
+// re-runs every claimed finding from scratch so only reproducible ones are
+// reported. Findings worth keeping become tests that replay with no model.
+//   stillhunter start [--depth quick|default|deep] [--minutes N] [--min-severity high|medium|low] [--focus "<text>"]
+//   stillhunter probe --run <id> --name <name>     (script on stdin)
+//   stillhunter finish --run <id> [--proof-seconds N]  (findings JSON on stdin)
+//   stillhunter diff [--file <path>]               (the change, or one file's whole diff)
+//   stillhunter keep --run <id> --probe <name> --check <check> [--as <name>]
+//   stillhunter replay [names] [--watch]
+import { randomBytes } from 'node:crypto';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { chooseBrowser, installedOnThisMachine } from './browser.mjs';
+import { baseRef, changeList, changeText, collectChange, ensureRunsIgnored, fileDiff, readSpecs } from './change.mjs';
+import { block, brief, fail, FOLDER, flag, loadRun, PROBE_NAME, readConfig, readStdin, remaining, repoRoot, runsDir, signInVariableNames, stamp, findTargetUrl } from './common.mjs';
+import { finish } from './finish.mjs';
+import { parseStartFlags } from './flags.mjs';
+import { keep, replay } from './keep.mjs';
+import { runProbe, sessionPath } from './probe-host.mjs';
+import { resetAndSignIn, resetApp, seconds } from './reset.mjs';
+import { latestFile, nextRunNumber, numberedFile, probesDir, signInProbe } from './results.mjs';
+import { answers } from './target.mjs';
+const NO_CHANGE = 'No change found. Name a feature to hunt, for example: /stillhunter:hunt the checkout flow';
+async function start(args) {
+    const root = repoRoot();
+    const parsed = parseStartFlags(args);
+    if (!parsed.ok)
+        fail(parsed.message);
+    const { depth, minutes, minSeverity, focus } = parsed.flags;
+    const base = baseRef(root);
+    const change = collectChange(root, base);
+    if (change.empty && !focus)
+        fail(NO_CHANGE);
+    const config = readConfig(root);
+    const target = await findTargetUrl(root, config);
+    if (!target)
+        fail(`No running app found. Start your dev server, or set "target": { "url": ... } in ${FOLDER}/config.json.`);
+    if ((await answers(target)) === null)
+        fail(`Nothing answers at ${target}. Start your dev server first.`);
+    const choice = chooseBrowser(config.browser ?? null, installedOnThisMachine());
+    if (!choice.ok)
+        fail(choice.message);
+    // Every probe and every proof re-run starts from the reset, so a reset that
+    // does not work must stop the hunt here, not surface at finish. Its time
+    // counts against the budget like any other.
+    const now = Date.now();
+    const resetError = await resetApp(root, config);
+    if (resetError)
+        fail(`The app's reset failed, so the hunt did not start: ${resetError}\nFix the reset ("reset" in ${FOLDER}/config.json, run in the repository root), then start again.`);
+    const resetSeconds = seconds(Date.now() - now);
+    const runId = `${stamp()}-hunt-${randomBytes(3).toString('hex').slice(0, 4)}`;
+    const dir = path.join(runsDir(root), runId);
+    mkdirSync(probesDir(dir), { recursive: true });
+    ensureRunsIgnored(path.join(root, FOLDER, '.gitignore'));
+    const mode = change.empty ? 'feature' : 'change';
+    const signInVariables = signInVariableNames();
+    const ctx = {
+        runId, root, target, browser: choice.name, base, startedAt: now, deadline: now + minutes * 60_000,
+        depth, budgetMinutes: minutes, minSeverity, focus, mode, signInVariables,
+    };
+    writeFileSync(path.join(dir, 'context.json'), JSON.stringify(ctx, null, 2) + '\n');
+    const specs = readSpecs(root);
+    const lines = [
+        `stillhunter | run ${runId}`,
+        `target: ${target} | browser: ${choice.name} | budget: ${minutes} min (depth ${depth}) | report: ${minSeverity === 'low' ? 'every severity' : `${minSeverity} severity and above`}`,
+        `focus: ${focus ?? 'none (the whole change)'}`,
+        mode === 'feature'
+            ? `mode: feature. There is no change against the base, so this hunt tests "${focus}" as it stands.`
+            : `mode: change. diff base: ${base.slice(0, 12)} (one file's whole diff: diff --file <file>)`,
+        `sign-in variables set in the environment (names only): ${signInVariables.length ? signInVariables.join(', ') : 'none'}`,
+        ...(config.reset ? [`reset: app data is reset before every probe and every re-run, back to what the app holds now, and the sign-in probe then signs in again; the reset took ${resetSeconds} s`] : []),
+        'changed files:',
+        ...(change.files.length ? change.files.map((l) => `  ${l}`) : ['  none']),
+        `specs: ${specs.length ? specs.map((s) => s.file).join(', ') : 'none'}`,
+        ...specs.flatMap((s) => ['', `=== spec: ${s.file} ===`, s.text.replace(/\n$/, '')]),
+        '',
+        `=== the change: git diff ${base.slice(0, 12)} of tracked files, then every untracked file ===`,
+        ...changeText(change),
+    ];
+    process.stdout.write(block(lines));
+}
+// Plain words for what went wrong before the probe ran, said first in what
+// the probe prints: without them a probe that ran signed out, or on data the
+// reset did not put back, looks like the app's fault.
+function beforeProblems(before) {
+    if (!before)
+        return [];
+    const s = before.signIn;
+    return [
+        ...(before.resetError ? [`Before this probe, the ${before.resetError}. App data may not be in its reset state.`] : []),
+        ...(s?.error && s.savedSession ? [`After the reset, the sign-in probe "${s.probe}" saved a session, then failed: ${brief(s.error)}. This probe ran with that session. Keep the sign-in probe to signing in: it runs before every probe.`] : []),
+        ...(s?.error && !s.savedSession ? [`After the reset, the sign-in probe "${s.probe}" failed: ${brief(s.error)}. This probe ran signed out.`] : []),
+    ];
+}
+async function probe(args) {
+    const { ctx, dir, root } = loadRun(args);
+    const name = flag(args, '--name');
+    if (!name || !PROBE_NAME.test(name))
+        fail('Give the probe a --name of lowercase letters, digits and dashes.');
+    if (remaining(ctx) === 0) {
+        process.stdout.write(block([`stillhunter | probe refused | run ${ctx.runId}`, 'The budget is spent. Call finish now, accounting for every failed check.']));
+        process.exit(2);
+    }
+    const script = readStdin();
+    if (!/export\s+default/.test(script))
+        fail('The probe script on stdin needs `export default async function (rr) { ... }`.');
+    const probeFile = path.join(probesDir(dir), `${name}.mjs`);
+    writeFileSync(probeFile, script);
+    const n = nextRunNumber(dir, name);
+    const run = (probeName, timeoutMs) => runProbe({
+        runDir: dir, probe: probeName, probeFile: path.join(probesDir(dir), `${probeName}.mjs`), target: ctx.target, browser: ctx.browser,
+        signInVariables: ctx.signInVariables, shotsDir: path.join(dir, 'shots'), timeoutMs,
+    });
+    // The same start the proof gives each re-run: app data reset, then signed
+    // in again by the run's sign-in probe, so what a probe finds is what finish
+    // can prove. The probe runs whatever went wrong, and says so.
+    const config = readConfig(root);
+    const signIn = signInProbe(dir);
+    const before = config.reset
+        ? await resetAndSignIn({
+            root, config, probe: name, signIn, sessionFile: sessionPath(dir),
+            runSignIn: async (s, timeoutMs) => {
+                const r = await run(s, timeoutMs);
+                writeFileSync(path.join(probesDir(dir), `${name}.${n}.signin.result.json`), JSON.stringify(r, null, 2) + '\n');
+                return r;
+            },
+        })
+        : null;
+    const problems = beforeProblems(before);
+    const result = await run(name);
+    result.run = n;
+    if (before)
+        result.before = before;
+    const json = JSON.stringify(result, null, 2) + '\n';
+    writeFileSync(numberedFile(dir, name, n), json);
+    writeFileSync(latestFile(dir, name), json);
+    const shown = {
+        ...(problems.length ? { warning: problems.join(' ') } : {}),
+        probe: name,
+        run: n,
+        remainingSeconds: remaining(ctx),
+        ...(before ? { before } : {}),
+        resultFile: numberedFile(dir, name, n),
+        checks: result.checks,
+        notes: result.notes,
+        error: result.error,
+        shots: result.shots,
+        blocked: result.blocked,
+        refused: result.refused ?? [],
+        requests: { total: result.requests.length, last: result.requests.slice(-30).map(({ method, url, status, fault, via }) => ({ method, url, status, fault, via })) },
+        pages: result.pages,
+        final: result.final,
+    };
+    process.stdout.write(`${JSON.stringify(shown, null, 1)}\n`);
+}
+// The change, read without a shell: the list of changed files, or one file's
+// whole diff against the base. The hunter's only Bash is this runner, so this
+// is how it reads what start had to leave out.
+function diff(args) {
+    const root = repoRoot();
+    const file = flag(args, '--file');
+    if (args.length !== 0 && !(args.length === 2 && file))
+        fail('Usage: diff [--file <path>]');
+    const base = baseRef(root);
+    if (file === null) {
+        const change = collectChange(root, base);
+        process.stdout.write(block([`stillhunter | diff | base ${base.slice(0, 12)} | changed and untracked files, with the size of each file's diff`, ...changeList(change)]));
+        return;
+    }
+    const r = fileDiff(root, base, file);
+    if (!r.ok)
+        fail(r.message);
+    process.stdout.write(block([`stillhunter | diff | base ${base.slice(0, 12)} | ${file}`, r.text.replace(/\n$/, '') || '(no difference)']));
+}
+const USAGE = 'Usage: stillhunter start [--depth quick|default|deep] [--minutes N] [--min-severity high|medium|low] [--focus "<text>"] | probe --run <id> --name <name> | finish --run <id> [--proof-seconds N] | diff [--file <path>] | keep --run <id> --probe <name> --check <check> [--as <name>] | replay [names] [--watch]';
+const [sub, ...rest] = process.argv.slice(2);
+if (sub === 'start')
+    await start(rest);
+else if (sub === 'probe')
+    await probe(rest);
+else if (sub === 'finish')
+    await finish(rest);
+else if (sub === 'diff')
+    diff(rest);
+else if (sub === 'keep')
+    keep(rest);
+else if (sub === 'replay')
+    await replay(rest);
+else
+    fail(USAGE);