probatio 0.3.0 → 0.5.0

package/dist/ledger/build.js

@@ -1,348 +0,0 @@
-import { randomBytes } from "node:crypto";
-import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import path from "node:path";
-import { scrub } from "../contract.js";
-import { filesInDiff, git, patchApplies, readHeader } from "../mutate/patch.js";
-/** Revert each fix commit's src diff. A clean apply becomes a forward patch. A conflict is reported, never fuzzy-applied. */
-export function buildLedger(options) {
-    const repo = realpathSync(path.resolve(options.repoDir));
-    const packageDir = realpathSync(path.resolve(options.packageDir));
-    const outDir = path.resolve(options.outDir);
-    const resolved = git(repo, ["rev-parse", "--verify", `${options.commit}^{commit}`]);
-    if (resolved.status !== 0)
-        return fail(outDir, options.commit, "that commit does not resolve");
-    const commit = resolved.stdout.trim();
-    const packageRel = path.relative(repo, packageDir);
-    if (packageRel.startsWith(".."))
-        return fail(outDir, commit, "package is outside the repository");
-    const srcPrefix = gitPath(packageRel, options.srcDir);
-    const testsPrefix = gitPath(packageRel, options.testsDir);
-    const collected = collect(repo, commit, srcPrefix, testsPrefix, options.maxCommits ?? null);
-    const candidates = collected.found;
-    mkdirSync(outDir, { recursive: true });
-    const ids = assignIds(candidates.map((item) => item.commit));
-    const disk = loadDisk(outDir);
-    let workDir = null;
-    try {
-        const entries = [];
-        const bodies = new Map();
-        for (const candidate of candidates) {
-            const id = ids.get(candidate.commit) ?? candidate.commit.slice(0, 7);
-            if (candidate.lines > options.maxLines) {
-                entries.push(entry(candidate, id, "skipped", [], `src diff is ${candidate.lines} lines, over the ${options.maxLines} line cap`));
-                continue;
-            }
-            const fixDiff = git(repo, ["diff", `${candidate.commit}^`, candidate.commit, "--", srcPrefix]).stdout;
-            const bugDiff = git(repo, ["diff", "-R", `${candidate.commit}^`, candidate.commit, "--", srcPrefix]).stdout;
-            if (!bugDiff.trim())
-                continue;
-            if (!workDir) {
-                const work = withWorktree(repo, commit);
-                if ("error" in work)
-                    return fail(outDir, commit, work.error);
-                workDir = work.dir;
-            }
-            // A hand-made mutant that names this fix wins over the history revert. The revert can be
-            // unviable (it removes an export a test imports), and the hand mutant keeps the API.
-            const named = disk.find((item) => item.kind === "hand" && item.fix === candidate.commit);
-            if (named && handmadeApplies(workDir, named.raw)) {
-                entries.push({ ...entry(candidate, id, "handmade", filesInDiff(named.raw)), patch: `${named.id}.patch` });
-                continue;
-            }
-            const applied = git(workDir, ["apply", "--check", "--whitespace=nowarn", "-"], bugDiff);
-            if (applied.status === 0) {
-                bodies.set(id, historyPatch(candidate.commit, bugDiff));
-                entries.push({ ...entry(candidate, id, "clean", filesInDiff(fixDiff)), patch: `${id}.patch` });
-                continue;
-            }
-            const where = failedAt(applied.stderr) ?? filesInDiff(fixDiff)[0] ?? { file: srcPrefix, line: 1 };
-            const owned = disk.find((item) => item.kind !== "history" && (item.fix === candidate.commit || item.id === id));
-            if (owned && handmadeApplies(workDir, owned.raw)) {
-                entries.push({ ...entry(candidate, id, "handmade", [where]), patch: `${owned.id}.patch` });
-                continue;
-            }
-            const reason = owned ? "hand-made patch does not apply" : "patch does not apply";
-            entries.push(entry(candidate, id, "hand", [where], reason));
-        }
-        writeOut(outDir, commit, packageRel || ".", entries, bodies, disk);
-        return report(outDir, repo, packageDir, commit, entries, collected);
-    }
-    finally {
-        if (workDir)
-            removeWorktree(repo, workDir);
-    }
-}
-function collect(repo, commit, srcPrefix, testsPrefix, maxCommits) {
-    const args = ["rev-list", "--no-merges"];
-    if (maxCommits !== null)
-        args.push("-n", String(maxCommits + 1));
-    args.push(commit);
-    const list = git(repo, args);
-    if (list.status !== 0)
-        return { found: [], scanned: 0, budgetHit: false };
-    const shas = list.stdout.split("\n").filter((sha) => sha.length > 0);
-    const budgetHit = maxCommits !== null && shas.length > maxCommits;
-    const limited = budgetHit ? shas.slice(0, maxCommits) : shas;
-    const found = [];
-    for (const sha of limited) {
-        if (git(repo, ["rev-parse", "--verify", `${sha}^`]).status !== 0)
-            continue;
-        const rows = numstat(repo, sha);
-        let lines = 0;
-        let src = false;
-        let tests = false;
-        for (const row of rows) {
-            if (under(row.file, srcPrefix)) {
-                src = true;
-                if (row.add !== "-" && row.del !== "-")
-                    lines += Number(row.add) + Number(row.del);
-            }
-            if (under(row.file, testsPrefix))
-                tests = true;
-        }
-        if (!src)
-            continue;
-        const meta = git(repo, ["log", "-1", `--format=%cI%n%s%n%b`, sha]);
-        if (meta.status !== 0)
-            continue;
-        const [date, subject, ...rest] = meta.stdout.replace(/\n$/, "").split("\n");
-        const body = rest.join("\n");
-        const hasTrailer = /^Fixes-bug:/m.test(body);
-        if (!hasTrailer && !(src && tests))
-            continue;
-        if (!hasTrailer && lines === 0)
-            continue;
-        // A version bump ships whatever was finished. Reverting it puts back several changes, not one bug.
-        if (!hasTrailer && bumpsVersion(repo, sha))
-            continue;
-        found.push({
-            commit: sha,
-            date: date ?? "",
-            subject: subject ?? "",
-            fixer: /^Made-by:[ \t]*\S+[ \t]*$/m.test(body) ? "agent" : "human",
-            lines,
-        });
-    }
-    return { found, scanned: limited.length, budgetHit };
-}
-const MANIFESTS = ["package.json", "Cargo.toml", "pyproject.toml", "setup.cfg"].map((name) => `:(glob)**/${name}`);
-/** True when the commit changes a project version line in a manifest. */
-function bumpsVersion(repo, sha) {
-    const diff = git(repo, ["show", "--format=", "-U0", sha, "--", ...MANIFESTS]);
-    if (diff.status !== 0)
-        return false;
-    return diff.stdout
-        .split("\n")
-        .filter((line) => /^[-+]/.test(line) && !line.startsWith("+++") && !line.startsWith("---"))
-        .some((line) => /"version"\s*:/.test(line) || /^[-+]\s*version\s*=/.test(line));
-}
-function entry(candidate, id, status, files, reason) {
-    return {
-        id,
-        commit: candidate.commit,
-        date: candidate.date,
-        subject: candidate.subject,
-        fixer: candidate.fixer,
-        status,
-        lines: candidate.lines,
-        files,
-        ...(reason ? { reason } : {}),
-    };
-}
-function writeOut(outDir, commit, packageRel, entries, bodies, disk) {
-    const cleanIds = new Set(entries.filter((item) => item.status === "clean").map((item) => item.id));
-    for (const item of disk) {
-        if (item.kind === "history" && !cleanIds.has(item.id))
-            unlinkSync(path.join(outDir, `${item.id}.patch`));
-    }
-    for (const item of entries) {
-        const body = bodies.get(item.id);
-        if (item.status !== "clean" || !body)
-            continue;
-        const dest = path.join(outDir, `${item.id}.patch`);
-        const temp = `${dest}.${process.pid}.tmp`;
-        writeFileSync(temp, body);
-        renameSync(temp, dest);
-    }
-    const recorded = {
-        commit,
-        package: packageRel.split(path.sep).join("/"),
-        entries,
-    };
-    const dest = path.join(outDir, "ledger.json");
-    const temp = `${dest}.${process.pid}.tmp`;
-    writeFileSync(temp, `${JSON.stringify(scrub(recorded), null, 2)}\n`);
-    renameSync(temp, dest);
-}
-function report(outDir, repo, packageDir, commit, entries, collected) {
-    const clean = entries.filter((item) => item.status === "clean").length;
-    const handEntries = entries.filter((item) => item.status === "hand").sort(byNewest);
-    const handmade = entries.filter((item) => item.status === "handmade").length;
-    const skipped = entries.filter((item) => item.status === "skipped").length;
-    const shown = handEntries.slice(0, 10).map((item) => ({
-        id: item.id,
-        commit: item.commit,
-        file: item.files[0]?.file ?? "",
-        line: item.files[0]?.line ?? 0,
-    }));
-    const parts = [`${clean} fixes reverse cleanly`];
-    if (handEntries.length > 0)
-        parts.push(`${handEntries.length} need a hand-made mutant`);
-    if (handmade > 0)
-        parts.push(`${handmade} hand-made mutants apply`);
-    if (skipped > 0)
-        parts.push(`${skipped} are over the line cap`);
-    if (collected.budgetHit) {
-        const commits = collected.scanned === 1 ? "1 commit" : `${collected.scanned} commits`;
-        parts.push(`Stopped after ${commits}. History was not fully scanned`);
-    }
-    const summary = `${parts.join(". ")}.`;
-    let next = "No fix commit to revert.";
-    if (handEntries.length > 0) {
-        const first = shown[0];
-        next = first.file
-            ? `First fix that needs a hand-made mutant is ${first.id} at ${first.file}:${first.line}.`
-            : `First fix that needs a hand-made mutant is ${first.id}.`;
-    }
-    else if (clean + handmade > 0)
-        next = "Run the ledger mutants.";
-    const nextCall = handEntries.length === 0 && clean + handmade > 0
-        ? {
-            argv: [
-                "mutate",
-                "run",
-                "--package",
-                packageDir,
-                "--repo",
-                repo,
-                "--commit",
-                commit,
-                "--patches",
-                outDir,
-                "--out",
-                path.join(outDir, "runs"),
-            ],
-        }
-        : null;
-    return {
-        ok: true,
-        summary,
-        next,
-        nextCall,
-        commit,
-        clean,
-        hand: handEntries.length,
-        handmade,
-        skipped,
-        handNeeded: shown,
-        rest: Math.max(handEntries.length - shown.length, 0),
-        full: path.join(outDir, "ledger.json"),
-        historyBudgetHit: collected.budgetHit,
-        commitsScanned: collected.scanned,
-    };
-}
-function fail(outDir, commit, error) {
-    return {
-        ok: false,
-        summary: error,
-        next: "Fix that and run the same command again.",
-        nextCall: null,
-        commit,
-        clean: 0,
-        hand: 0,
-        handmade: 0,
-        skipped: 0,
-        handNeeded: [],
-        rest: 0,
-        full: path.join(outDir, "ledger.json"),
-        historyBudgetHit: false,
-        commitsScanned: 0,
-    };
-}
-function historyPatch(commit, bugDiff) {
-    const body = bugDiff.endsWith("\n") ? bugDiff : `${bugDiff}\n`;
-    return `# probatio-mutant direction=forward meaning=apply-to-introduce-the-bug\n# probatio-ledger source=history fix=${commit}\n${body}`;
-}
-function handmadeApplies(worktree, raw) {
-    const named = readHeader(raw).direction;
-    if (named)
-        return patchApplies(worktree, raw, named);
-    return patchApplies(worktree, raw, "forward");
-}
-function failedAt(stderr) {
-    const match = stderr.match(/patch failed: ([^:\n]+):(\d+)/);
-    if (!match)
-        return null;
-    return { file: match[1], line: Number(match[2]) };
-}
-function numstat(repo, sha) {
-    const result = git(repo, ["diff-tree", "-r", "--numstat", "--no-commit-id", sha]);
-    if (result.status !== 0)
-        return [];
-    const rows = [];
-    for (const line of result.stdout.split("\n")) {
-        if (!line.trim())
-            continue;
-        const [add, del, file] = line.split("\t");
-        if (!file)
-            continue;
-        rows.push({ add, del, file });
-    }
-    return rows;
-}
-function loadDisk(outDir) {
-    if (!existsSync(outDir))
-        return [];
-    const found = [];
-    for (const name of readdirSync(outDir)) {
-        if (!name.endsWith(".patch"))
-            continue;
-        const raw = readFileSync(path.join(outDir, name), "utf8");
-        const header = raw.split("\n").filter((line) => line.startsWith("#")).join("\n");
-        const kind = /source=hand\b/.test(header) ? "hand" : /source=history\b/.test(header) ? "history" : "user";
-        const fix = header.match(/\bfix=([0-9a-f]{7,40})\b/)?.[1] ?? null;
-        found.push({ id: path.basename(name, ".patch"), raw, kind, fix });
-    }
-    return found;
-}
-function assignIds(shas) {
-    const used = new Set();
-    const ids = new Map();
-    for (const sha of [...shas].sort()) {
-        let size = Math.min(7, sha.length);
-        let id = sha.slice(0, size);
-        while (used.has(id) && size < sha.length) {
-            size += 1;
-            id = sha.slice(0, size);
-        }
-        used.add(id);
-        ids.set(sha, id);
-    }
-    return ids;
-}
-function withWorktree(repo, commit) {
-    const dir = path.join(tmpdir(), `probatio-ledger-${randomBytes(4).toString("hex")}`);
-    rmSync(dir, { recursive: true, force: true });
-    const added = git(repo, ["worktree", "add", "--detach", "--quiet", dir, commit]);
-    if (added.status !== 0)
-        return { error: (added.stderr || "worktree add failed").trim() };
-    return { dir };
-}
-function removeWorktree(repo, dir) {
-    git(repo, ["worktree", "remove", "--force", dir]);
-    rmSync(dir, { recursive: true, force: true });
-    git(repo, ["worktree", "prune"]);
-}
-function gitPath(packageRel, dir) {
-    const joined = packageRel ? path.join(packageRel, dir) : dir;
-    return joined.split(path.sep).join("/");
-}
-function under(file, prefix) {
-    return file === prefix || file.startsWith(`${prefix}/`);
-}
-function byNewest(left, right) {
-    if (left.date !== right.date)
-        return right.date < left.date ? -1 : 1;
-    return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
-}

package/dist/ledger/check.js

@@ -1,133 +0,0 @@
-import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import { SCHEMA_VERSION } from "../contract.js";
-import { goldenChanges } from "../golden/check.js";
-import { normalizeResult, runMutants } from "../mutate/run.js";
-/** The contract is the outcome. The reason follows from it, so a renamed test does not fail the check. */
-const REASON = {
-    killed: "A test the suite already had caught this ledger bug.",
-    survived: "The suite ran the line and did not catch this ledger bug.",
-    "no coverage": "No test runs the line this ledger bug changes.",
-    unviable: "This ledger bug does not build as it stands. A hand-made mutant that keeps the API would guard it.",
-    flaky: "A test failed once and passed on the rerun. That is not a catch.",
-    timeout: "The run passed the baseline clock. That is not a catch.",
-    error: "The ledger patch did not run. It may no longer apply: rebuild the ledger or refresh the hand-made mutant.",
-    missing: "No result was written for this ledger bug.",
-};
-export async function checkLedger(options) {
-    const fail = (summary, next = "Fix that and run the same command again.") => ({
-        schemaVersion: SCHEMA_VERSION,
-        ok: false,
-        command: "ledger.check",
-        summary,
-        next,
-        nextCall: null,
-    });
-    const ledgerFile = path.join(options.ledgerDir, "ledger.json");
-    if (!existsSync(ledgerFile))
-        return fail(`No ledger.json in ${options.ledgerDir}.`, "Run probatio ledger build first.");
-    const ledger = JSON.parse(readFileSync(ledgerFile, "utf8"));
-    const guarded = (ledger.entries ?? []).filter((item) => (item.status === "clean" || item.status === "handmade") && item.patch);
-    if (guarded.length === 0)
-        return fail("The ledger has no bug that applies.", "Rebuild the ledger, or write a hand-made mutant for a fix.");
-    const outDir = path.resolve(options.outDir);
-    rmSync(outDir, { recursive: true, force: true });
-    const patchDir = path.join(outDir, "patches");
-    mkdirSync(patchDir, { recursive: true });
-    for (const item of guarded) {
-        const source = path.join(options.ledgerDir, item.patch);
-        if (!existsSync(source))
-            return fail(`${item.patch} is listed in ledger.json and is not in ${options.ledgerDir}.`);
-        copyFileSync(source, path.join(patchDir, item.patch));
-    }
-    const report = await runMutants({
-        packageDir: options.packageDir,
-        repoDir: options.repoDir,
-        commit: options.commit,
-        patchDirs: [patchDir],
-        outDir: path.join(outDir, "run"),
-        direction: "forward",
-        workers: 1,
-        concurrency: options.concurrency,
-        maxMinutes: null,
-        maxMutants: null,
-        build: null,
-        testsDir: "tests",
-        unset: [],
-        suiteTimeoutMs: options.suiteTimeoutMs,
-        testTimeoutMs: options.testTimeoutMs,
-        confirm: true,
-        affected: false,
-        agent: null,
-        onProgress: options.onProgress,
-    });
-    if (!report.ok)
-        return fail(`The ledger run did not score: ${report.summary}`, report.next);
-    const allowed = goldenChanges(options.message);
-    const rows = [];
-    const regressed = [];
-    const changed = [];
-    const unrecorded = [];
-    const unauthorized = [];
-    for (const item of guarded) {
-        const patchId = path.basename(item.patch, ".patch");
-        const resultFile = path.join(outDir, "run", "results", `${patchId}.json`);
-        const result = existsSync(resultFile) ? normalizeResult(JSON.parse(readFileSync(resultFile, "utf8"))) : null;
-        const outcome = result?.outcome ?? "missing";
-        const next = outcome === "killed" ? `Killed by ${(result?.killedBy ?? []).join(", ")}.` : result?.next || result?.error || REASON[outcome] || outcome;
-        const actual = { ok: true, outcome, reason: REASON[outcome] ?? outcome, next };
-        const goldenFile = path.join(options.ledgerDir, `${item.id}.golden.json`);
-        const recorded = existsSync(goldenFile)
-            ? (JSON.parse(readFileSync(goldenFile, "utf8")).rows?.[item.id]?.outcome ?? null)
-            : null;
-        rows.push({ id: item.id, recorded, actual: outcome, next });
-        if (recorded === null)
-            unrecorded.push(item.id);
-        else if (recorded === "killed" && outcome !== "killed")
-            regressed.push(item.id);
-        else if (recorded !== outcome)
-            changed.push(item.id);
-        const differs = recorded !== outcome;
-        if (differs && recorded !== null && !allowed.has(item.id))
-            unauthorized.push(item.id);
-        if (options.update && (!differs || recorded === null || allowed.has(item.id))) {
-            writeFileSync(goldenFile, `${JSON.stringify({ rows: { [item.id]: actual } }, null, 2)}\n`);
-        }
-    }
-    const killed = rows.filter((row) => row.actual === "killed").length;
-    const written = options.update ? rows.filter((row) => !unauthorized.includes(row.id)).length : 0;
-    const ok = options.update ? unauthorized.length === 0 : regressed.length === 0 && changed.length === 0 && unrecorded.length === 0;
-    const parts = [`${killed} of ${rows.length} ledger bugs caught.`];
-    if (regressed.length > 0)
-        parts.push(`Regressed: ${regressed.join(", ")}.`);
-    if (changed.length > 0)
-        parts.push(`Changed: ${changed.join(", ")}.`);
-    if (unrecorded.length > 0)
-        parts.push(`Not recorded yet: ${unrecorded.join(", ")}.`);
-    if (options.update)
-        parts.push(`${written} golden row(s) written.`);
-    let next = "Every ledger bug has the outcome recorded for it.";
-    if (regressed.length > 0)
-        next = `A fixed bug is back in reach of the suite: ${regressed[0]} was caught and is not now. Find the test that stopped guarding it.`;
-    else if (unauthorized.length > 0 && options.update)
-        next = `Not written: ${unauthorized.join(", ")} changed outcome. Add Golden-Change: <id>: <why> to --message for each.`;
-    else if (options.update)
-        next = "Re-recorded. Commit the goldens with the reason for each changed outcome.";
-    else if (changed.length > 0 || unrecorded.length > 0)
-        next = "Review the changed outcomes, then re-record with --update and a Golden-Change line for each changed id.";
-    return {
-        schemaVersion: SCHEMA_VERSION,
-        ok,
-        command: "ledger.check",
-        summary: parts.join(" "),
-        next,
-        nextCall: null,
-        caught: killed,
-        total: rows.length,
-        regressed,
-        changed,
-        unrecorded,
-        rows,
-        run: { summary: report.summary, commit: report.commit, full: report.full },
-    };
-}