probatio 0.0.0-stage → 0.0.1

package/dist/matrix/report.js

@@ -0,0 +1,256 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { SCHEMA_VERSION } from "../contract.js";
+import { filesInDiff } from "../mutate/patch.js";
+const METRIC = /^(holdout|real \(train\)|synthetic-1|synthetic-2):\s+(\d+)\/(\d+)(?:\s+\([^)]*\))?(?:\s+missed\s+(.+))?$/;
+const PREFIX = /^(?:ho|r|h|v)-/;
+/** Names of tests that killed nothing. This never deletes a file. */
+export function advisePrune(kills) {
+    const advice = [...kills.entries()]
+        .filter(([, killed]) => killed === 0)
+        .map(([name]) => name)
+        .sort();
+    return { mode: "advisory", deletedTests: 0, advice };
+}
+export function parseUnionTsv(text) {
+    const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0);
+    if (lines.length === 0)
+        return [];
+    const columns = lines[0].split("\t").map(parseHeader);
+    return lines.slice(1).map((line) => {
+        const cells = line.split("\t");
+        const row = emptyRow("union");
+        for (let i = 0; i < columns.length; i++) {
+            const column = columns[i];
+            const cell = (cells[i] ?? "").trim();
+            if (column.kind === "set") {
+                row.set = cell;
+                row.name = cell;
+            }
+            else if (column.kind === "lines") {
+                const lines = Number(cell);
+                if (!Number.isInteger(lines))
+                    throw new Error(`lines is not an integer: ${cell}`);
+                row.lines = lines;
+            }
+            else if (column.total === null) {
+                throw new Error(`column ${column.kind} has no total`);
+            }
+            else {
+                assign(row, column.kind, parseCaught(cell, column.total));
+            }
+        }
+        return row;
+    });
+}
+export function parseDirectScores(text) {
+    const rows = [];
+    let current = null;
+    for (const raw of text.split(/\r?\n/)) {
+        const line = raw.trim();
+        if (!line)
+            continue;
+        const metric = line.match(METRIC);
+        if (!metric) {
+            current = emptyRow("direct");
+            current.name = line;
+            current.set = line.split("(")[0].trim().split(/\s+/).join("");
+            rows.push(current);
+            continue;
+        }
+        if (!current)
+            throw new Error(`metric before a suite: ${line}`);
+        assign(current, metricKind(metric[1]), { caught: Number(metric[2]), total: Number(metric[3]) });
+        if (metric[4])
+            current.missed.push(...metric[4].trim().split(/\s+/).filter(Boolean));
+    }
+    return rows;
+}
+export function reportMatrix(input) {
+    const rows = [...parseUnionTsv(input.scores), ...parseDirectScores(input.direct)];
+    if (rows.length === 0) {
+        return {
+            schemaVersion: SCHEMA_VERSION,
+            ok: false,
+            command: "matrix.report",
+            summary: "The score files have no suite rows.",
+            next: "Pass --scores and --direct that contain the recorded table.",
+            nextCall: null,
+        };
+    }
+    const reference = rows.find((row) => row.source === "union" && row.holdout) ?? null;
+    const gapRow = weakestHoldout(rows.filter((row) => row.source === "direct"));
+    const gaps = resolveGaps(gapRow?.missed ?? [], input.patchDirs);
+    const shown = gaps.slice(0, 10);
+    const collapse = {
+        batch2: rows
+            .filter((row) => row.source === "union" && lowerRate(row.synthetic2, row.synthetic1))
+            .map((row) => ({ set: row.set, synthetic1: row.synthetic1, synthetic2: row.synthetic2 })),
+        holdoutBelowReference: reference
+            ? rows
+                .filter((row) => row !== reference && lowerRate(row.holdout, reference.holdout))
+                .map((row) => ({ set: row.set, holdout: row.holdout, reference: reference.holdout }))
+            : [],
+    };
+    const pruning = advisePrune(input.kills);
+    const fullPath = path.join(input.outDir, "matrix.json");
+    const complete = {
+        schemaVersion: SCHEMA_VERSION,
+        ok: true,
+        command: "matrix.report",
+        summary: summarize(gapRow, collapse.batch2, gaps.length, pruning.deletedTests),
+        next: nextSentence(gaps),
+        nextCall: null,
+        rows,
+        collapse,
+        gaps,
+        rest: 0,
+        pruning,
+        full: fullPath,
+    };
+    mkdirSync(input.outDir, { recursive: true });
+    writeFileSync(fullPath, `${JSON.stringify(complete, null, 2)}\n`);
+    return {
+        ...complete,
+        gaps: shown,
+        rest: gaps.length - shown.length,
+    };
+}
+export function readKills(file) {
+    const parsed = JSON.parse(readFileSync(file, "utf8"));
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+        throw new Error("--kills must be a JSON object of test name to kill count");
+    const kills = new Map();
+    for (const [name, value] of Object.entries(parsed)) {
+        if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
+            throw new Error(`kill count for ${name} must be a non-negative integer`);
+        }
+        kills.set(name, value);
+    }
+    return kills;
+}
+function summarize(gapRow, batch2, gapCount, deletedTests) {
+    const parts = [];
+    if (gapRow?.holdout) {
+        const train = gapRow.real ? ` Train real is ${gapRow.real.caught}/${gapRow.real.total}.` : "";
+        parts.push(`${gapRow.set} holdout is ${gapRow.holdout.caught}/${gapRow.holdout.total}.${train}`);
+    }
+    if (batch2.length > 0) {
+        const bits = batch2.map((item) => {
+            const left = item.synthetic1;
+            const right = item.synthetic2;
+            return `${item.set} synthetic-1 ${left?.caught}/${left?.total} against synthetic-2 ${right?.caught}/${right?.total}`;
+        });
+        parts.push(`Cover collapse on batch 2: ${bits.join("; ")}.`);
+    }
+    parts.push(`${gapCount} gaps. Pruning is advisory and deleted ${deletedTests} tests.`);
+    return parts.join(" ");
+}
+function nextSentence(gaps) {
+    const first = gaps.find((gap) => gap.file !== null && gap.line !== null) ?? gaps[0];
+    if (!first)
+        return "No gaps.";
+    if (first.file === null || first.line === null)
+        return `${first.id} has no patch, so there is no file:line.`;
+    return `${first.id} at ${first.file}:${first.line}`;
+}
+function weakestHoldout(rows) {
+    let best = null;
+    let bestRate = Number.POSITIVE_INFINITY;
+    for (const row of rows) {
+        const value = rate(row.holdout);
+        if (value === null || value >= bestRate)
+            continue;
+        best = row;
+        bestRate = value;
+    }
+    return best;
+}
+function lowerRate(left, right) {
+    const a = rate(left);
+    const b = rate(right);
+    return a !== null && b !== null && a < b;
+}
+function rate(count) {
+    if (!count || count.total === 0)
+        return null;
+    return count.caught / count.total;
+}
+function resolveGaps(ids, patchDirs) {
+    const unique = [];
+    const seen = new Set();
+    for (const id of ids) {
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        unique.push(id);
+    }
+    unique.sort();
+    return unique.map((id) => locate(id, patchDirs));
+}
+function locate(id, patchDirs) {
+    const bare = id.replace(PREFIX, "");
+    const names = [...new Set([`${id}.patch`, `r-${bare}.patch`, `h-${bare}.patch`, `${bare}.patch`, `ho-${bare}.patch`])];
+    for (const dir of patchDirs) {
+        for (const name of names) {
+            const candidate = path.join(dir, name);
+            let raw;
+            try {
+                raw = readFileSync(candidate, "utf8");
+            }
+            catch {
+                continue;
+            }
+            const locations = filesInDiff(raw);
+            const first = locations[0];
+            return { id, file: first?.file ?? null, line: first?.line ?? null, locations };
+        }
+    }
+    return { id, file: null, line: null, locations: [] };
+}
+function parseHeader(cell) {
+    const match = cell.trim().match(/^(.*?)\s*\((\d+)\)$/);
+    if (!match)
+        return { kind: cell.trim(), total: null };
+    return { kind: metricKind(match[1].trim()), total: Number(match[2]) };
+}
+function metricKind(name) {
+    if (name === "synthetic-1")
+        return "synthetic1";
+    if (name === "synthetic-2")
+        return "synthetic2";
+    if (name === "real (train)" || name === "real")
+        return "real";
+    return name;
+}
+function parseCaught(cell, total) {
+    const match = cell.match(/^(\d+)/);
+    if (!match)
+        throw new Error(`cannot read a count from "${cell}"`);
+    return { caught: Number(match[1]), total };
+}
+function assign(row, kind, count) {
+    if (kind === "holdout")
+        row.holdout = count;
+    else if (kind === "real")
+        row.real = count;
+    else if (kind === "synthetic1")
+        row.synthetic1 = count;
+    else if (kind === "synthetic2")
+        row.synthetic2 = count;
+    else
+        throw new Error(`unknown score column ${kind}`);
+}
+function emptyRow(source) {
+    return {
+        set: "",
+        name: "",
+        source,
+        lines: null,
+        holdout: null,
+        real: null,
+        synthetic1: null,
+        synthetic2: null,
+        missed: [],
+    };
+}

package/dist/mcp.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const here = path.dirname(fileURLToPath(import.meta.url));
+let buffer = Buffer.alloc(0);
+process.stdin.on("data", (chunk) => {
+    buffer = Buffer.concat([buffer, chunk]);
+    drain();
+});
+process.stdin.on("end", () => process.exit(0));
+function drain() {
+    for (;;) {
+        const headerEnd = buffer.indexOf("\r\n\r\n");
+        if (headerEnd === -1)
+            return;
+        const header = buffer.subarray(0, headerEnd).toString("utf8");
+        const match = header.match(/Content-Length:\s*(\d+)/i);
+        const start = headerEnd + 4;
+        if (!match) {
+            buffer = buffer.subarray(start);
+            continue;
+        }
+        const length = Number(match[1]);
+        if (buffer.length < start + length)
+            return;
+        const body = buffer.subarray(start, start + length).toString("utf8");
+        buffer = buffer.subarray(start + length);
+        let message;
+        try {
+            message = JSON.parse(body);
+        }
+        catch {
+            continue;
+        }
+        handle(message);
+    }
+}
+function handle(message) {
+    if (!message.method)
+        return;
+    if (message.id === undefined || message.id === null)
+        return;
+    if (message.method === "initialize") {
+        send({
+            jsonrpc: "2.0",
+            id: message.id,
+            result: {
+                protocolVersion: "2024-11-05",
+                capabilities: { tools: {} },
+                serverInfo: { name: "probatio", version: "0.0.1" },
+            },
+        });
+        return;
+    }
+    if (message.method === "tools/list") {
+        send({
+            jsonrpc: "2.0",
+            id: message.id,
+            result: {
+                tools: [
+                    {
+                        name: "probatio",
+                        description: "Run one probatio command. argv is the CLI words and flags.",
+                        inputSchema: {
+                            type: "object",
+                            properties: { argv: { type: "array", items: { type: "string" } } },
+                            required: ["argv"],
+                        },
+                    },
+                ],
+            },
+        });
+        return;
+    }
+    if (message.method === "tools/call") {
+        const argv = message.params?.arguments?.argv;
+        if (!Array.isArray(argv) || argv.some((item) => typeof item !== "string")) {
+            send({ jsonrpc: "2.0", id: message.id, error: { code: -32602, message: "argv must be an array of strings" } });
+            return;
+        }
+        const cli = cliCommand();
+        const child = spawnSync(cli.bin, [...cli.args, ...argv], { encoding: "utf8" });
+        const text = child.stdout ?? "";
+        let envelope = null;
+        try {
+            envelope = JSON.parse(text);
+        }
+        catch {
+            envelope = null;
+        }
+        if (!envelope || typeof envelope !== "object") {
+            send({ jsonrpc: "2.0", id: message.id, error: { code: -32000, message: "the command did not return one JSON object" } });
+            return;
+        }
+        send({
+            jsonrpc: "2.0",
+            id: message.id,
+            result: {
+                content: [{ type: "text", text: text.trim() }],
+                structuredContent: envelope,
+                isError: envelope.ok !== true,
+            },
+        });
+    }
+}
+function cliCommand() {
+    const compiled = path.join(here, "cli.js");
+    if (existsSync(compiled))
+        return { bin: process.execPath, args: [compiled] };
+    const tsx = path.resolve(here, "..", "node_modules", ".bin", "tsx");
+    return { bin: tsx, args: [path.join(here, "cli.ts")] };
+}
+function send(message) {
+    const payload = Buffer.from(JSON.stringify(message), "utf8");
+    process.stdout.write(`Content-Length: ${payload.length}\r\n\r\n`);
+    process.stdout.write(payload);
+}

package/dist/memory/state.js

@@ -0,0 +1,222 @@
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { SCHEMA_VERSION } from "../contract.js";
+import { makeRng, shuffle } from "../mutate/ids.js";
+export const YARDSTICK = "these tests fit the yardstick, not the code.";
+const FINDING_STATUS = new Set(["open", "fixed", "equivalent", "wont-fix"]);
+export function lineHash(line) {
+    return createHash("sha256").update(line).digest("hex").slice(0, 16);
+}
+export function readSourceLine(file, line) {
+    if (!existsSync(file))
+        return null;
+    const lines = readFileSync(file, "utf8").split(/\r?\n/);
+    if (line < 1 || line > lines.length)
+        return null;
+    return lines[line - 1];
+}
+/** `Guard-Change: <guard>: <why>`. A guard such as `connect.test.ts:118` keeps its colon. */
+export function guardChanges(message) {
+    const named = new Map();
+    for (const match of message.matchAll(/^Guard-Change:\s*(.+?):\s+(.+)$/gm))
+        named.set(match[1], match[2]);
+    return named;
+}
+export function emptyState() {
+    return { schemaVersion: SCHEMA_VERSION, gaps: [], decisions: [] };
+}
+export function loadState(dir) {
+    const file = path.join(dir, "state.json");
+    if (!existsSync(file))
+        return emptyState();
+    const parsed = JSON.parse(readFileSync(file, "utf8"));
+    return { schemaVersion: SCHEMA_VERSION, gaps: parsed.gaps ?? [], decisions: parsed.decisions ?? [] };
+}
+export function saveState(dir, state) {
+    mkdirSync(dir, { recursive: true });
+    const ordered = {
+        schemaVersion: SCHEMA_VERSION,
+        gaps: [...state.gaps].sort((a, b) => a.id.localeCompare(b.id)),
+        decisions: [...state.decisions].sort((a, b) => a.id.localeCompare(b.id)),
+    };
+    writeFileSync(path.join(dir, "state.json"), `${JSON.stringify(ordered, null, 2)}\n`);
+}
+export function loadSealed(dir) {
+    const file = path.join(dir, "sealed.json");
+    if (!existsSync(file))
+        return [];
+    const parsed = JSON.parse(readFileSync(file, "utf8"));
+    return [...new Set(parsed.ids ?? [])].sort();
+}
+export function sealIds(dir, ids) {
+    const merged = [...new Set([...loadSealed(dir), ...ids])].sort();
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(path.join(dir, "sealed.json"), `${JSON.stringify({ ids: merged }, null, 2)}\n`);
+}
+/** A stable share of the ids. The same seed returns the same slice. */
+export function chooseSealed(ids, fraction, seed) {
+    if (ids.length === 0 || fraction <= 0)
+        return [];
+    const count = Math.min(ids.length, Math.max(1, Math.round(ids.length * fraction)));
+    return shuffle([...ids], makeRng(seed)).slice(0, count).sort();
+}
+export function parseGuard(guard) {
+    const match = guard.match(/^(.*):(\d+)$/);
+    if (!match)
+        throw new Error("guard must be file:line");
+    return { file: match[1], line: Number(match[2]) };
+}
+export function fixGap(state, input) {
+    const located = parseGuard(input.guard);
+    const absolute = path.resolve(input.root, located.file);
+    const text = readSourceLine(absolute, located.line);
+    if (text === null)
+        throw new Error("guard line does not exist");
+    const hash = lineHash(text);
+    const gaps = state.gaps.filter((gap) => gap.id !== input.id);
+    gaps.push({
+        id: input.id,
+        status: "fixed",
+        file: input.file,
+        line: input.line,
+        guard: input.guard,
+        fixedIn: input.commit,
+        guardFile: located.file,
+        guardLineNo: located.line,
+        guardLine: text,
+        guardHash: hash,
+    });
+    const decisions = state.decisions.filter((item) => item.id !== input.id);
+    decisions.push({ id: input.id, file: located.file, line: located.line, hash, status: "fixed", reason: `guarded by ${input.guard}` });
+    return { ...state, gaps, decisions };
+}
+export function revertGap(state, id) {
+    const gap = state.gaps.find((item) => item.id === id);
+    if (!gap || !gap.fixedIn || !gap.guard)
+        throw new Error("that id is not a fixed gap");
+    const text = `fixed in ${gap.fixedIn}, guarded by ${gap.guard}; the guard broke`;
+    const gaps = state.gaps.map((item) => (item.id === id ? { ...item, status: "regression" } : item));
+    return { state: { ...state, gaps }, text };
+}
+export function guardAllows(state, guard, message, root) {
+    const owners = state.gaps.filter((gap) => gap.guard === guard && (gap.status === "fixed" || gap.status === "regression"));
+    if (owners.length === 0)
+        return { ok: true, summary: `${guard} guards nothing.` };
+    const owner = owners[0];
+    if (guardChanges(message).has(guard))
+        return { ok: true, summary: `${guard} changed with a Guard-Change trailer.` };
+    const file = owner.guardFile ? path.resolve(root, owner.guardFile) : "";
+    const line = owner.guardLineNo && file ? readSourceLine(file, owner.guardLineNo) : null;
+    const weakened = line === null || (owner.guardLine !== null && line !== owner.guardLine);
+    if (!weakened)
+        return { ok: true, summary: `${guard} is intact.` };
+    return { ok: false, summary: `Refusing to change ${guard}. It is the guard of ${owner.id}.` };
+}
+export function appendFinding(dir, finding) {
+    if (!FINDING_STATUS.has(finding.status))
+        throw new Error("status must be open, fixed, equivalent, or wont-fix");
+    if (!finding.reason || !finding.reason.trim())
+        throw new Error("a finding needs a reason");
+    mkdirSync(dir, { recursive: true });
+    appendFileSync(path.join(dir, "findings.jsonl"), `${JSON.stringify(finding)}\n`);
+}
+export function loadFindings(dir) {
+    const file = path.join(dir, "findings.jsonl");
+    if (!existsSync(file))
+        return [];
+    return readFileSync(file, "utf8")
+        .split(/\r?\n/)
+        .filter((line) => line.trim().length > 0)
+        .map((line) => JSON.parse(line));
+}
+export function yardstickSentence(score) {
+    if (!score)
+        return null;
+    if (score.visible.after > score.visible.before && score.sealed.after <= score.sealed.before)
+        return YARDSTICK;
+    return null;
+}
+export function statusView(dir, root) {
+    const state = loadState(dir);
+    const sealed = new Set(loadSealed(dir));
+    const hidden = (id) => sealed.has(id);
+    const open = state.gaps.filter((gap) => gap.status === "open" && !hidden(gap.id));
+    const shown = open.slice(0, 10);
+    const fixes = state.gaps
+        .filter((gap) => gap.status === "fixed" && !hidden(gap.id))
+        .map((gap) => ({ id: gap.id, fixedIn: gap.fixedIn, guard: gap.guard }));
+    const guards = [...new Set(state.gaps.filter((gap) => gap.guard && !hidden(gap.id)).map((gap) => gap.guard))].sort();
+    const decisions = state.decisions
+        .filter((item) => !hidden(item.id))
+        .map((item) => {
+        const current = readSourceLine(path.resolve(root, item.file), item.line);
+        const review = current !== null && lineHash(current) === item.hash ? "current" : "stale";
+        return { ...item, review };
+    });
+    const regressions = state.gaps
+        .filter((gap) => gap.status === "regression" && !hidden(gap.id) && gap.fixedIn && gap.guard)
+        .map((gap) => ({ id: gap.id, text: `fixed in ${gap.fixedIn}, guarded by ${gap.guard}; the guard broke` }));
+    const findings = loadFindings(dir).filter((item) => !hidden(item.id)).slice(-10);
+    return {
+        gaps: shown,
+        rest: Math.max(open.length - shown.length, 0),
+        recentFixes: fixes,
+        protectedGuards: guards,
+        decisions,
+        regressions,
+        findings,
+        sealedCount: sealed.size,
+    };
+}
+export function renderPage(view, summary) {
+    const lines = ["# PROBATIO", "", summary, "", "## Gaps"];
+    if (view.gaps.length === 0)
+        lines.push("None.");
+    for (const gap of view.gaps)
+        lines.push(`- ${gap.id} at ${gap.file}:${gap.line}`);
+    if (view.rest > 0)
+        lines.push(`- ${view.rest} more`);
+    lines.push("", "## Recent fixes");
+    if (view.recentFixes.length === 0)
+        lines.push("None.");
+    for (const fix of view.recentFixes)
+        lines.push(`- ${fix.id} fixed in ${fix.fixedIn}, guard ${fix.guard}`);
+    lines.push("", "## Protected guards");
+    if (view.protectedGuards.length === 0)
+        lines.push("None.");
+    for (const guard of view.protectedGuards)
+        lines.push(`- ${guard}`);
+    lines.push("", "## Decisions");
+    if (view.decisions.length === 0)
+        lines.push("None.");
+    for (const decision of view.decisions)
+        lines.push(`- ${decision.id} ${decision.status} ${decision.review} ${decision.reason}`);
+    lines.push("", "## Regressions");
+    if (view.regressions.length === 0)
+        lines.push("None.");
+    for (const item of view.regressions)
+        lines.push(`- ${item.id}: ${item.text}`);
+    lines.push("");
+    return lines.join("\n");
+}
+export function statusEnvelope(dir, root, score, agent) {
+    const view = statusView(dir, root);
+    const yard = yardstickSentence(score);
+    const summary = yard ?? `${view.gaps.length} open gaps, ${view.recentFixes.length} fixes, ${view.decisions.length} decisions.`;
+    const first = view.regressions[0]?.text ?? (view.gaps[0] ? `${view.gaps[0].id} at ${view.gaps[0].file}:${view.gaps[0].line}` : "Nothing waiting.");
+    const page = renderPage(view, summary);
+    return {
+        page,
+        envelope: {
+            schemaVersion: SCHEMA_VERSION,
+            ok: true,
+            command: "status",
+            summary,
+            next: first,
+            nextCall: null,
+            agent: agent ?? "unknown",
+            ...view,
+        },
+    };
+}