probatio 0.1.0 → 0.1.3

package/dist/mutate/suites.js

@@ -386,8 +386,8 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
     if (spec.kind === "swift")
         return tag(runSwift(pkg, timeoutMs, env), spec.command, wantsNames, selection);
     if (spec.kind === "maven") {
-        const tests = selection.map(surefireTest).filter((name) => name.length > 0);
-        const args = ["-B", "test", ...(tests.length > 0 ? [`-Dtest=${tests.join(",")}`] : [])];
+        const tests = surefireArg(uniqueSurefire(selection));
+        const args = ["-B", "test", ...(tests.length > 0 ? [`-Dtest=${tests}`] : [])];
         const whole = wantsNames && tests.length === 0;
         // Surefire leaves the previous run's XML in place. A confirm that did not
         // re-execute a class would otherwise read that class as failed again.
@@ -400,8 +400,11 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
     }
     if (spec.kind === "dotnet-exe")
         return tag(runDotnetExe(pkg, spec, files, timeoutMs, env), spec.command, wantsNames, selection);
-    if (spec.kind === "mocha")
-        return tag(runMocha(pkg, files, timeoutMs, env), spec.command, wantsNames, selection);
+    if (spec.kind === "mocha") {
+        const grep = selection.length > 0 ? selection.map(escapeRegExp).join("|") : "";
+        const command = grep ? `${spec.command} --grep ${JSON.stringify(grep)}` : spec.command;
+        return tag(runMocha(pkg, files, timeoutMs, env, selection), command, false, selection);
+    }
     // A tty is only for a suite that runs `docker run -it` (tini). Everything else is bash.
     if (spec.kind === "script") {
         const launched = scriptInvocation(pkg);
@@ -465,12 +468,47 @@ function goRunFilter(names, pattern) {
         return null;
     return `^(${names.map(escapeRegExp).join("|")})$`;
 }
+function uniqueSurefire(names) {
+    const out = [];
+    const seen = new Set();
+    for (const name of names) {
+        const selector = surefireTest(name);
+        if (!selector || seen.has(selector))
+            continue;
+        seen.add(selector);
+        out.push(selector);
+    }
+    return out;
+}
+/** Surefire 2.12 splits `-Dtest` on commas and then reads `#method+method`. `Class#a,Class#b` runs only `a`. */
+export function surefireArg(tests) {
+    const groups = new Map();
+    for (const test of tests) {
+        const hash = test.indexOf("#");
+        const klass = hash === -1 ? test : test.slice(0, hash);
+        const method = hash === -1 ? "" : test.slice(hash + 1);
+        const methods = groups.get(klass);
+        if (methods) {
+            if (method)
+                methods.push(method);
+        }
+        else {
+            groups.set(klass, method ? [method] : []);
+        }
+    }
+    return [...groups.entries()]
+        .map(([klass, methods]) => (methods.length > 0 ? `${klass}#${methods.join("+")}` : klass))
+        .join(",");
+}
 function surefireTest(name) {
     const leaf = name.split("::").pop() ?? name;
     const dot = leaf.lastIndexOf(".");
     const body = dot <= 0 ? leaf : `${leaf.slice(0, dot)}#${leaf.slice(dot + 1)}`;
-    // testShut[0] collects nothing. method(Parser)[1] keeps the parameter types.
-    return body.replace(/\[[^\]]*\]$/, "");
+    // testShut[0] matches nothing. A bare testShut matches on Surefire 3.6 and runs
+    // zero tests on 3.5. testShut* matches every invocation on both. method(Parser)[1] keeps the types.
+    const invocation = /\[[^\]]*\]$/.test(body);
+    const method = body.replace(/\[[^\]]*\]$/, "");
+    return invocation ? `${method}*` : method;
 }
 function escapeRegExp(value) {
     return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -481,18 +519,46 @@ function runShell(pkg, command, timeoutMs, env) {
     const child = /\b(java|javac|mvn)\b/.test(command) ? javaEnv(env) : env;
     return spawnCollected("sh", ["-c", command], pkg, { ...child, PYTHONDONTWRITEBYTECODE: "1" }, timeoutMs);
 }
+/**
+ * Node's `--test-timeout` caps the file, not only a test that left the option unset.
+ * A test that sets a longer `timeout` is cancelled, and the failure name is the file path.
+ * The flag stays at least that long, and it stays inside the suite cap.
+ */
+export function nodeTestTimeoutMs(pkg, files, testTimeoutMs, suiteTimeoutMs) {
+    let declared = 0;
+    for (const rel of files) {
+        let text = "";
+        try {
+            text = readFileSync(path.join(pkg, rel), "utf8");
+        }
+        catch {
+            continue;
+        }
+        for (const match of text.matchAll(/timeout\s*:\s*(\d[\d_]*)/g)) {
+            const value = Number(match[1].replaceAll("_", ""));
+            if (Number.isFinite(value) && value > declared)
+                declared = value;
+        }
+    }
+    const floor = Number.isFinite(testTimeoutMs) && testTimeoutMs > 0 ? testTimeoutMs : 0;
+    let chosen = Math.max(floor, declared);
+    if (Number.isFinite(suiteTimeoutMs) && suiteTimeoutMs > 0 && chosen > suiteTimeoutMs)
+        chosen = suiteTimeoutMs;
+    return chosen > 0 ? chosen : floor;
+}
 async function runNode(pkg, files, pattern, timeoutMs, env, hooks) {
     const tsx = path.join(pkg, "node_modules", ".bin", "tsx");
     const bin = existsSync(tsx) ? tsx : process.execPath;
     const prefix = existsSync(tsx) ? [] : ["--experimental-strip-types"];
     const hook = env.PROBATIO_COVERAGE_MAP ? fileURLToPath(new URL("./node-coverage.mjs", import.meta.url)) : null;
+    const fileTimeout = nodeTestTimeoutMs(pkg, files, hooks.testTimeoutMs, timeoutMs);
     const result = await spawnCollected(bin, [
         ...prefix,
         ...(hook ? ["--import", hook] : []),
         "--test",
         `--test-reporter=${hooks.reporterPath}`,
         `--test-concurrency=${hooks.concurrency}`,
-        `--test-timeout=${hooks.testTimeoutMs}`,
+        `--test-timeout=${fileTimeout}`,
         ...(pattern ? [`--test-name-pattern=${pattern}`] : []),
         ...files,
     ], pkg, env, timeoutMs);
@@ -545,7 +611,7 @@ function runPython(pkg, files, pattern, timeoutMs, env, script, nodeIds) {
     const targets = nodeIds && nodeIds.length > 0 ? nodeIds : files;
     return spawnCollected("python3", [reporter, ...targets, ...(pattern ? ["--pattern", pattern] : [])], pkg, next, timeoutMs);
 }
-async function runMocha(pkg, files, timeoutMs, env) {
+async function runMocha(pkg, files, timeoutMs, env, names = null) {
     const mocha = ["mocha.js", "mocha"]
         .map((name) => path.join(pkg, "node_modules", "mocha", "bin", name))
         .find((file) => existsSync(file));
@@ -556,6 +622,10 @@ async function runMocha(pkg, files, timeoutMs, env) {
         return prepared;
     const config = readMochaConfig(pkg);
     const args = [mocha, ...buildMochaArgs(pkg, files)];
+    if (names && names.length > 0)
+        args.push("--grep", names.map(escapeRegExp).join("|"));
+    if (env.PROBATIO_COVERAGE_MAP)
+        args.push("--require", fileURLToPath(new URL("./mocha-coverage.cjs", import.meta.url)));
     if (config && config.require.includes("esm")) {
         // The `esm` loader throws on current Node before any test runs. Drop that one require and keep the rest of the project's mocha config.
         const requires = config.require.filter((name) => name !== "esm");
@@ -729,7 +799,8 @@ function scriptInvocation(pkg) {
     if (!suiteNeedsTty(pkg))
         return { bin: "bash", args: ["run_tests.sh"], command: "bash run_tests.sh" };
     if (gnuScript()) {
-        const args = ["-q", "-c", "bash run_tests.sh", "/dev/null"];
+        // -e returns the child status. Without it, util-linux script exits 0 and a failing suite looks green.
+        const args = ["-q", "-e", "-c", "bash run_tests.sh", "/dev/null"];
         return { bin: "script", args, command: shown("script", args) };
     }
     const args = ["-q", "/dev/null", "bash", "run_tests.sh"];
@@ -1368,6 +1439,11 @@ function mavenReport(pkg, text, code) {
     }
     return parseSurefire(pkg) ?? parseMavenText(text);
 }
+/** An attribute on the opening tag. `classname` must not be read as `name`. */
+export function surefireAttr(fragment, key) {
+    const open = fragment.slice(0, fragment.indexOf(">") === -1 ? fragment.length : fragment.indexOf(">"));
+    return new RegExp(`(?:^|\\s)${key}="([^"]*)"`).exec(open)?.[1] ?? "";
+}
 function parseSurefire(pkg) {
     const dir = path.join(pkg, "target", "surefire-reports");
     if (!existsSync(dir))
@@ -1379,11 +1455,11 @@ function parseSurefire(pkg) {
             continue;
         const xml = readFileSync(path.join(dir, name), "utf8");
         for (const part of xml.split("<testcase ").slice(1)) {
-            const test = /name="([^"]+)"/.exec(part)?.[1];
-            const klass = /classname="([^"]+)"/.exec(part)?.[1] ?? "";
+            const test = surefireAttr(part, "name");
+            const klass = surefireAttr(part, "classname");
             if (!test)
                 continue;
-            const id = `${klass}.${test}`;
+            const id = klass ? `${klass}.${test}` : test;
             names.push(id);
             if (part.includes("<failure") || part.includes("<error"))
                 failed.push({ name: id, file: klass, line: 0 });

package/dist/mutate/tally.js

@@ -105,7 +105,7 @@ export function tallyRun(outDir) {
         command: "mutate.tally",
         summary,
         next,
-        nextCall: keep.length > 0 ? { argv: ["mutate", "run", ...keep.flatMap((id) => ["--only-test", id])] } : null,
+        nextCall: resumeArgv(outDir, keep),
         keep,
         drop,
         gaps,
@@ -117,6 +117,43 @@ export function tallyRun(outDir) {
     writeFileSync(fullPath, `${JSON.stringify(complete, null, 2)}\n`);
     return { ...complete, gaps: shown, rest: gaps.length - shown.length };
 }
+function resumeArgv(outDir, keep) {
+    if (keep.length === 0)
+        return null;
+    let stored = {};
+    try {
+        stored = JSON.parse(readFileSync(path.join(outDir, "run.json"), "utf8"));
+    }
+    catch {
+        stored = {};
+    }
+    const argv = ["mutate", "run"];
+    if (stored.package)
+        argv.push("--package", stored.package);
+    for (const dir of stored.patches ?? [])
+        argv.push("--patches", dir);
+    argv.push("--out", stored.out || outDir);
+    if (stored.repo)
+        argv.push("--repo", stored.repo);
+    if (stored.commit)
+        argv.push("--commit", stored.commit);
+    if (typeof stored.workers === "number")
+        argv.push("--workers", String(stored.workers));
+    if (stored.confirm === false)
+        argv.push("--no-confirm");
+    if (stored.testsDir && stored.testsDir !== "tests")
+        argv.push("--tests-dir", stored.testsDir);
+    if (typeof stored.suiteTimeoutMs === "number")
+        argv.push("--suite-timeout-ms", String(stored.suiteTimeoutMs));
+    if (typeof stored.testTimeoutMs === "number")
+        argv.push("--test-timeout-ms", String(stored.testTimeoutMs));
+    if (stored.direction && stored.direction !== "auto")
+        argv.push("--direction", stored.direction);
+    if (stored.suiteCommand)
+        argv.push("--suite-command", stored.suiteCommand);
+    argv.push(...keep.flatMap((id) => ["--only-test", id]));
+    return { argv };
+}
 function readResults(dir) {
     const out = [];
     for (const name of readdirSync(dir)) {

package/dist/node-version.js

@@ -0,0 +1,13 @@
+/** `--experimental-strip-types` needs this. Older Node is refused before a command runs. */
+export const MIN_NODE = "22.6";
+export function nodeTooOld(version) {
+    const match = /^(\d+)\.(\d+)/.exec(version);
+    const required = `Node ${version} is too old. Probatio needs Node ${MIN_NODE} or newer.`;
+    if (!match)
+        return required;
+    const major = Number(match[1]);
+    const minor = Number(match[2]);
+    if (major > 22 || (major === 22 && minor >= 6))
+        return null;
+    return required;
+}

package/dist/swarm/queue.js

@@ -1,13 +1,19 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { loadSealed } from "../memory/state.js";
+/** One path segment. Rejects slashes and `..` so a claim cannot leave the queue directory. */
+export function safeFilePart(value) {
+    if (!/^[A-Za-z0-9._-]+$/.test(value) || value === "." || value === "..")
+        return null;
+    return value;
+}
 export function seedQueue(root, ids) {
     const sealed = new Set(loadSealed(root));
     const queue = path.join(root, "queue");
     mkdirSync(queue, { recursive: true });
     const written = [];
     for (const id of [...ids].sort()) {
-        if (sealed.has(id))
+        if (sealed.has(id) || !safeFilePart(id))
             continue;
         const dest = path.join(queue, `${id}.json`);
         if (existsSync(dest))
@@ -19,23 +25,23 @@ export function seedQueue(root, ids) {
 }
 /** Claim by atomic rename. A second claim fails because the queue file is already gone. */
 export function claimItem(root, id, agent, now, leaseMs) {
+    if (!safeFilePart(id) || !safeFilePart(agent))
+        return { ok: false, summary: "agent and id must be a single file name.", file: null };
     const source = path.join(root, "queue", `${id}.json`);
     const claimed = path.join(root, "claimed");
     const dest = path.join(claimed, `${agent}-${id}.json`);
     if (!existsSync(source))
         return { ok: false, summary: `${id} is not in the queue.`, file: null };
     mkdirSync(claimed, { recursive: true });
-    const body = JSON.parse(readFileSync(source, "utf8"));
-    body.id = id;
-    body.agent = agent;
-    body.leasedUntil = now + leaseMs;
-    writeFileSync(source, `${JSON.stringify(body)}\n`);
+    // Rename first. Writing the body before the rename lets the other claim overwrite the winner.
     try {
         renameSync(source, dest);
     }
     catch {
         return { ok: false, summary: `${id} was claimed by someone else.`, file: null };
     }
+    const body = { id, agent, leasedUntil: now + leaseMs };
+    writeFileSync(dest, `${JSON.stringify(body)}\n`);
     return { ok: true, summary: `${agent} claimed ${id} until ${body.leasedUntil}.`, file: dest };
 }
 /** Move claims whose lease has expired back to the queue. */
@@ -53,7 +59,7 @@ export function reapClaims(root, now) {
         const body = JSON.parse(readFileSync(file, "utf8"));
         if (typeof body.leasedUntil !== "number" || body.leasedUntil > now)
             continue;
-        if (!body.id)
+        if (!body.id || !safeFilePart(body.id))
             continue;
         renameSync(file, path.join(queue, `${body.id}.json`));
         returned.push(body.id);

package/dist/verify/change.js

@@ -1,4 +1,4 @@
-import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { SCHEMA_VERSION } from "../contract.js";
 import { asTable } from "../golden/check.js";
@@ -23,12 +23,18 @@ export async function verifyChange(options) {
     const ranges = parseUnified(diff.stdout);
     const names = git(repo, ["diff", "--name-only", "--no-renames", options.base, options.commit, ...scope]);
     const changed = names.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    const status = git(repo, ["status", "--porcelain", ...scope]);
+    const dirty = status.stdout.trim().length > 0;
+    if (dirty && (options.base === options.commit || changed.length === 0)) {
+        return fail(options.outDir, "The working tree has uncommitted changes. verify-change did not score them. Commit the edit, or pass --base and --commit that name different commits with a diff.");
+    }
     const goldenContract = goldenDiffs(repo, options.base, options.commit, changed);
     const packageFiles = [];
     const chosen = [];
+    const libImported = testsImportLib(packageDir);
     for (const [file, lines] of ranges) {
         const rel = toPackage(prefix, file);
-        if (!rel || !rel.startsWith("src/") || rel.endsWith(".d.ts"))
+        if (!rel || !isVerifySource(rel, packageDir, libImported))
             continue;
         packageFiles.push(rel);
         const text = showFile(repo, options.commit, file);
@@ -269,3 +275,66 @@ function toPackage(prefix, file) {
         return null;
     return file.slice(prefix.length + 1);
 }
+const SKIPPED_SOURCE = new Set(["node_modules", "dist", "docs", "tests", "test"]);
+/** src/, a lib/ tree the tests import, or a Go file beside go.mod. Tests, docs, dist, and node_modules stay out. */
+function isVerifySource(rel, packageDir, libImported) {
+    if (rel.endsWith(".d.ts"))
+        return false;
+    const parts = rel.split("/");
+    if (parts.some((part) => SKIPPED_SOURCE.has(part)))
+        return false;
+    if (rel.startsWith("src/"))
+        return true;
+    if (libImported && rel.startsWith("lib/"))
+        return true;
+    if (!rel.includes("/") && rel.endsWith(".go") && !rel.endsWith("_test.go") && existsSync(path.join(packageDir, "go.mod")))
+        return true;
+    return false;
+}
+function testsImportLib(packageDir) {
+    const pattern = /(?:require\s*\(\s*|from\s+)['"](?:@[^'"]+\/)?(?:\.\.\/|\.\/)*lib\//;
+    for (const dirName of ["tests", "test"]) {
+        const dir = path.join(packageDir, dirName);
+        if (!existsSync(dir))
+            continue;
+        if (walkText(dir).some((text) => pattern.test(text)))
+            return true;
+    }
+    return false;
+}
+function walkText(dir) {
+    const out = [];
+    const visit = (current) => {
+        let entries;
+        try {
+            entries = readdirSync(current);
+        }
+        catch {
+            return;
+        }
+        for (const name of entries) {
+            if (name === "node_modules" || name === "dist")
+                continue;
+            const full = path.join(current, name);
+            let info;
+            try {
+                info = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (info.isDirectory())
+                visit(full);
+            else if (info.isFile() && info.size < 1_000_000) {
+                try {
+                    out.push(readFileSync(full, "utf8"));
+                }
+                catch {
+                    // An unreadable test file does not count as an import of lib/.
+                }
+            }
+        }
+    };
+    visit(dir);
+    return out;
+}

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "probatio",
-  "version": "0.1.0",
+  "version": "0.1.3",
   "description": "Mutation runner for AI agents. Scores a test suite by the bugs it catches.",
   "type": "module",
   "bin": {
-    "probatio": "dist/cli.js"
+    "probatio": "dist/cli.js",
+    "probatio-mcp": "dist/mcp.js"
   },
   "files": [
     "dist"
@@ -19,12 +20,13 @@
     "url": "https://github.com/IronAdamant/Probatio/issues"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
+    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/precise-lines.mjs src/mutate/child-lines.mjs src/mutate/mocha-coverage.cjs src/mutate/node-batch.mjs src/mutate/node-batch-loader.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
+    "prepack": "npm run build",
     "test": "tsx --test tests/*.test.ts",
     "probatio": "tsx src/cli.ts"
   },
   "engines": {
-    "node": ">=22"
+    "node": ">=22.6"
   },
   "dependencies": {
     "typescript": "^5.7.2"