probatio 0.0.5 → 0.1.2

package/dist/mutate/suites.js

@@ -400,8 +400,11 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
     }
     if (spec.kind === "dotnet-exe")
         return tag(runDotnetExe(pkg, spec, files, timeoutMs, env), spec.command, wantsNames, selection);
-    if (spec.kind === "mocha")
-        return tag(runMocha(pkg, files, timeoutMs, env), spec.command, wantsNames, selection);
+    if (spec.kind === "mocha") {
+        const grep = selection.length > 0 ? selection.map(escapeRegExp).join("|") : "";
+        const command = grep ? `${spec.command} --grep ${JSON.stringify(grep)}` : spec.command;
+        return tag(runMocha(pkg, files, timeoutMs, env, selection), command, false, selection);
+    }
     // A tty is only for a suite that runs `docker run -it` (tini). Everything else is bash.
     if (spec.kind === "script") {
         const launched = scriptInvocation(pkg);
@@ -435,6 +438,10 @@ function pytestLeaf(name) {
 function pytestPlan(selection) {
     if (selection.length === 0)
         return { pattern: null, nodeIds: null };
+    // A doubled keep id (file.py::tests/file.py::test_low) has more than one ::.
+    // Reducing it to the leaf would run -k test_low and still kill the mutant.
+    if (selection.some((name) => name.split("::").length > 2))
+        return { pattern: null, nodeIds: selection };
     const leaves = selection.map(pytestLeaf);
     if (leaves.every((leaf) => PYTEST_KEYWORD.test(leaf)))
         return { pattern: leaves.join(" or "), nodeIds: null };
@@ -464,9 +471,9 @@ function goRunFilter(names, pattern) {
 function surefireTest(name) {
     const leaf = name.split("::").pop() ?? name;
     const dot = leaf.lastIndexOf(".");
-    if (dot <= 0)
-        return leaf;
-    return `${leaf.slice(0, dot)}#${leaf.slice(dot + 1)}`;
+    const body = dot <= 0 ? leaf : `${leaf.slice(0, dot)}#${leaf.slice(dot + 1)}`;
+    // testShut[0] collects nothing. method(Parser)[1] keeps the parameter types.
+    return body.replace(/\[[^\]]*\]$/, "");
 }
 function escapeRegExp(value) {
     return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -477,18 +484,46 @@ function runShell(pkg, command, timeoutMs, env) {
     const child = /\b(java|javac|mvn)\b/.test(command) ? javaEnv(env) : env;
     return spawnCollected("sh", ["-c", command], pkg, { ...child, PYTHONDONTWRITEBYTECODE: "1" }, timeoutMs);
 }
+/**
+ * Node's `--test-timeout` caps the file, not only a test that left the option unset.
+ * A test that sets a longer `timeout` is cancelled, and the failure name is the file path.
+ * The flag stays at least that long, and it stays inside the suite cap.
+ */
+export function nodeTestTimeoutMs(pkg, files, testTimeoutMs, suiteTimeoutMs) {
+    let declared = 0;
+    for (const rel of files) {
+        let text = "";
+        try {
+            text = readFileSync(path.join(pkg, rel), "utf8");
+        }
+        catch {
+            continue;
+        }
+        for (const match of text.matchAll(/timeout\s*:\s*(\d[\d_]*)/g)) {
+            const value = Number(match[1].replaceAll("_", ""));
+            if (Number.isFinite(value) && value > declared)
+                declared = value;
+        }
+    }
+    const floor = Number.isFinite(testTimeoutMs) && testTimeoutMs > 0 ? testTimeoutMs : 0;
+    let chosen = Math.max(floor, declared);
+    if (Number.isFinite(suiteTimeoutMs) && suiteTimeoutMs > 0 && chosen > suiteTimeoutMs)
+        chosen = suiteTimeoutMs;
+    return chosen > 0 ? chosen : floor;
+}
 async function runNode(pkg, files, pattern, timeoutMs, env, hooks) {
     const tsx = path.join(pkg, "node_modules", ".bin", "tsx");
     const bin = existsSync(tsx) ? tsx : process.execPath;
     const prefix = existsSync(tsx) ? [] : ["--experimental-strip-types"];
     const hook = env.PROBATIO_COVERAGE_MAP ? fileURLToPath(new URL("./node-coverage.mjs", import.meta.url)) : null;
+    const fileTimeout = nodeTestTimeoutMs(pkg, files, hooks.testTimeoutMs, timeoutMs);
     const result = await spawnCollected(bin, [
         ...prefix,
         ...(hook ? ["--import", hook] : []),
         "--test",
         `--test-reporter=${hooks.reporterPath}`,
         `--test-concurrency=${hooks.concurrency}`,
-        `--test-timeout=${hooks.testTimeoutMs}`,
+        `--test-timeout=${fileTimeout}`,
         ...(pattern ? [`--test-name-pattern=${pattern}`] : []),
         ...files,
     ], pkg, env, timeoutMs);
@@ -541,7 +576,7 @@ function runPython(pkg, files, pattern, timeoutMs, env, script, nodeIds) {
     const targets = nodeIds && nodeIds.length > 0 ? nodeIds : files;
     return spawnCollected("python3", [reporter, ...targets, ...(pattern ? ["--pattern", pattern] : [])], pkg, next, timeoutMs);
 }
-async function runMocha(pkg, files, timeoutMs, env) {
+async function runMocha(pkg, files, timeoutMs, env, names = null) {
     const mocha = ["mocha.js", "mocha"]
         .map((name) => path.join(pkg, "node_modules", "mocha", "bin", name))
         .find((file) => existsSync(file));
@@ -552,6 +587,10 @@ async function runMocha(pkg, files, timeoutMs, env) {
         return prepared;
     const config = readMochaConfig(pkg);
     const args = [mocha, ...buildMochaArgs(pkg, files)];
+    if (names && names.length > 0)
+        args.push("--grep", names.map(escapeRegExp).join("|"));
+    if (env.PROBATIO_COVERAGE_MAP)
+        args.push("--require", fileURLToPath(new URL("./mocha-coverage.cjs", import.meta.url)));
     if (config && config.require.includes("esm")) {
         // The `esm` loader throws on current Node before any test runs. Drop that one require and keep the rest of the project's mocha config.
         const requires = config.require.filter((name) => name !== "esm");
@@ -1148,6 +1187,9 @@ function adapt(kind, pkg, stdout, stderr, code) {
         return parseMocha(stdout);
     if (kind === "make-test" || kind === "command") {
         const text = `${stdout}\n${stderr}`;
+        const ctest = parseCtest(text);
+        if (ctest)
+            return ctest;
         if (/Tests run:\s*\d+/.test(text))
             return parseMakeTest(text);
         if (kind === "make-test")
@@ -1267,6 +1309,32 @@ function parseMocha(stdout) {
         return null;
     }
 }
+/** CTest prints `Test #N: name` and `N - name (Failed)`. Those names are the kill, not `::command`. */
+function parseCtest(text) {
+    if (!/Test #\d+:|The following tests FAILED:/.test(text))
+        return null;
+    const names = [];
+    const failed = [];
+    for (const raw of text.split(/\r?\n/)) {
+        const line = raw.trim();
+        const ran = /^\d+\/\d+\s+Test\s+#\d+:\s+(\S+)/.exec(line);
+        if (ran && !names.includes(ran[1]))
+            names.push(ran[1]);
+        if (ran && /\*\*\*Failed/.test(line) && !failed.some((item) => item.name === ran[1])) {
+            failed.push({ name: ran[1], file: "", line: 0 });
+        }
+        const listed = /^\d+\s+-\s+(\S+)\s+\(Failed\)/.exec(line);
+        if (listed && !failed.some((item) => item.name === listed[1]))
+            failed.push({ name: listed[1], file: "", line: 0 });
+    }
+    if (names.length === 0 && failed.length === 0)
+        return null;
+    for (const item of failed) {
+        if (!names.includes(item.name))
+            names.push(item.name);
+    }
+    return { tests: names.length, pass: Math.max(names.length - failed.length, 0), fail: failed.length, failed, names };
+}
 function parseMakeTest(text) {
     const run = text.match(/Tests run:\s*(\d+)/);
     if (!run)

package/dist/mutate/tally.js

@@ -105,7 +105,7 @@ export function tallyRun(outDir) {
         command: "mutate.tally",
         summary,
         next,
-        nextCall: null,
+        nextCall: keep.length > 0 ? { argv: ["mutate", "run", ...keep.flatMap((id) => ["--only-test", id])] } : null,
         keep,
         drop,
         gaps,

package/dist/verify/change.js

@@ -1,4 +1,4 @@
-import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { SCHEMA_VERSION } from "../contract.js";
 import { asTable } from "../golden/check.js";
@@ -7,7 +7,7 @@ import { findInSource } from "../mutate/find.js";
 import { mutantId } from "../mutate/ids.js";
 import { forwardDiff, git } from "../mutate/patch.js";
 import { runMutants } from "../mutate/run.js";
-import { leadSummary } from "../mutate/suite-decision.js";
+import { WHOLE_PROGRAM_TEST, leadSummary } from "../mutate/suite-decision.js";
 import { discoverSuite } from "../mutate/suites.js";
 /** Diff-scoped mutants, the tests that can see the edit, and golden contract rows that changed. */
 export async function verifyChange(options) {
@@ -26,9 +26,10 @@ export async function verifyChange(options) {
     const goldenContract = goldenDiffs(repo, options.base, options.commit, changed);
     const packageFiles = [];
     const chosen = [];
+    const libImported = testsImportLib(packageDir);
     for (const [file, lines] of ranges) {
         const rel = toPackage(prefix, file);
-        if (!rel || !rel.startsWith("src/") || rel.endsWith(".d.ts"))
+        if (!rel || !isVerifySource(rel, packageDir, libImported))
             continue;
         packageFiles.push(rel);
         const text = showFile(repo, options.commit, file);
@@ -102,15 +103,17 @@ export async function verifyChange(options) {
         other = results
             .filter((item) => item.outcome !== "killed" && item.outcome !== "survived")
             .map((item) => ({ id: item.id, outcome: item.outcome, file: byId.get(item.id)?.file ?? "", line: byId.get(item.id)?.line ?? 0 }));
+        const executed = executedNames(results);
+        const envelope = bodyOf(options.outDir, true, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand, executed);
+        writeFileSync(path.join(options.outDir, "verify.json"), `${JSON.stringify(envelope, null, 2)}\n`);
+        return envelope;
     }
-    else {
-        notRun.push(...written);
-    }
-    const envelope = bodyOf(options.outDir, true, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand);
+    notRun.push(...written);
+    const envelope = bodyOf(options.outDir, true, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand, []);
     writeFileSync(path.join(options.outDir, "verify.json"), `${JSON.stringify(envelope, null, 2)}\n`);
     return envelope;
 }
-function bodyOf(outDir, ok, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand) {
+function bodyOf(outDir, ok, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand, executed) {
     const uncovered = other.filter((item) => item.outcome === "no coverage");
     const summary = ok
         ? leadSummary(uncovered.length, missed.length, "missed", `${caught.length} caught, ${notRun.length} not run, ${affected.length} affected tests, ${goldenContract.length} golden contract changes.`)
@@ -127,6 +130,8 @@ function bodyOf(outDir, ok, affected, ran, notRan, caught, missed, other, golden
         ran,
         notRan,
         suiteCommand,
+        executed,
+        confirmed: false,
         caught,
         missed,
         other,
@@ -158,6 +163,8 @@ function fail(outDir, summary, extra) {
         ran: extra?.ran ?? [],
         notRan: extra?.notRan ?? [],
         suiteCommand: extra?.suiteCommand ?? "",
+        executed: [],
+        confirmed: false,
         caught: [],
         missed: [],
         other: [],
@@ -173,14 +180,17 @@ function readOutcomes(outDir, ids) {
     for (const id of ids) {
         try {
             const result = JSON.parse(readFileSync(path.join(file, `${id}.json`), "utf8"));
-            out.push({ id, outcome: result.outcome });
+            out.push({ id, outcome: result.outcome, selectedTests: result.selectedTests ?? [] });
         }
         catch {
-            out.push({ id, outcome: "missing" });
+            out.push({ id, outcome: "missing", selectedTests: [] });
         }
     }
     return out;
 }
+function executedNames(results) {
+    return [...new Set(results.flatMap((item) => item.selectedTests))].filter((name) => name.length > 0 && name !== WHOLE_PROGRAM_TEST).sort();
+}
 function goldenDiffs(repo, base, commit, files) {
     const found = [];
     for (const file of files) {
@@ -260,3 +270,66 @@ function toPackage(prefix, file) {
         return null;
     return file.slice(prefix.length + 1);
 }
+const SKIPPED_SOURCE = new Set(["node_modules", "dist", "docs", "tests", "test"]);
+/** src/, a lib/ tree the tests import, or a Go file beside go.mod. Tests, docs, dist, and node_modules stay out. */
+function isVerifySource(rel, packageDir, libImported) {
+    if (rel.endsWith(".d.ts"))
+        return false;
+    const parts = rel.split("/");
+    if (parts.some((part) => SKIPPED_SOURCE.has(part)))
+        return false;
+    if (rel.startsWith("src/"))
+        return true;
+    if (libImported && rel.startsWith("lib/"))
+        return true;
+    if (!rel.includes("/") && rel.endsWith(".go") && !rel.endsWith("_test.go") && existsSync(path.join(packageDir, "go.mod")))
+        return true;
+    return false;
+}
+function testsImportLib(packageDir) {
+    const pattern = /(?:require\s*\(\s*|from\s+)['"](?:@[^'"]+\/)?(?:\.\.\/|\.\/)*lib\//;
+    for (const dirName of ["tests", "test"]) {
+        const dir = path.join(packageDir, dirName);
+        if (!existsSync(dir))
+            continue;
+        if (walkText(dir).some((text) => pattern.test(text)))
+            return true;
+    }
+    return false;
+}
+function walkText(dir) {
+    const out = [];
+    const visit = (current) => {
+        let entries;
+        try {
+            entries = readdirSync(current);
+        }
+        catch {
+            return;
+        }
+        for (const name of entries) {
+            if (name === "node_modules" || name === "dist")
+                continue;
+            const full = path.join(current, name);
+            let info;
+            try {
+                info = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (info.isDirectory())
+                visit(full);
+            else if (info.isFile() && info.size < 1_000_000) {
+                try {
+                    out.push(readFileSync(full, "utf8"));
+                }
+                catch {
+                    // An unreadable test file does not count as an import of lib/.
+                }
+            }
+        }
+    };
+    visit(dir);
+    return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "probatio",
-  "version": "0.0.5",
+  "version": "0.1.2",
   "description": "Mutation runner for AI agents. Scores a test suite by the bugs it catches.",
   "type": "module",
   "bin": {
@@ -19,7 +19,7 @@
     "url": "https://github.com/IronAdamant/Probatio/issues"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
+    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/precise-lines.mjs src/mutate/child-lines.mjs src/mutate/mocha-coverage.cjs src/mutate/node-batch.mjs src/mutate/node-batch-loader.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
     "test": "tsx --test tests/*.test.ts",
     "probatio": "tsx src/cli.ts"
   },