probatio 0.1.3 → 0.2.0

package/dist/mutate/sealed.js

@@ -1,3 +1,4 @@
+import { spawnSync } from "node:child_process";
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import path from "node:path";
 /** The label stays in its own file. The run report must not repeat these three values. */
@@ -62,3 +63,56 @@ function field(text, key) {
     const match = text.match(new RegExp(`^${key}=(.*)$`, "m"));
     return match?.[1]?.trim() ?? "";
 }
+/**
+ * A sealed run hides the revealing test, but a fix commit often edits an older test too.
+ * A kill by that edited test is the fix's own test, not an older test that caught the bug.
+ * Returns the edited test files that killed, and whether any kill came from a test the fix left alone.
+ */
+export function fixEditedKillers(repo, packageDir, commit, hide, kills) {
+    if (kills.length === 0)
+        return { files: [], olderTestCatch: false };
+    const diff = spawnSync("git", ["-C", repo, "diff", "--name-only", `${commit}^`, commit], { encoding: "utf8" });
+    // No parent, or git failed: nothing is known to be edited, so every kill stays an older-test catch.
+    if (diff.status !== 0)
+        return { files: [], olderTestCatch: true };
+    const hidden = new Set(hide.map((rel) => rel.split(path.sep).join("/")));
+    const edited = diff.stdout
+        .split("\n")
+        .filter((rel) => rel.length > 0)
+        .map((rel) => ({ repoRel: rel, pkgRel: path.relative(packageDir, path.join(repo, rel)).split(path.sep).join("/") }))
+        .filter((item) => !item.pkgRel.startsWith("..") && !hidden.has(item.pkgRel));
+    const text = new Map();
+    const contents = (repoRel) => {
+        if (!text.has(repoRel)) {
+            const shown = spawnSync("git", ["-C", repo, "show", `${commit}:${repoRel}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+            text.set(repoRel, shown.status === 0 ? shown.stdout : "");
+        }
+        return text.get(repoRel) ?? "";
+    };
+    const fileFor = (id) => edited.find((item) => killerInFile(id, item.pkgRel, () => contents(item.repoRel)));
+    const files = new Set();
+    let olderTestCatch = false;
+    for (const kill of kills) {
+        const owners = kill.killedBy.map(fileFor);
+        for (const owner of owners)
+            if (owner)
+                files.add(owner.pkgRel);
+        if (owners.some((owner) => !owner))
+            olderTestCatch = true;
+    }
+    return { files: [...files].sort(), olderTestCatch };
+}
+/** True when a kill id names this file, its class, or a test written in it. */
+function killerInFile(id, rel, read) {
+    if (id.includes(rel) || id.startsWith(`${path.posix.basename(rel)}::`))
+        return true;
+    const stem = path.posix.basename(rel).replace(/\.[^.]+$/, "");
+    // JUnit `pkg.ClassTest.method` or `ClassTest#method`, Go and Rust paths.
+    if (stem.length > 0 && id.split(/[.#:/]+/).includes(stem))
+        return true;
+    const leaf = id.split(/::|#/).pop()?.split(/\.(?=[^.]+$)/).pop()?.replace(/\[.*\]$/, "").trim() ?? "";
+    if (leaf.length < 4)
+        return false;
+    const body = read();
+    return [`"${leaf}"`, `'${leaf}'`, `\`${leaf}\``, `${leaf}(`, `def ${leaf}`, `func ${leaf}`, `fn ${leaf}`].some((needle) => body.includes(needle));
+}

package/dist/mutate/suite-decision.js

@@ -1,6 +1,7 @@
 /** Pure suite decisions. The process spawn stays in the runner. */
 export const UNKNOWN_NEXT = "The suite command is unknown. Pass --suite-command with the command that runs this package's tests.";
 export const TIMEOUT_NEXT = "Timed out. The mutant ran longer than the baseline allows.";
+export const UNVIABLE_NEXT = "The mutant did not build. No test saw it, so it is not a kill and not a gap. No test is needed.";
 export const WHOLE_SUITE_NEXT = "The suite command cannot take a test name, so the whole suite ran.";
 export const TIMEOUT_MULTIPLE = 5;
 /** cargo-mutants uses 20s. A fixture can pass a smaller positive floor. */
@@ -22,7 +23,7 @@ export function leadSummary(noCoverage, survivors, survivorWord, rest) {
     return `${noCoverage} no coverage, ${survivors} ${survivorWord}, ${rest}`;
 }
 export function campaignSummary(counts) {
-    return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
+    return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.unviable ? `${counts.unviable} did not build, ` : ""}${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
 }
 export function timeoutLimitMs(baselineMs, multiple, floorMs) {
     const floor = Math.max(1, floorMs);

package/dist/mutate/suites.js

@@ -1,8 +1,8 @@
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
+import { projectConfig } from "./project-config.js";
 const SKIP_DIR = new Set(["node_modules", ".git", "target", "dist", "bin", "obj", ".build", "vendor"]);
 export function discoverSuite(pkg, testsDir) {
     return (scriptSuite(pkg) ||
@@ -34,13 +34,14 @@ export async function runDiscoveredSuite(pkg, files, testsDir, pattern, timeoutM
     if (!spec)
         return empty;
     const selected = files.length > 0 ? files : spec.files;
-    const child = { ...env };
+    // The project's own .probatio.json env, then Probatio's collectors.
+    const child = { ...env, ...projectConfig(pkg).env };
     if (extra?.coverageMap)
         child.PROBATIO_COVERAGE_MAP = extra.coverageMap;
     if (extra?.profileFile)
         child.LLVM_PROFILE_FILE = extra.profileFile;
     const result = await launch(pkg, spec, selected, pattern, extra?.names ?? null, timeoutMs, child, hooks);
-    const compileToken = result.timedOut ? null : detectCompile(spec.kind, result.stdout, result.stderr, result.code);
+    const compileToken = result.timedOut ? null : detectCompile(spec.kind, result.stdout, result.stderr, result.code, projectConfig(pkg));
     // Ignore a report read from disk when this run failed to compile. The previous
     // suite's files are still in the worktree and are not this mutant's result.
     const report = result.timedOut || compileToken ? null : adapt(spec.kind, pkg, result.stdout, result.stderr, result.code);
@@ -72,7 +73,11 @@ function cargoSuite(pkg) {
     return { kind: "cargo", files, command: "cargo test" };
 }
 function goSuite(pkg) {
-    const files = walk(pkg, (rel) => rel.endsWith("_test.go"));
+    // `go test ./...` needs a module at or above the package and skips a nested module.
+    // Go files in an example module inside a Node or Python package are not this suite.
+    if (!moduleAbove(pkg, "go.mod"))
+        return null;
+    const files = outsideNested(pkg, ["go.mod"], walk(pkg, (rel) => rel.endsWith("_test.go")));
     if (files.length === 0)
         return null;
     return { kind: "go", files, command: "go test -json ./..." };
@@ -186,17 +191,39 @@ const SOURCE_EXT = /\.(?:cobol|cbl|cob|cpp|cxx|cc|java|asm|nasm|yasm|swift|py|go
 export function sourceCandidates(pkg) {
     return walk(pkg, (rel) => SOURCE_EXT.test(rel) && (hasSegment(rel, "tests") || hasSegment(rel, "test")));
 }
+/** Files that mark a directory as its own Python project. */
+const PYTHON_PROJECT = ["pytest.ini", "pyproject.toml", "setup.py", "setup.cfg", "tox.ini"];
 function pythonTests(pkg, testsDir) {
     const configured = pytestTestPaths(pkg);
     const roots = new Set(configured ?? [testsDir, "tests", "test"]);
-    return walk(pkg, (rel) => {
+    // A nested Python project (an example with its own pytest.ini) is not this package's suite.
+    return outsideNested(pkg, PYTHON_PROJECT, walk(pkg, (rel) => {
         const base = path.posix.basename(rel);
         if (!/^test_.*\.py$/.test(base) && !/_test\.py$/.test(base) && !/^tests_.*\.py$/.test(base))
             return false;
         if (configured)
             return configured.some((root) => rel === root || rel.startsWith(`${root}/`));
         return rel.split("/").some((part) => roots.has(part));
-    });
+    }));
+}
+/** True when `name` sits in the package or one of its parents. */
+function moduleAbove(pkg, name) {
+    let dir = path.resolve(pkg);
+    for (;;) {
+        if (existsSync(path.join(dir, name)))
+            return true;
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            return false;
+        dir = parent;
+    }
+}
+/** Drops files inside a subdirectory that has one of these manifests. That subdirectory is another project. */
+function outsideNested(pkg, manifests, files) {
+    const nested = walk(pkg, (rel) => rel.includes("/") && manifests.includes(path.posix.basename(rel))).map((rel) => path.posix.dirname(rel));
+    if (nested.length === 0)
+        return files;
+    return files.filter((rel) => !nested.some((dir) => rel.startsWith(`${dir}/`)));
 }
 /** Directories pytest itself would collect. A missing setting keeps the broader walk. */
 function pytestTestPaths(pkg) {
@@ -405,7 +432,7 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
         const command = grep ? `${spec.command} --grep ${JSON.stringify(grep)}` : spec.command;
         return tag(runMocha(pkg, files, timeoutMs, env, selection), command, false, selection);
     }
-    // A tty is only for a suite that runs `docker run -it` (tini). Everything else is bash.
+    // A tty is only for a suite whose scripts run `docker run -it`. Everything else is bash.
     if (spec.kind === "script") {
         const launched = scriptInvocation(pkg);
         return tag(runScript(pkg, timeoutMs, env, launched), launched.command, wantsNames, selection);
@@ -807,139 +834,10 @@ function scriptInvocation(pkg) {
     return { bin: "script", args, command: shown("script", args) };
 }
 function runScript(pkg, timeoutMs, env, launched) {
-    const next = { ...env };
-    const shim = writeDockerShim(pkg);
-    if (shim)
-        next.PATH = `${shim}${path.delimiter}${next.PATH ?? ""}`;
-    // This project's CI sets ARCH_NATIVE=1. Without it, ci/run_build.sh skips the tests.
-    const inner = path.join(pkg, "test", "run_inner_tests.py");
-    const build = path.join(pkg, "ci", "run_build.sh");
-    if (!next.ARCH_NATIVE && existsSync(inner) && existsSync(build) && readFileSync(build, "utf8").includes("ARCH_NATIVE")) {
-        next.ARCH_NATIVE = "1";
-    }
-    return spawnCollected(launched.bin, launched.args, pkg, next, timeoutMs);
-}
-function writeDockerShim(pkg) {
-    const found = spawnSync("sh", ["-c", "command -v docker"], { encoding: "utf8" });
-    const real = found.stdout?.trim() ?? "";
-    if (found.status !== 0 || !real)
-        return null;
-    const dir = path.join(pkg, ".probatio-suite", "bin");
-    mkdirSync(dir, { recursive: true });
-    const file = path.join(dir, "docker");
-    writeFileSync(file, dockerShimScript(real), { mode: 0o755 });
-    return dir;
-}
-function dockerShimScript(real) {
-    return `#!/bin/bash
-set -euo pipefail
-REAL=${JSON.stringify(real)}
-if [[ "\${1:-}" != "build" ]]; then
-  exec "$REAL" "$@"
-fi
-shift
-args=()
-context=""
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --build-arg|--file|-f|--tag|-t|--target|--platform|--iidfile|--label|--network|--output|-o|--secret|--ssh|--cache-from|--cache-to|--build-context|--ulimit|--add-host)
-      args+=("$1" "$2"); shift 2 ;;
-    --build-arg=*|--file=*|--tag=*|--target=*|--platform=*|--network=*|--output=*)
-      args+=("$1"); shift ;;
-    --)
-      shift; context="\${1:-}"; break ;;
-    -*)
-      args+=("$1"); shift ;;
-    *)
-      context="$1"; shift ;;
-  esac
-done
-if [[ -z "$context" ]]; then
-  exec "$REAL" build --pull=false "\${args[@]}"
-fi
-src=$(cd "$context" && pwd)
-tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
-cp -a "$src"/. "$tmp"/
-deps="$tmp/ci/install_deps.sh"
-if [[ -f "$deps" ]] && grep -q "pip install --upgrade pip" "$deps"; then
-  python3 - "$deps" << 'PY'
-import sys
-from pathlib import Path
-p = Path(sys.argv[1])
-text = p.read_text()
-old = "python3 -m pip install --upgrade pip\\npython3 -m pip install virtualenv\\n"
-new = """python3 -m pip install 'pip==20.3.4'
-printf '%s\\\\n' '[global]' 'constraint = /etc/pip-constraints.txt' > /etc/pip.conf
-printf '%s\\\\n' 'virtualenv==16.7.10' 'psutil==5.6.7' 'python-prctl==1.8.1' > /etc/pip-constraints.txt
-python3 -m pip install 'virtualenv==16.7.10'
-"""
-if old not in text:
-    raise SystemExit(0)
-p.write_text(text.replace(old, new, 1))
-p.chmod(0o755)
-PY
-fi
-"$REAL" build --pull=false "\${args[@]}" "$tmp"
-`;
+    return spawnCollected(launched.bin, launched.args, pkg, env, timeoutMs);
 }
 function runSwift(pkg, timeoutMs, env) {
-    const args = ["test"];
-    if (declaresOwnPredicate(pkg)) {
-        const overlay = foundationPredicateOverlay();
-        if (overlay) {
-            args.push("-Xswiftc", "-vfsoverlay", "-Xswiftc", overlay.file, "-Xswiftc", "-module-cache-path", "-Xswiftc", overlay.cache);
-        }
-    }
-    return spawnCollected("swift", args, pkg, env, timeoutMs);
-}
-function declaresOwnPredicate(pkg) {
-    return walk(pkg, (rel) => rel.endsWith(".swift")).some((rel) => /struct\s+Predicate\s*</.test(readFileSync(path.join(pkg, rel), "utf8")));
-}
-function foundationPredicateOverlay() {
-    const sdk = spawnSync("xcrun", ["--sdk", "macosx", "--show-sdk-path"], { encoding: "utf8" });
-    const sdkPath = sdk.stdout?.trim() ?? "";
-    if (sdk.status !== 0 || !sdkPath)
-        return null;
-    const moduleDir = path.join(sdkPath, "System/Library/Frameworks/Foundation.framework/Modules/Foundation.swiftmodule");
-    if (!existsSync(moduleDir))
-        return null;
-    const interfaces = readdirSync(moduleDir).filter((name) => name.endsWith(".swiftinterface"));
-    if (interfaces.length === 0)
-        return null;
-    const stamp = String(statSync(path.join(moduleDir, interfaces[0])).mtimeMs);
-    const root = path.join(tmpdir(), "probatio-foundation-overlay");
-    const edited = path.join(root, "interfaces");
-    const file = path.join(root, "overlay.json");
-    const cache = path.join(root, "cache");
-    const stampFile = path.join(root, "stamp");
-    mkdirSync(cache, { recursive: true });
-    if (!existsSync(stampFile) || readFileSync(stampFile, "utf8") !== stamp || !existsSync(file)) {
-        mkdirSync(edited, { recursive: true });
-        const contents = [];
-        for (const name of readdirSync(moduleDir).sort()) {
-            const original = path.join(moduleDir, name);
-            if (!statSync(original).isFile())
-                continue;
-            let external = original;
-            if (name.endsWith(".swiftinterface")) {
-                external = path.join(edited, name);
-                writeFileSync(external, hideFoundationPredicate(readFileSync(original, "utf8")));
-            }
-            contents.push({ name, type: "file", "external-contents": external });
-        }
-        writeFileSync(file, `${JSON.stringify({ version: 0, roots: [{ name: moduleDir, type: "directory", contents }] }, null, 2)}\n`);
-        writeFileSync(stampFile, stamp);
-    }
-    return { file, cache };
-}
-function hideFoundationPredicate(text) {
-    return text
-        .split("public struct Predicate<each Input>")
-        .join("public struct HiddenPredicate<each Input>")
-        .split("public macro Predicate<each Input>")
-        .join("public macro HiddenPredicate<each Input>")
-        .replace(/Foundation::Predicate(?![A-Za-z])/g, "Foundation::HiddenPredicate");
+    return spawnCollected("swift", ["test"], pkg, env, timeoutMs);
 }
 function readMochaConfig(pkg) {
     const file = path.join(pkg, "package.json");
@@ -1145,7 +1043,7 @@ function compileFailed(result) {
     return { ...result, stderr: `PROB_COMPILE_FAILED\n${result.stderr}` };
 }
 /** A compiler token when this run failed before any test result exists. Null when tests ran. */
-export function detectCompile(kind, stdout, stderr, code) {
+export function detectCompile(kind, stdout, stderr, code, config = projectConfig("")) {
     if (code === 0)
         return null;
     const text = `${stdout}\n${stderr}`;
@@ -1169,14 +1067,19 @@ export function detectCompile(kind, stdout, stderr, code) {
         return "tsc";
     if (kind === "make-test" && !/Tests run:\s*\d+/.test(text) && /error:|cobc:|syntax error/i.test(text))
         return "cobc";
-    // A shell suite that dies in the compiler never prints a test title. That is a compile kill.
-    if (kind === "script" && !scriptReported(text) && /:\s*error:|fatal error:|undefined reference to|\[-Werror/.test(text))
+    // A shell suite that dies in the compiler never prints a test line. That mutant did not build.
+    if (kind === "script" && !scriptReported(text, config) && COMPILER_ERROR.test(text))
         return "clang";
     return null;
 }
-function scriptReported(text) {
-    const testLine = /^(Testing (?!with FORCE_SUBREAPER\b).+|Running (?:exit code|reaping|process group|zombie|signal configuration|parent death).+|running signal test .+)$/;
-    return text.split(/\r?\n/).some((raw) => testLine.test(scriptLine(raw)));
+/** A C or C++ compiler or linker error: `file.c:12:5: error:`, or an undefined symbol. */
+const COMPILER_ERROR = /^[^\s:]+\.(?:c|cc|cpp|cxx|h|hh|hpp|m|mm):\d+:\d+:\s+(?:fatal\s+)?error:|undefined reference to|Undefined symbols for architecture/m;
+/** True when the output already has a test line, so a later error is a test, not the build. */
+function scriptReported(text, config) {
+    return text.split(/\r?\n/).some((raw) => {
+        const line = scriptLine(raw);
+        return (config.testLine !== null && config.testLine.test(line)) || TAP_LINE.test(line);
+    });
 }
 function scriptLine(raw) {
     // `script` prints ^D and a backspace before the program's own line.
@@ -1214,7 +1117,7 @@ function adapt(kind, pkg, stdout, stderr, code) {
     if (kind === "swift")
         return parseSwift(`${stdout}\n${stderr}`, code);
     if (kind === "script")
-        return parseScriptReport(`${stdout}\n${stderr}`, code);
+        return parseScriptReport(`${stdout}\n${stderr}`, code, projectConfig(pkg));
     if (kind === "maven")
         return mavenReport(pkg, `${stdout}\n${stderr}`, code);
     if (kind === "dotnet")
@@ -1291,19 +1194,48 @@ function parseCargo(text) {
         return null;
     return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
 }
-function parseScriptReport(text, code) {
+/** TAP: `ok 1 - name`, `not ok 2 - name # SKIP`. Many shell suites print it. */
+const TAP_LINE = /^(not )?ok\b\s*\d*\s*(?:-\s*)?(.*?)\s*(?:#\s*(SKIP|TODO)\b.*)?$/i;
+/**
+ * A shell suite names its tests with the project's `testLine` from .probatio.json, or with TAP.
+ * With neither, it is one command: exit 0 passes, and a failure is `::command`, not an invented name.
+ */
+function parseScriptReport(text, code, config) {
+    const lines = text.split(/\r?\n/).map(scriptLine);
+    if (config.testLine) {
+        const names = [];
+        for (const line of lines) {
+            const match = config.testLine.exec(line);
+            if (!match)
+                continue;
+            const name = (match[1] ?? match[0]).trim();
+            if (name && !names.includes(name))
+                names.push(name);
+        }
+        // A configured test line that never printed is not a report. The baseline stops on that.
+        if (names.length === 0)
+            return null;
+        const passed = code === 0 || (config.passLine !== null && lines.some((line) => config.passLine.test(line)));
+        // The suite stops at the test that failed, so the last test line is the one that was running.
+        const failed = passed ? [] : [{ name: names[names.length - 1], file: "", line: 0 }];
+        return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    }
     const names = [];
-    const testLine = /^(Testing (?!with FORCE_SUBREAPER\b).+|Running (?:exit code|reaping|process group|zombie|signal configuration|parent death).+|running signal test .+)$/;
-    for (const raw of text.split(/\r?\n/)) {
-        const line = scriptLine(raw);
-        if (!testLine.test(line) || names.includes(line))
+    const failed = [];
+    for (const line of lines) {
+        const match = TAP_LINE.exec(line);
+        if (!match || !match[2])
             continue;
-        names.push(line);
+        const name = match[2];
+        if (names.includes(name))
+            continue;
+        names.push(name);
+        if (match[1] && !match[3])
+            failed.push({ name, file: "", line: 0 });
     }
-    if (names.length === 0)
-        return null;
-    const failed = text.includes("All done, tests as expected") || code === 0 ? [] : [{ name: names[names.length - 1], file: "", line: 0 }];
-    return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    if (names.length > 0)
+        return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    return exitReport("command", code);
 }
 function parseSwift(text, code) {
     const names = [];

package/dist/mutate/tally.js

@@ -1,8 +1,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { SCHEMA_VERSION } from "../contract.js";
-import { advisePrune } from "../matrix/report.js";
 import { isBuildName } from "./kill-label.js";
+import { normalizeResult } from "./run.js";
 import { leadSummary, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST } from "./suite-decision.js";
 const NOT_A_TEST = new Set(["suite", "pytest", "unittest", "command", WHOLE_PROGRAM_TEST]);
 /**
@@ -21,7 +21,7 @@ export function tallyRun(outDir) {
             next: "Run mutate run, then tally the same --out directory.",
             nextCall: null,
             keep: [],
-            drop: [],
+            noKillsYet: [],
             gaps: [],
             rest: 0,
             pruning: { mode: "advisory", deletedTests: 0, advice: [] },
@@ -38,7 +38,7 @@ export function tallyRun(outDir) {
             next: "Run mutate run, then tally the same --out directory.",
             nextCall: null,
             keep: [],
-            drop: [],
+            noKillsYet: [],
             gaps: [],
             rest: 0,
             pruning: { mode: "advisory", deletedTests: 0, advice: [] },
@@ -55,26 +55,37 @@ export function tallyRun(outDir) {
             killCounts.set(name, (killCounts.get(name) ?? 0) + 1);
         }
     }
-    const seen = new Set(killCounts.keys());
-    for (const name of coverageNames(path.join(outDir, "coverage-map.json"))) {
-        if (isTestName(name))
-            seen.add(name);
-    }
+    // One test can be spelled two ways: the kill names its file, the line map does not.
+    // The first spelling seen wins, and kill spellings are added first.
+    const seen = [];
+    const see = (name) => {
+        if (!isTestName(name))
+            return;
+        if (seen.some((known) => namesMatch(known, name)))
+            return;
+        seen.push(name);
+    };
+    for (const name of killCounts.keys())
+        see(name);
+    for (const name of coverageNames(path.join(outDir, "coverage-map.json")))
+        see(name);
     for (const result of results) {
-        for (const name of result.selectedTests ?? []) {
-            if (isTestName(name))
-                seen.add(name);
-        }
+        for (const name of result.selectedTests ?? [])
+            see(name);
     }
     const counts = new Map();
     for (const name of seen)
         counts.set(name, countFor(name, killCounts));
-    const pruning = advisePrune(counts);
     const keep = [...counts.entries()]
         .filter(([, killed]) => killed > 0)
         .map(([name]) => name)
         .sort();
-    const drop = pruning.advice;
+    // One batch is not evidence for a deletion. These tests saw no mutant die in this batch, nothing more.
+    const noKillsYet = [...counts.entries()]
+        .filter(([, killed]) => killed === 0)
+        .map(([name]) => name)
+        .sort();
+    const pruning = { mode: "advisory", deletedTests: 0, advice: [] };
     const noCoverage = results.filter((result) => result.outcome === "no coverage").length;
     const survived = results.filter((result) => result.outcome === "survived").length;
     const timeouts = results.filter((result) => result.outcome === "timeout").length;
@@ -88,12 +99,13 @@ export function tallyRun(outDir) {
     }))
         .sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line || a.id.localeCompare(b.id));
     const shown = gaps.slice(0, 10);
-    const summary = leadSummary(noCoverage, survived, "survived", `${keep.length} tests to keep, ${drop.length} tests killed nothing, ${gaps.length} gaps. No test file was deleted.`);
+    const summary = leadSummary(noCoverage, survived, "survived", `${keep.length} tests to keep, ${noKillsYet.length} tests saw no kill in this batch, ${gaps.length} gaps. No test file was deleted.`);
     const notes = [];
     if (noCoverage > 0)
         notes.push(`${noCoverage} no coverage. That line was not seen. It is not a pass and not a gap.`);
-    if (gaps.length > 0)
-        notes.push(`First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}. A survivor is not a pass.`);
+    if (gaps.length > 0) {
+        notes.push(`First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}. A survivor is not a pass. Add one test that fails on that mutant and passes on the code, commit it, then run nextCall.`);
+    }
     if (timeouts > 0)
         notes.push(TIMEOUT_NEXT);
     if (notes.length === 0)
@@ -105,9 +117,9 @@ export function tallyRun(outDir) {
         command: "mutate.tally",
         summary,
         next,
-        nextCall: resumeArgv(outDir, keep),
+        nextCall: nextArgv(outDir, keep, gaps.length > 0),
         keep,
-        drop,
+        noKillsYet,
         gaps,
         rest: 0,
         pruning,
@@ -117,8 +129,13 @@ export function tallyRun(outDir) {
     writeFileSync(fullPath, `${JSON.stringify(complete, null, 2)}\n`);
     return { ...complete, gaps: shown, rest: gaps.length - shown.length };
 }
-function resumeArgv(outDir, keep) {
-    if (keep.length === 0)
+/**
+ * The command after this tally. A gap reruns the batch on HEAD once the new test is committed.
+ * Otherwise the keep ids are rescored at the same commit. Either way the out dir is fresh:
+ * the stored out dir would skip every finished mutant and echo the old result.
+ */
+function nextArgv(outDir, keep, hasGaps) {
+    if (!hasGaps && keep.length === 0)
         return null;
     let stored = {};
     try {
@@ -127,15 +144,16 @@ function resumeArgv(outDir, keep) {
     catch {
         stored = {};
     }
+    const base = stored.out || outDir;
     const argv = ["mutate", "run"];
     if (stored.package)
         argv.push("--package", stored.package);
     for (const dir of stored.patches ?? [])
         argv.push("--patches", dir);
-    argv.push("--out", stored.out || outDir);
+    argv.push("--out", freshDir(base, hasGaps ? "after-gaps" : "rescore"));
     if (stored.repo)
         argv.push("--repo", stored.repo);
-    if (stored.commit)
+    if (!hasGaps && stored.commit)
         argv.push("--commit", stored.commit);
     if (typeof stored.workers === "number")
         argv.push("--workers", String(stored.workers));
@@ -149,11 +167,21 @@ function resumeArgv(outDir, keep) {
         argv.push("--test-timeout-ms", String(stored.testTimeoutMs));
     if (stored.direction && stored.direction !== "auto")
         argv.push("--direction", stored.direction);
-    if (stored.suiteCommand)
+    // --only-test refuses --suite-command, so a keep rescore uses discovery.
+    if (stored.suiteCommand && hasGaps)
         argv.push("--suite-command", stored.suiteCommand);
-    argv.push(...keep.flatMap((id) => ["--only-test", id]));
+    if (!hasGaps)
+        argv.push(...keep.flatMap((id) => ["--only-test", id]));
     return { argv };
 }
+/** First sibling of the out dir that does not exist yet. The same tally prints the same path. */
+function freshDir(base, label) {
+    for (let index = 1;; index++) {
+        const candidate = index === 1 ? `${base}.${label}` : `${base}.${label}-${index}`;
+        if (!existsSync(candidate))
+            return candidate;
+    }
+}
 function readResults(dir) {
     const out = [];
     for (const name of readdirSync(dir)) {
@@ -165,7 +193,7 @@ function readResults(dir) {
                 continue;
             if (!parsed.id)
                 parsed.id = name.slice(0, -".json".length);
-            out.push(parsed);
+            out.push(normalizeResult(parsed));
         }
         catch {
             continue;

package/dist/swarm/check-kill.js

@@ -4,8 +4,8 @@ import { SCHEMA_VERSION } from "../contract.js";
 import { directImporters } from "../mutate/affected.js";
 import { filesInDiff } from "../mutate/patch.js";
 import { killLabel } from "../mutate/kill-label.js";
-import { runMutants } from "../mutate/run.js";
-import { campaignSummary } from "../mutate/suite-decision.js";
+import { normalizeResult, runMutants } from "../mutate/run.js";
+import { campaignSummary, UNVIABLE_NEXT } from "../mutate/suite-decision.js";
 /** The gap is done only when this mutant is killed and the baseline suite is green. */
 export async function checkKill(id, options) {
     const importers = importersOf(id, options);
@@ -48,18 +48,24 @@ export async function checkKill(id, options) {
     let outcome = null;
     let killedBy = [];
     if (existsSync(resultFile)) {
-        const result = JSON.parse(readFileSync(resultFile, "utf8"));
+        const result = normalizeResult(JSON.parse(readFileSync(resultFile, "utf8")));
         outcome = result.outcome ?? null;
         killedBy = Array.isArray(result.killedBy) ? result.killedBy : [];
     }
-    const done = suiteGreen && outcome === "killed";
-    const label = done ? killLabel(killedBy) : null;
+    // A mutant that does not build needs no test. The task is closed, and it is not called a kill.
+    const unviableMutant = suiteGreen && outcome === "unviable";
+    const done = (suiteGreen && outcome === "killed") || unviableMutant;
+    const label = unviableMutant ? { cause: "build", next: UNVIABLE_NEXT } : done ? killLabel(killedBy) : null;
     const head = checkHead(report);
     return {
         schemaVersion: SCHEMA_VERSION,
         ok: done,
         command: "check-kill",
-        summary: done ? `${head} ${id} is dead and the suite is green.` : `${head} ${id} is not done.`,
+        summary: unviableMutant
+            ? `${head} ${id} does not build. No test is needed.`
+            : done
+                ? `${head} ${id} is dead and the suite is green.`
+                : `${head} ${id} is not done.`,
         next: label?.next ?? (done ? "The gap-fixing task is done." : "The task is not done."),
         nextCall: null,
         done,
@@ -78,11 +84,12 @@ function importersOf(id, options) {
     return directImporters(options.packageDir, rels, options.testsDir);
 }
 function checkHead(report) {
-    const finished = report.killed + report.survived + report.flaky + report.timeouts + report.errors + report.noCoverage;
+    const finished = report.killed + report.unviable + report.survived + report.flaky + report.timeouts + report.errors + report.noCoverage;
     return campaignSummary({
         noCoverage: report.noCoverage,
         survived: report.survived,
         killed: report.killed,
+        unviable: report.unviable,
         flaky: report.flaky,
         timeouts: report.timeouts,
         errors: report.errors,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "probatio",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "Mutation runner for AI agents. Scores a test suite by the bugs it catches.",
   "type": "module",
   "bin": {
@@ -20,7 +20,7 @@
     "url": "https://github.com/IronAdamant/Probatio/issues"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/precise-lines.mjs src/mutate/child-lines.mjs src/mutate/mocha-coverage.cjs src/mutate/node-batch.mjs src/mutate/node-batch-loader.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
+    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/precise-lines.mjs src/mutate/child-lines.mjs src/mutate/mocha-coverage.cjs src/mutate/node-batch.mjs src/mutate/node-batch-loader.mjs src/mutate/load-failure.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
     "prepack": "npm run build",
     "test": "tsx --test tests/*.test.ts",
     "probatio": "tsx src/cli.ts"