probatio 0.1.2 → 0.2.0

package/dist/mutate/suites.js

@@ -1,8 +1,8 @@
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
+import { projectConfig } from "./project-config.js";
 const SKIP_DIR = new Set(["node_modules", ".git", "target", "dist", "bin", "obj", ".build", "vendor"]);
 export function discoverSuite(pkg, testsDir) {
     return (scriptSuite(pkg) ||
@@ -34,13 +34,14 @@ export async function runDiscoveredSuite(pkg, files, testsDir, pattern, timeoutM
     if (!spec)
         return empty;
     const selected = files.length > 0 ? files : spec.files;
-    const child = { ...env };
+    // The project's own .probatio.json env, then Probatio's collectors.
+    const child = { ...env, ...projectConfig(pkg).env };
     if (extra?.coverageMap)
         child.PROBATIO_COVERAGE_MAP = extra.coverageMap;
     if (extra?.profileFile)
         child.LLVM_PROFILE_FILE = extra.profileFile;
     const result = await launch(pkg, spec, selected, pattern, extra?.names ?? null, timeoutMs, child, hooks);
-    const compileToken = result.timedOut ? null : detectCompile(spec.kind, result.stdout, result.stderr, result.code);
+    const compileToken = result.timedOut ? null : detectCompile(spec.kind, result.stdout, result.stderr, result.code, projectConfig(pkg));
     // Ignore a report read from disk when this run failed to compile. The previous
     // suite's files are still in the worktree and are not this mutant's result.
     const report = result.timedOut || compileToken ? null : adapt(spec.kind, pkg, result.stdout, result.stderr, result.code);
@@ -72,7 +73,11 @@ function cargoSuite(pkg) {
     return { kind: "cargo", files, command: "cargo test" };
 }
 function goSuite(pkg) {
-    const files = walk(pkg, (rel) => rel.endsWith("_test.go"));
+    // `go test ./...` needs a module at or above the package and skips a nested module.
+    // Go files in an example module inside a Node or Python package are not this suite.
+    if (!moduleAbove(pkg, "go.mod"))
+        return null;
+    const files = outsideNested(pkg, ["go.mod"], walk(pkg, (rel) => rel.endsWith("_test.go")));
     if (files.length === 0)
         return null;
     return { kind: "go", files, command: "go test -json ./..." };
@@ -186,17 +191,39 @@ const SOURCE_EXT = /\.(?:cobol|cbl|cob|cpp|cxx|cc|java|asm|nasm|yasm|swift|py|go
 export function sourceCandidates(pkg) {
     return walk(pkg, (rel) => SOURCE_EXT.test(rel) && (hasSegment(rel, "tests") || hasSegment(rel, "test")));
 }
+/** Files that mark a directory as its own Python project. */
+const PYTHON_PROJECT = ["pytest.ini", "pyproject.toml", "setup.py", "setup.cfg", "tox.ini"];
 function pythonTests(pkg, testsDir) {
     const configured = pytestTestPaths(pkg);
     const roots = new Set(configured ?? [testsDir, "tests", "test"]);
-    return walk(pkg, (rel) => {
+    // A nested Python project (an example with its own pytest.ini) is not this package's suite.
+    return outsideNested(pkg, PYTHON_PROJECT, walk(pkg, (rel) => {
         const base = path.posix.basename(rel);
         if (!/^test_.*\.py$/.test(base) && !/_test\.py$/.test(base) && !/^tests_.*\.py$/.test(base))
             return false;
         if (configured)
             return configured.some((root) => rel === root || rel.startsWith(`${root}/`));
         return rel.split("/").some((part) => roots.has(part));
-    });
+    }));
+}
+/** True when `name` sits in the package or one of its parents. */
+function moduleAbove(pkg, name) {
+    let dir = path.resolve(pkg);
+    for (;;) {
+        if (existsSync(path.join(dir, name)))
+            return true;
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            return false;
+        dir = parent;
+    }
+}
+/** Drops files inside a subdirectory that has one of these manifests. That subdirectory is another project. */
+function outsideNested(pkg, manifests, files) {
+    const nested = walk(pkg, (rel) => rel.includes("/") && manifests.includes(path.posix.basename(rel))).map((rel) => path.posix.dirname(rel));
+    if (nested.length === 0)
+        return files;
+    return files.filter((rel) => !nested.some((dir) => rel.startsWith(`${dir}/`)));
 }
 /** Directories pytest itself would collect. A missing setting keeps the broader walk. */
 function pytestTestPaths(pkg) {
@@ -386,8 +413,8 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
     if (spec.kind === "swift")
         return tag(runSwift(pkg, timeoutMs, env), spec.command, wantsNames, selection);
     if (spec.kind === "maven") {
-        const tests = selection.map(surefireTest).filter((name) => name.length > 0);
-        const args = ["-B", "test", ...(tests.length > 0 ? [`-Dtest=${tests.join(",")}`] : [])];
+        const tests = surefireArg(uniqueSurefire(selection));
+        const args = ["-B", "test", ...(tests.length > 0 ? [`-Dtest=${tests}`] : [])];
         const whole = wantsNames && tests.length === 0;
         // Surefire leaves the previous run's XML in place. A confirm that did not
         // re-execute a class would otherwise read that class as failed again.
@@ -405,7 +432,7 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
         const command = grep ? `${spec.command} --grep ${JSON.stringify(grep)}` : spec.command;
         return tag(runMocha(pkg, files, timeoutMs, env, selection), command, false, selection);
     }
-    // A tty is only for a suite that runs `docker run -it` (tini). Everything else is bash.
+    // A tty is only for a suite whose scripts run `docker run -it`. Everything else is bash.
     if (spec.kind === "script") {
         const launched = scriptInvocation(pkg);
         return tag(runScript(pkg, timeoutMs, env, launched), launched.command, wantsNames, selection);
@@ -468,12 +495,47 @@ function goRunFilter(names, pattern) {
         return null;
     return `^(${names.map(escapeRegExp).join("|")})$`;
 }
+function uniqueSurefire(names) {
+    const out = [];
+    const seen = new Set();
+    for (const name of names) {
+        const selector = surefireTest(name);
+        if (!selector || seen.has(selector))
+            continue;
+        seen.add(selector);
+        out.push(selector);
+    }
+    return out;
+}
+/** Surefire 2.12 splits `-Dtest` on commas and then reads `#method+method`. `Class#a,Class#b` runs only `a`. */
+export function surefireArg(tests) {
+    const groups = new Map();
+    for (const test of tests) {
+        const hash = test.indexOf("#");
+        const klass = hash === -1 ? test : test.slice(0, hash);
+        const method = hash === -1 ? "" : test.slice(hash + 1);
+        const methods = groups.get(klass);
+        if (methods) {
+            if (method)
+                methods.push(method);
+        }
+        else {
+            groups.set(klass, method ? [method] : []);
+        }
+    }
+    return [...groups.entries()]
+        .map(([klass, methods]) => (methods.length > 0 ? `${klass}#${methods.join("+")}` : klass))
+        .join(",");
+}
 function surefireTest(name) {
     const leaf = name.split("::").pop() ?? name;
     const dot = leaf.lastIndexOf(".");
     const body = dot <= 0 ? leaf : `${leaf.slice(0, dot)}#${leaf.slice(dot + 1)}`;
-    // testShut[0] collects nothing. method(Parser)[1] keeps the parameter types.
-    return body.replace(/\[[^\]]*\]$/, "");
+    // testShut[0] matches nothing. A bare testShut matches on Surefire 3.6 and runs
+    // zero tests on 3.5. testShut* matches every invocation on both. method(Parser)[1] keeps the types.
+    const invocation = /\[[^\]]*\]$/.test(body);
+    const method = body.replace(/\[[^\]]*\]$/, "");
+    return invocation ? `${method}*` : method;
 }
 function escapeRegExp(value) {
     return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -764,146 +826,18 @@ function scriptInvocation(pkg) {
     if (!suiteNeedsTty(pkg))
         return { bin: "bash", args: ["run_tests.sh"], command: "bash run_tests.sh" };
     if (gnuScript()) {
-        const args = ["-q", "-c", "bash run_tests.sh", "/dev/null"];
+        // -e returns the child status. Without it, util-linux script exits 0 and a failing suite looks green.
+        const args = ["-q", "-e", "-c", "bash run_tests.sh", "/dev/null"];
         return { bin: "script", args, command: shown("script", args) };
     }
     const args = ["-q", "/dev/null", "bash", "run_tests.sh"];
     return { bin: "script", args, command: shown("script", args) };
 }
 function runScript(pkg, timeoutMs, env, launched) {
-    const next = { ...env };
-    const shim = writeDockerShim(pkg);
-    if (shim)
-        next.PATH = `${shim}${path.delimiter}${next.PATH ?? ""}`;
-    // This project's CI sets ARCH_NATIVE=1. Without it, ci/run_build.sh skips the tests.
-    const inner = path.join(pkg, "test", "run_inner_tests.py");
-    const build = path.join(pkg, "ci", "run_build.sh");
-    if (!next.ARCH_NATIVE && existsSync(inner) && existsSync(build) && readFileSync(build, "utf8").includes("ARCH_NATIVE")) {
-        next.ARCH_NATIVE = "1";
-    }
-    return spawnCollected(launched.bin, launched.args, pkg, next, timeoutMs);
-}
-function writeDockerShim(pkg) {
-    const found = spawnSync("sh", ["-c", "command -v docker"], { encoding: "utf8" });
-    const real = found.stdout?.trim() ?? "";
-    if (found.status !== 0 || !real)
-        return null;
-    const dir = path.join(pkg, ".probatio-suite", "bin");
-    mkdirSync(dir, { recursive: true });
-    const file = path.join(dir, "docker");
-    writeFileSync(file, dockerShimScript(real), { mode: 0o755 });
-    return dir;
-}
-function dockerShimScript(real) {
-    return `#!/bin/bash
-set -euo pipefail
-REAL=${JSON.stringify(real)}
-if [[ "\${1:-}" != "build" ]]; then
-  exec "$REAL" "$@"
-fi
-shift
-args=()
-context=""
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --build-arg|--file|-f|--tag|-t|--target|--platform|--iidfile|--label|--network|--output|-o|--secret|--ssh|--cache-from|--cache-to|--build-context|--ulimit|--add-host)
-      args+=("$1" "$2"); shift 2 ;;
-    --build-arg=*|--file=*|--tag=*|--target=*|--platform=*|--network=*|--output=*)
-      args+=("$1"); shift ;;
-    --)
-      shift; context="\${1:-}"; break ;;
-    -*)
-      args+=("$1"); shift ;;
-    *)
-      context="$1"; shift ;;
-  esac
-done
-if [[ -z "$context" ]]; then
-  exec "$REAL" build --pull=false "\${args[@]}"
-fi
-src=$(cd "$context" && pwd)
-tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
-cp -a "$src"/. "$tmp"/
-deps="$tmp/ci/install_deps.sh"
-if [[ -f "$deps" ]] && grep -q "pip install --upgrade pip" "$deps"; then
-  python3 - "$deps" << 'PY'
-import sys
-from pathlib import Path
-p = Path(sys.argv[1])
-text = p.read_text()
-old = "python3 -m pip install --upgrade pip\\npython3 -m pip install virtualenv\\n"
-new = """python3 -m pip install 'pip==20.3.4'
-printf '%s\\\\n' '[global]' 'constraint = /etc/pip-constraints.txt' > /etc/pip.conf
-printf '%s\\\\n' 'virtualenv==16.7.10' 'psutil==5.6.7' 'python-prctl==1.8.1' > /etc/pip-constraints.txt
-python3 -m pip install 'virtualenv==16.7.10'
-"""
-if old not in text:
-    raise SystemExit(0)
-p.write_text(text.replace(old, new, 1))
-p.chmod(0o755)
-PY
-fi
-"$REAL" build --pull=false "\${args[@]}" "$tmp"
-`;
+    return spawnCollected(launched.bin, launched.args, pkg, env, timeoutMs);
 }
 function runSwift(pkg, timeoutMs, env) {
-    const args = ["test"];
-    if (declaresOwnPredicate(pkg)) {
-        const overlay = foundationPredicateOverlay();
-        if (overlay) {
-            args.push("-Xswiftc", "-vfsoverlay", "-Xswiftc", overlay.file, "-Xswiftc", "-module-cache-path", "-Xswiftc", overlay.cache);
-        }
-    }
-    return spawnCollected("swift", args, pkg, env, timeoutMs);
-}
-function declaresOwnPredicate(pkg) {
-    return walk(pkg, (rel) => rel.endsWith(".swift")).some((rel) => /struct\s+Predicate\s*</.test(readFileSync(path.join(pkg, rel), "utf8")));
-}
-function foundationPredicateOverlay() {
-    const sdk = spawnSync("xcrun", ["--sdk", "macosx", "--show-sdk-path"], { encoding: "utf8" });
-    const sdkPath = sdk.stdout?.trim() ?? "";
-    if (sdk.status !== 0 || !sdkPath)
-        return null;
-    const moduleDir = path.join(sdkPath, "System/Library/Frameworks/Foundation.framework/Modules/Foundation.swiftmodule");
-    if (!existsSync(moduleDir))
-        return null;
-    const interfaces = readdirSync(moduleDir).filter((name) => name.endsWith(".swiftinterface"));
-    if (interfaces.length === 0)
-        return null;
-    const stamp = String(statSync(path.join(moduleDir, interfaces[0])).mtimeMs);
-    const root = path.join(tmpdir(), "probatio-foundation-overlay");
-    const edited = path.join(root, "interfaces");
-    const file = path.join(root, "overlay.json");
-    const cache = path.join(root, "cache");
-    const stampFile = path.join(root, "stamp");
-    mkdirSync(cache, { recursive: true });
-    if (!existsSync(stampFile) || readFileSync(stampFile, "utf8") !== stamp || !existsSync(file)) {
-        mkdirSync(edited, { recursive: true });
-        const contents = [];
-        for (const name of readdirSync(moduleDir).sort()) {
-            const original = path.join(moduleDir, name);
-            if (!statSync(original).isFile())
-                continue;
-            let external = original;
-            if (name.endsWith(".swiftinterface")) {
-                external = path.join(edited, name);
-                writeFileSync(external, hideFoundationPredicate(readFileSync(original, "utf8")));
-            }
-            contents.push({ name, type: "file", "external-contents": external });
-        }
-        writeFileSync(file, `${JSON.stringify({ version: 0, roots: [{ name: moduleDir, type: "directory", contents }] }, null, 2)}\n`);
-        writeFileSync(stampFile, stamp);
-    }
-    return { file, cache };
-}
-function hideFoundationPredicate(text) {
-    return text
-        .split("public struct Predicate<each Input>")
-        .join("public struct HiddenPredicate<each Input>")
-        .split("public macro Predicate<each Input>")
-        .join("public macro HiddenPredicate<each Input>")
-        .replace(/Foundation::Predicate(?![A-Za-z])/g, "Foundation::HiddenPredicate");
+    return spawnCollected("swift", ["test"], pkg, env, timeoutMs);
 }
 function readMochaConfig(pkg) {
     const file = path.join(pkg, "package.json");
@@ -1109,7 +1043,7 @@ function compileFailed(result) {
     return { ...result, stderr: `PROB_COMPILE_FAILED\n${result.stderr}` };
 }
 /** A compiler token when this run failed before any test result exists. Null when tests ran. */
-export function detectCompile(kind, stdout, stderr, code) {
+export function detectCompile(kind, stdout, stderr, code, config = projectConfig("")) {
     if (code === 0)
         return null;
     const text = `${stdout}\n${stderr}`;
@@ -1133,14 +1067,19 @@ export function detectCompile(kind, stdout, stderr, code) {
         return "tsc";
     if (kind === "make-test" && !/Tests run:\s*\d+/.test(text) && /error:|cobc:|syntax error/i.test(text))
         return "cobc";
-    // A shell suite that dies in the compiler never prints a test title. That is a compile kill.
-    if (kind === "script" && !scriptReported(text) && /:\s*error:|fatal error:|undefined reference to|\[-Werror/.test(text))
+    // A shell suite that dies in the compiler never prints a test line. That mutant did not build.
+    if (kind === "script" && !scriptReported(text, config) && COMPILER_ERROR.test(text))
         return "clang";
     return null;
 }
-function scriptReported(text) {
-    const testLine = /^(Testing (?!with FORCE_SUBREAPER\b).+|Running (?:exit code|reaping|process group|zombie|signal configuration|parent death).+|running signal test .+)$/;
-    return text.split(/\r?\n/).some((raw) => testLine.test(scriptLine(raw)));
+/** A C or C++ compiler or linker error: `file.c:12:5: error:`, or an undefined symbol. */
+const COMPILER_ERROR = /^[^\s:]+\.(?:c|cc|cpp|cxx|h|hh|hpp|m|mm):\d+:\d+:\s+(?:fatal\s+)?error:|undefined reference to|Undefined symbols for architecture/m;
+/** True when the output already has a test line, so a later error is a test, not the build. */
+function scriptReported(text, config) {
+    return text.split(/\r?\n/).some((raw) => {
+        const line = scriptLine(raw);
+        return (config.testLine !== null && config.testLine.test(line)) || TAP_LINE.test(line);
+    });
 }
 function scriptLine(raw) {
     // `script` prints ^D and a backspace before the program's own line.
@@ -1178,7 +1117,7 @@ function adapt(kind, pkg, stdout, stderr, code) {
     if (kind === "swift")
         return parseSwift(`${stdout}\n${stderr}`, code);
     if (kind === "script")
-        return parseScriptReport(`${stdout}\n${stderr}`, code);
+        return parseScriptReport(`${stdout}\n${stderr}`, code, projectConfig(pkg));
     if (kind === "maven")
         return mavenReport(pkg, `${stdout}\n${stderr}`, code);
     if (kind === "dotnet")
@@ -1255,19 +1194,48 @@ function parseCargo(text) {
         return null;
     return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
 }
-function parseScriptReport(text, code) {
+/** TAP: `ok 1 - name`, `not ok 2 - name # SKIP`. Many shell suites print it. */
+const TAP_LINE = /^(not )?ok\b\s*\d*\s*(?:-\s*)?(.*?)\s*(?:#\s*(SKIP|TODO)\b.*)?$/i;
+/**
+ * A shell suite names its tests with the project's `testLine` from .probatio.json, or with TAP.
+ * With neither, it is one command: exit 0 passes, and a failure is `::command`, not an invented name.
+ */
+function parseScriptReport(text, code, config) {
+    const lines = text.split(/\r?\n/).map(scriptLine);
+    if (config.testLine) {
+        const names = [];
+        for (const line of lines) {
+            const match = config.testLine.exec(line);
+            if (!match)
+                continue;
+            const name = (match[1] ?? match[0]).trim();
+            if (name && !names.includes(name))
+                names.push(name);
+        }
+        // A configured test line that never printed is not a report. The baseline stops on that.
+        if (names.length === 0)
+            return null;
+        const passed = code === 0 || (config.passLine !== null && lines.some((line) => config.passLine.test(line)));
+        // The suite stops at the test that failed, so the last test line is the one that was running.
+        const failed = passed ? [] : [{ name: names[names.length - 1], file: "", line: 0 }];
+        return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    }
     const names = [];
-    const testLine = /^(Testing (?!with FORCE_SUBREAPER\b).+|Running (?:exit code|reaping|process group|zombie|signal configuration|parent death).+|running signal test .+)$/;
-    for (const raw of text.split(/\r?\n/)) {
-        const line = scriptLine(raw);
-        if (!testLine.test(line) || names.includes(line))
+    const failed = [];
+    for (const line of lines) {
+        const match = TAP_LINE.exec(line);
+        if (!match || !match[2])
             continue;
-        names.push(line);
+        const name = match[2];
+        if (names.includes(name))
+            continue;
+        names.push(name);
+        if (match[1] && !match[3])
+            failed.push({ name, file: "", line: 0 });
     }
-    if (names.length === 0)
-        return null;
-    const failed = text.includes("All done, tests as expected") || code === 0 ? [] : [{ name: names[names.length - 1], file: "", line: 0 }];
-    return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    if (names.length > 0)
+        return { tests: names.length, pass: names.length - failed.length, fail: failed.length, failed, names };
+    return exitReport("command", code);
 }
 function parseSwift(text, code) {
     const names = [];
@@ -1403,6 +1371,11 @@ function mavenReport(pkg, text, code) {
     }
     return parseSurefire(pkg) ?? parseMavenText(text);
 }
+/** An attribute on the opening tag. `classname` must not be read as `name`. */
+export function surefireAttr(fragment, key) {
+    const open = fragment.slice(0, fragment.indexOf(">") === -1 ? fragment.length : fragment.indexOf(">"));
+    return new RegExp(`(?:^|\\s)${key}="([^"]*)"`).exec(open)?.[1] ?? "";
+}
 function parseSurefire(pkg) {
     const dir = path.join(pkg, "target", "surefire-reports");
     if (!existsSync(dir))
@@ -1414,11 +1387,11 @@ function parseSurefire(pkg) {
             continue;
         const xml = readFileSync(path.join(dir, name), "utf8");
         for (const part of xml.split("<testcase ").slice(1)) {
-            const test = /name="([^"]+)"/.exec(part)?.[1];
-            const klass = /classname="([^"]+)"/.exec(part)?.[1] ?? "";
+            const test = surefireAttr(part, "name");
+            const klass = surefireAttr(part, "classname");
             if (!test)
                 continue;
-            const id = `${klass}.${test}`;
+            const id = klass ? `${klass}.${test}` : test;
             names.push(id);
             if (part.includes("<failure") || part.includes("<error"))
                 failed.push({ name: id, file: klass, line: 0 });

package/dist/mutate/tally.js

@@ -1,8 +1,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { SCHEMA_VERSION } from "../contract.js";
-import { advisePrune } from "../matrix/report.js";
 import { isBuildName } from "./kill-label.js";
+import { normalizeResult } from "./run.js";
 import { leadSummary, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST } from "./suite-decision.js";
 const NOT_A_TEST = new Set(["suite", "pytest", "unittest", "command", WHOLE_PROGRAM_TEST]);
 /**
@@ -21,7 +21,7 @@ export function tallyRun(outDir) {
             next: "Run mutate run, then tally the same --out directory.",
             nextCall: null,
             keep: [],
-            drop: [],
+            noKillsYet: [],
             gaps: [],
             rest: 0,
             pruning: { mode: "advisory", deletedTests: 0, advice: [] },
@@ -38,7 +38,7 @@ export function tallyRun(outDir) {
             next: "Run mutate run, then tally the same --out directory.",
             nextCall: null,
             keep: [],
-            drop: [],
+            noKillsYet: [],
             gaps: [],
             rest: 0,
             pruning: { mode: "advisory", deletedTests: 0, advice: [] },
@@ -55,26 +55,37 @@ export function tallyRun(outDir) {
             killCounts.set(name, (killCounts.get(name) ?? 0) + 1);
         }
     }
-    const seen = new Set(killCounts.keys());
-    for (const name of coverageNames(path.join(outDir, "coverage-map.json"))) {
-        if (isTestName(name))
-            seen.add(name);
-    }
+    // One test can be spelled two ways: the kill names its file, the line map does not.
+    // The first spelling seen wins, and kill spellings are added first.
+    const seen = [];
+    const see = (name) => {
+        if (!isTestName(name))
+            return;
+        if (seen.some((known) => namesMatch(known, name)))
+            return;
+        seen.push(name);
+    };
+    for (const name of killCounts.keys())
+        see(name);
+    for (const name of coverageNames(path.join(outDir, "coverage-map.json")))
+        see(name);
     for (const result of results) {
-        for (const name of result.selectedTests ?? []) {
-            if (isTestName(name))
-                seen.add(name);
-        }
+        for (const name of result.selectedTests ?? [])
+            see(name);
     }
     const counts = new Map();
     for (const name of seen)
         counts.set(name, countFor(name, killCounts));
-    const pruning = advisePrune(counts);
     const keep = [...counts.entries()]
         .filter(([, killed]) => killed > 0)
         .map(([name]) => name)
         .sort();
-    const drop = pruning.advice;
+    // One batch is not evidence for a deletion. These tests saw no mutant die in this batch, nothing more.
+    const noKillsYet = [...counts.entries()]
+        .filter(([, killed]) => killed === 0)
+        .map(([name]) => name)
+        .sort();
+    const pruning = { mode: "advisory", deletedTests: 0, advice: [] };
     const noCoverage = results.filter((result) => result.outcome === "no coverage").length;
     const survived = results.filter((result) => result.outcome === "survived").length;
     const timeouts = results.filter((result) => result.outcome === "timeout").length;
@@ -88,12 +99,13 @@ export function tallyRun(outDir) {
     }))
         .sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line || a.id.localeCompare(b.id));
     const shown = gaps.slice(0, 10);
-    const summary = leadSummary(noCoverage, survived, "survived", `${keep.length} tests to keep, ${drop.length} tests killed nothing, ${gaps.length} gaps. No test file was deleted.`);
+    const summary = leadSummary(noCoverage, survived, "survived", `${keep.length} tests to keep, ${noKillsYet.length} tests saw no kill in this batch, ${gaps.length} gaps. No test file was deleted.`);
     const notes = [];
     if (noCoverage > 0)
         notes.push(`${noCoverage} no coverage. That line was not seen. It is not a pass and not a gap.`);
-    if (gaps.length > 0)
-        notes.push(`First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}. A survivor is not a pass.`);
+    if (gaps.length > 0) {
+        notes.push(`First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}. A survivor is not a pass. Add one test that fails on that mutant and passes on the code, commit it, then run nextCall.`);
+    }
     if (timeouts > 0)
         notes.push(TIMEOUT_NEXT);
     if (notes.length === 0)
@@ -105,9 +117,9 @@ export function tallyRun(outDir) {
         command: "mutate.tally",
         summary,
         next,
-        nextCall: keep.length > 0 ? { argv: ["mutate", "run", ...keep.flatMap((id) => ["--only-test", id])] } : null,
+        nextCall: nextArgv(outDir, keep, gaps.length > 0),
         keep,
-        drop,
+        noKillsYet,
         gaps,
         rest: 0,
         pruning,
@@ -117,6 +129,59 @@ export function tallyRun(outDir) {
     writeFileSync(fullPath, `${JSON.stringify(complete, null, 2)}\n`);
     return { ...complete, gaps: shown, rest: gaps.length - shown.length };
 }
+/**
+ * The command after this tally. A gap reruns the batch on HEAD once the new test is committed.
+ * Otherwise the keep ids are rescored at the same commit. Either way the out dir is fresh:
+ * the stored out dir would skip every finished mutant and echo the old result.
+ */
+function nextArgv(outDir, keep, hasGaps) {
+    if (!hasGaps && keep.length === 0)
+        return null;
+    let stored = {};
+    try {
+        stored = JSON.parse(readFileSync(path.join(outDir, "run.json"), "utf8"));
+    }
+    catch {
+        stored = {};
+    }
+    const base = stored.out || outDir;
+    const argv = ["mutate", "run"];
+    if (stored.package)
+        argv.push("--package", stored.package);
+    for (const dir of stored.patches ?? [])
+        argv.push("--patches", dir);
+    argv.push("--out", freshDir(base, hasGaps ? "after-gaps" : "rescore"));
+    if (stored.repo)
+        argv.push("--repo", stored.repo);
+    if (!hasGaps && stored.commit)
+        argv.push("--commit", stored.commit);
+    if (typeof stored.workers === "number")
+        argv.push("--workers", String(stored.workers));
+    if (stored.confirm === false)
+        argv.push("--no-confirm");
+    if (stored.testsDir && stored.testsDir !== "tests")
+        argv.push("--tests-dir", stored.testsDir);
+    if (typeof stored.suiteTimeoutMs === "number")
+        argv.push("--suite-timeout-ms", String(stored.suiteTimeoutMs));
+    if (typeof stored.testTimeoutMs === "number")
+        argv.push("--test-timeout-ms", String(stored.testTimeoutMs));
+    if (stored.direction && stored.direction !== "auto")
+        argv.push("--direction", stored.direction);
+    // --only-test refuses --suite-command, so a keep rescore uses discovery.
+    if (stored.suiteCommand && hasGaps)
+        argv.push("--suite-command", stored.suiteCommand);
+    if (!hasGaps)
+        argv.push(...keep.flatMap((id) => ["--only-test", id]));
+    return { argv };
+}
+/** First sibling of the out dir that does not exist yet. The same tally prints the same path. */
+function freshDir(base, label) {
+    for (let index = 1;; index++) {
+        const candidate = index === 1 ? `${base}.${label}` : `${base}.${label}-${index}`;
+        if (!existsSync(candidate))
+            return candidate;
+    }
+}
 function readResults(dir) {
     const out = [];
     for (const name of readdirSync(dir)) {
@@ -128,7 +193,7 @@ function readResults(dir) {
                 continue;
             if (!parsed.id)
                 parsed.id = name.slice(0, -".json".length);
-            out.push(parsed);
+            out.push(normalizeResult(parsed));
         }
         catch {
             continue;

package/dist/node-version.js

@@ -0,0 +1,13 @@
+/** `--experimental-strip-types` needs this. Older Node is refused before a command runs. */
+export const MIN_NODE = "22.6";
+export function nodeTooOld(version) {
+    const match = /^(\d+)\.(\d+)/.exec(version);
+    const required = `Node ${version} is too old. Probatio needs Node ${MIN_NODE} or newer.`;
+    if (!match)
+        return required;
+    const major = Number(match[1]);
+    const minor = Number(match[2]);
+    if (major > 22 || (major === 22 && minor >= 6))
+        return null;
+    return required;
+}