probatio 0.0.1 → 0.0.3

package/dist/mutate/suites.js

@@ -274,6 +274,9 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
         const tests = selection.map(surefireTest).filter((name) => name.length > 0);
         const args = ["-B", "test", ...(tests.length > 0 ? [`-Dtest=${tests.join(",")}`] : [])];
         const whole = wantsNames && tests.length === 0;
+        // Surefire leaves the previous run's XML in place. A confirm that did not
+        // re-execute a class would otherwise read that class as failed again.
+        rmSync(path.join(pkg, "target", "surefire-reports"), { recursive: true, force: true });
         return tag(spawnCollected("mvn", args, pkg, javaEnv(env), timeoutMs), shown("mvn", args), whole, selection);
     }
     if (spec.kind === "dotnet") {
@@ -284,9 +287,11 @@ async function launch(pkg, spec, files, pattern, names, timeoutMs, env, hooks) {
         return tag(runDotnetExe(pkg, spec, files, timeoutMs, env), spec.command, wantsNames, selection);
     if (spec.kind === "mocha")
         return tag(runMocha(pkg, files, timeoutMs, env), spec.command, wantsNames, selection);
-    // macOS `script` allocates a tty. tini's suite calls `docker run -it`, which exits immediately without one.
-    if (spec.kind === "script")
-        return tag(runScript(pkg, timeoutMs, env), spec.command, wantsNames, selection);
+    // A tty is only for a suite that runs `docker run -it` (tini). Everything else is bash.
+    if (spec.kind === "script") {
+        const launched = scriptInvocation(pkg);
+        return tag(runScript(pkg, timeoutMs, env, launched), launched.command, wantsNames, selection);
+    }
     if (spec.kind === "c" || spec.kind === "cpp")
         return tag(compileAndRun(pkg, files, spec.kind === "cpp", timeoutMs, env), spec.command, wantsNames, selection);
     if (spec.kind === "java")
@@ -549,7 +554,72 @@ function clearBytecode(pkg) {
         }
     }
 }
-function runScript(pkg, timeoutMs, env) {
+let gnuScriptCache = null;
+function suiteNeedsTty(pkg) {
+    const files = new Set();
+    const rootScript = path.join(pkg, "run_tests.sh");
+    if (existsSync(rootScript))
+        files.add(rootScript);
+    try {
+        for (const name of readdirSync(pkg)) {
+            if (name.endsWith(".sh"))
+                files.add(path.join(pkg, name));
+        }
+    }
+    catch {
+        // A missing directory is not a tty.
+    }
+    let entry = "";
+    try {
+        entry = readFileSync(rootScript, "utf8");
+    }
+    catch {
+        entry = "";
+    }
+    for (const match of entry.matchAll(/(?:^|[\s"'`/])([\w.-]+\.sh)\b/g)) {
+        const full = path.resolve(pkg, match[1]);
+        if (full.startsWith(pkg + path.sep) && existsSync(full))
+            files.add(full);
+    }
+    for (const file of files) {
+        let body = "";
+        try {
+            body = readFileSync(file, "utf8");
+        }
+        catch {
+            continue;
+        }
+        if (dockerWantsTty(body))
+            return true;
+    }
+    return false;
+}
+function dockerWantsTty(text) {
+    return text.split("\n").some((line) => {
+        if (!/\bdocker\s+run\b/.test(line))
+            return false;
+        return /(?:^|\s)--tty(?:\s|=|$)/.test(line) || /(?:^|\s)-[A-Za-z]*t[A-Za-z]*(?:\s|$)/.test(line);
+    });
+}
+function gnuScript() {
+    if (gnuScriptCache !== null)
+        return gnuScriptCache;
+    const probe = spawnSync("script", ["--version"], { encoding: "utf8", timeout: 2000 });
+    const text = `${probe.stdout ?? ""}\n${probe.stderr ?? ""}`;
+    gnuScriptCache = probe.status === 0 && /util-linux|\bGNU\b/.test(text);
+    return gnuScriptCache;
+}
+function scriptInvocation(pkg) {
+    if (!suiteNeedsTty(pkg))
+        return { bin: "bash", args: ["run_tests.sh"], command: "bash run_tests.sh" };
+    if (gnuScript()) {
+        const args = ["-q", "-c", "bash run_tests.sh", "/dev/null"];
+        return { bin: "script", args, command: shown("script", args) };
+    }
+    const args = ["-q", "/dev/null", "bash", "run_tests.sh"];
+    return { bin: "script", args, command: shown("script", args) };
+}
+function runScript(pkg, timeoutMs, env, launched) {
     const next = { ...env };
     const shim = writeDockerShim(pkg);
     if (shim)
@@ -560,7 +630,7 @@ function runScript(pkg, timeoutMs, env) {
     if (!next.ARCH_NATIVE && existsSync(inner) && existsSync(build) && readFileSync(build, "utf8").includes("ARCH_NATIVE")) {
         next.ARCH_NATIVE = "1";
     }
-    return spawnCollected("script", ["-q", "/dev/null", "bash", "run_tests.sh"], pkg, next, timeoutMs);
+    return spawnCollected(launched.bin, launched.args, pkg, next, timeoutMs);
 }
 function writeDockerShim(pkg) {
     const found = spawnSync("sh", ["-c", "command -v docker"], { encoding: "utf8" });
@@ -1132,7 +1202,9 @@ function parseDotnet(text, code) {
     const names = [];
     const failed = [];
     for (const line of text.split("\n")) {
-        const match = line.match(/^\s*(Passed|Failed)\s+(\S+)/);
+        // A result line carries a duration: "Passed StaysClosed [4 ms]".
+        // "Failed to load ..." is a diagnostic, not a test, and must not redden the baseline.
+        const match = line.match(/^\s*(Passed|Failed)\s+(.+?)\s+\[(?:<\s*)?\d+(?:\.\d+)?\s*(?:ms|s)\]\s*$/);
         if (!match)
             continue;
         names.push(match[2]);

package/dist/mutate/tally.js

@@ -3,11 +3,11 @@ import path from "node:path";
 import { SCHEMA_VERSION } from "../contract.js";
 import { advisePrune } from "../matrix/report.js";
 import { isBuildName } from "./kill-label.js";
-import { WHOLE_PROGRAM_TEST } from "./suite-decision.js";
+import { leadSummary, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST } from "./suite-decision.js";
 const NOT_A_TEST = new Set(["suite", "pytest", "unittest", "command", WHOLE_PROGRAM_TEST]);
 /**
  * Read a finished mutate run. Name tests that killed a mutant, tests that killed
- * nothing, and one gap per survivor or uncovered line. This never deletes a file.
+ * nothing, and one gap per survivor. An unseen line is not a gap. This never deletes a file.
  */
 export function tallyRun(outDir) {
     const resultsDir = path.join(outDir, "results");
@@ -75,18 +75,30 @@ export function tallyRun(outDir) {
         .map(([name]) => name)
         .sort();
     const drop = pruning.advice;
+    const noCoverage = results.filter((result) => result.outcome === "no coverage").length;
+    const survived = results.filter((result) => result.outcome === "survived").length;
+    const timeouts = results.filter((result) => result.outcome === "timeout").length;
     const gaps = results
-        .filter((result) => result.outcome === "survived" || result.outcome === "no coverage")
+        .filter((result) => result.outcome === "survived")
         .map((result) => ({
         id: result.id,
         file: result.files?.[0]?.file ?? "",
         line: result.files?.[0]?.line ?? 0,
-        outcome: result.outcome,
+        outcome: "survived",
     }))
         .sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line || a.id.localeCompare(b.id));
     const shown = gaps.slice(0, 10);
-    const summary = `${keep.length} tests to keep, ${drop.length} tests killed nothing, ${gaps.length} gaps. No test file was deleted.`;
-    const next = gaps.length === 0 ? "No gap in this run." : `First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}.`;
+    const summary = leadSummary(noCoverage, survived, "survived", `${keep.length} tests to keep, ${drop.length} tests killed nothing, ${gaps.length} gaps. No test file was deleted.`);
+    const notes = [];
+    if (noCoverage > 0)
+        notes.push(`${noCoverage} no coverage. That line was not seen. It is not a pass and not a gap.`);
+    if (gaps.length > 0)
+        notes.push(`First gap is ${gaps[0].id} at ${gaps[0].file}:${gaps[0].line}. A survivor is not a pass.`);
+    if (timeouts > 0)
+        notes.push(TIMEOUT_NEXT);
+    if (notes.length === 0)
+        notes.push("No survivor in this run. No test file was deleted.");
+    const next = notes.join(" ");
     const complete = {
         schemaVersion: SCHEMA_VERSION,
         ok: true,

package/dist/swarm/check-kill.js

@@ -5,6 +5,7 @@ import { directImporters } from "../mutate/affected.js";
 import { filesInDiff } from "../mutate/patch.js";
 import { killLabel } from "../mutate/kill-label.js";
 import { runMutants } from "../mutate/run.js";
+import { campaignSummary } from "../mutate/suite-decision.js";
 /** The gap is done only when this mutant is killed and the baseline suite is green. */
 export async function checkKill(id, options) {
     const importers = importersOf(id, options);
@@ -28,6 +29,21 @@ export async function checkKill(id, options) {
         const baseline = JSON.parse(readFileSync(baselineFile, "utf8"));
         suiteGreen = baseline.ok === true;
     }
+    // A red baseline stops inside the run. Do not read a result that names the broken test.
+    if (!report.ok) {
+        return {
+            schemaVersion: SCHEMA_VERSION,
+            ok: false,
+            command: "check-kill",
+            summary: `${id} is not done. ${report.summary}`,
+            next: "The task is not done.",
+            nextCall: null,
+            done: false,
+            outcome: null,
+            suiteGreen,
+            killedBy: [],
+        };
+    }
     const resultFile = path.join(options.outDir, "results", `${id}.json`);
     let outcome = null;
     let killedBy = [];
@@ -38,11 +54,12 @@ export async function checkKill(id, options) {
     }
     const done = suiteGreen && outcome === "killed";
     const label = done ? killLabel(killedBy) : null;
+    const head = checkHead(report);
     return {
         schemaVersion: SCHEMA_VERSION,
         ok: done,
         command: "check-kill",
-        summary: done ? `${id} is dead and the suite is green.` : `${id} is not done. ${report.summary}`,
+        summary: done ? `${head} ${id} is dead and the suite is green.` : `${head} ${id} is not done.`,
         next: label?.next ?? (done ? "The gap-fixing task is done." : "The task is not done."),
         nextCall: null,
         done,
@@ -60,6 +77,18 @@ function importersOf(id, options) {
     const rels = filesInDiff(readFileSync(patch, "utf8")).map((item) => packageRelative(options.repoDir, options.packageDir, item.file));
     return directImporters(options.packageDir, rels, options.testsDir);
 }
+function checkHead(report) {
+    const finished = report.killed + report.survived + report.flaky + report.timeouts + report.errors + report.noCoverage;
+    return campaignSummary({
+        noCoverage: report.noCoverage,
+        survived: report.survived,
+        killed: report.killed,
+        flaky: report.flaky,
+        timeouts: report.timeouts,
+        errors: report.errors,
+        finished,
+    });
+}
 function packageRelative(repoDir, packageDir, file) {
     const norm = file.split(path.sep).join("/");
     let repo = repoDir;

package/dist/verify/change.js

@@ -7,6 +7,7 @@ import { findInSource } from "../mutate/find.js";
 import { mutantId } from "../mutate/ids.js";
 import { forwardDiff, git } from "../mutate/patch.js";
 import { runMutants } from "../mutate/run.js";
+import { leadSummary } from "../mutate/suite-decision.js";
 import { discoverSuite } from "../mutate/suites.js";
 /** Diff-scoped mutants, the tests that can see the edit, and golden contract rows that changed. */
 export async function verifyChange(options) {
@@ -112,7 +113,7 @@ export async function verifyChange(options) {
 function bodyOf(outDir, ok, affected, ran, notRan, caught, missed, other, goldenContract, notRun, suiteCommand) {
     const uncovered = other.filter((item) => item.outcome === "no coverage");
     const summary = ok
-        ? `${caught.length} caught, ${missed.length} missed, ${notRun.length} not run, ${uncovered.length} no coverage, ${affected.length} affected tests, ${goldenContract.length} golden contract changes.`
+        ? leadSummary(uncovered.length, missed.length, "missed", `${caught.length} caught, ${notRun.length} not run, ${affected.length} affected tests, ${goldenContract.length} golden contract changes.`)
         : "verify-change did not finish.";
     const first = missed[0] ?? uncovered[0] ?? notRun[0] ?? caught[0];
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "probatio",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "Mutation runner for AI agents. Scores a test suite by the bugs it catches.",
   "type": "module",
   "bin": {
@@ -18,7 +18,7 @@
     "url": "https://github.com/IronAdamant/Probatio/issues"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py dist/mutate/",
+    "build": "tsc -p tsconfig.json && mkdir -p dist/mutate && cp src/mutate/fail-reporter.mjs src/mutate/node-coverage.mjs src/mutate/py-reporter.py src/mutate/py-pytest-reporter.py src/mutate/probatio-jacoco-run.java dist/mutate/",
     "test": "tsx --test tests/*.test.ts",
     "probatio": "tsx src/cli.ts"
   },