probatio 0.1.2 → 0.2.0

package/dist/mutate/child-lines.mjs

@@ -1,34 +1,39 @@
-// Nameless line dump for a plain script. This is not a node:test process and it
+// Nameless line dump for a node child of a test. This is not a node:test process and it
 // does not write a coverage shard. The parent test's afterEach reads the dump.
+// The test-process collector puts this file in NODE_OPTIONS, so the child needs no import.
+// The dump is written on `exit`, which also runs after process.exit(), so it is synchronous.
 import { randomBytes } from "node:crypto"
 import { mkdirSync, writeFileSync } from "node:fs"
 import path from "node:path"
-import { openLineSampler } from "./precise-lines.mjs"
+import { openSyncLineSampler } from "./precise-lines.mjs"
 
 const mapPath = process.env.PROBATIO_COVERAGE_MAP
-if (mapPath) {
+if (mapPath && !globalThis.__probatioChildLines) {
+  globalThis.__probatioChildLines = true
   let sampler = null
   try {
-    sampler = await openLineSampler((rel) => rel.endsWith("child-lines.mjs") || rel.endsWith("precise-lines.mjs"))
+    sampler = openSyncLineSampler(
+      (rel) => rel.endsWith("child-lines.mjs") || rel.endsWith("precise-lines.mjs"),
+      process.env.PROBATIO_PACKAGE_ROOT || process.cwd(),
+    )
   } catch (err) {
     writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
   }
   if (sampler) {
-    let wrote = false
-    const dump = () => {
-      if (wrote) return
-      wrote = true
-      sampler.positiveLines().then((hits) => {
-        const dir = `${mapPath}.children`
+    process.on("exit", () => {
+      // A node:test process that loaded this through an inherited NODE_OPTIONS has its own
+      // hooks. Its hits are already named. A nameless dump would land on the wrong test.
+      if (globalThis.__probatioTestHooks) return
+      try {
+        const hits = sampler.positiveLines()
+        // A child started by a mapped test writes where that test's process reads.
+        // A child that was not (a script that imports this file itself) uses the shared dir.
+        const dir = process.env.PROBATIO_CHILD_DIR || `${mapPath}.children`
         mkdirSync(dir, { recursive: true })
-        writeFileSync(
-          path.join(dir, `${process.pid}-${randomBytes(4).toString("hex")}.json`),
-          JSON.stringify({ hits }),
-        )
-      }).catch((err) => {
+        writeFileSync(path.join(dir, `${process.pid}-${randomBytes(4).toString("hex")}.json`), JSON.stringify({ hits }))
+      } catch (err) {
         writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
-      })
-    }
-    process.on("beforeExit", dump)
+      }
+    })
   }
 }

package/dist/mutate/coverage-map.js

@@ -104,17 +104,10 @@ export function writeLlvmCoverage(pkg, profileDir, binary, dest) {
         return { ok: false, detail: "no profraw" };
     if (!binary)
         return { ok: false, detail: "no instrumented binary" };
-    const llvmCov = findTool("llvm-cov");
-    const llvmProf = findTool("llvm-profdata");
-    if (!llvmCov || !llvmProf)
-        return { ok: false, detail: `llvm tool missing (${llvmCov ? "llvm-profdata" : "llvm-cov"})` };
     const merged = path.join(profileDir, "baseline.profdata");
-    const merge = spawnSync(llvmProf, ["merge", "-sparse", "-o", merged, ...raws], { encoding: "utf8" });
-    if (merge.status !== 0)
-        return { ok: false, detail: (merge.stderr || merge.stdout || "llvm-profdata failed").trim() };
-    const exported = spawnSync(llvmCov, ["export", binary, `-instr-profile=${merged}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
-    if (exported.status !== 0)
-        return { ok: false, detail: (exported.stderr || exported.stdout || "llvm-cov failed").trim() };
+    const exported = exportLlvmProfile(binary, raws, merged);
+    if (!exported.ok)
+        return exported;
     const map = llvmExportToMap(exported.stdout, pkg);
     if (!map)
         return { ok: false, detail: "llvm-cov export was not a coverage map" };
@@ -159,14 +152,78 @@ function llvmExportToMap(stdout, pkg) {
         return null;
     return { files };
 }
-function findTool(name) {
+/**
+ * clang and rustc write different profraw versions. Each pair is one LLVM install.
+ * The first pair that can merge the profile wins.
+ */
+export function llvmToolsPresent() {
+    return llvmToolPairs().length > 0;
+}
+export function exportLlvmProfile(binary, raws, merged) {
+    const pairs = llvmToolPairs();
+    if (pairs.length === 0)
+        return { ok: false, detail: "llvm tool missing (llvm-cov)" };
+    let detail = "llvm-profdata failed";
+    for (const { cov, prof } of pairs) {
+        const merge = spawnSync(prof, ["merge", "-sparse", "-o", merged, ...raws], { encoding: "utf8" });
+        if (merge.status !== 0) {
+            detail = (merge.stderr || merge.stdout || "llvm-profdata failed").trim();
+            continue;
+        }
+        const exported = spawnSync(cov, ["export", binary, `-instr-profile=${merged}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+        if (exported.status !== 0) {
+            detail = (exported.stderr || exported.stdout || "llvm-cov failed").trim();
+            continue;
+        }
+        return { ok: true, stdout: exported.stdout };
+    }
+    return { ok: false, detail };
+}
+function llvmToolPairs() {
+    const covs = toolCandidates("llvm-cov");
+    const profs = toolCandidates("llvm-profdata");
+    const pairs = [];
+    const seen = new Set();
+    for (const cov of covs) {
+        const prof = profs.find((item) => path.dirname(item) === path.dirname(cov));
+        if (!prof)
+            continue;
+        const key = `${cov}\0${prof}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        pairs.push({ cov, prof });
+    }
+    return pairs;
+}
+function toolCandidates(name) {
+    const out = [];
+    const add = (file) => {
+        if (!file || out.includes(file) || !existsSync(file))
+            return;
+        out.push(file);
+    };
     const fromXcrun = spawnSync("xcrun", ["--find", name], { encoding: "utf8" });
-    const xcrunPath = fromXcrun.stdout?.trim() ?? "";
-    if (fromXcrun.status === 0 && xcrunPath && existsSync(xcrunPath))
-        return xcrunPath;
+    if (fromXcrun.status === 0)
+        add((fromXcrun.stdout ?? "").trim());
     const which = spawnSync("sh", ["-c", `command -v ${name}`], { encoding: "utf8" });
-    const found = which.stdout?.trim() ?? "";
-    if (which.status === 0 && found)
-        return found;
-    return null;
+    if (which.status === 0)
+        add((which.stdout ?? "").trim());
+    const rustc = spawnSync("rustc", ["--print", "sysroot"], { encoding: "utf8" });
+    if (rustc.status === 0) {
+        const root = path.join(rustc.stdout.trim(), "lib", "rustlib");
+        if (existsSync(root)) {
+            for (const host of readdirSync(root))
+                add(path.join(root, host, "bin", name));
+        }
+    }
+    const usr = "/usr/lib";
+    if (existsSync(usr)) {
+        for (const dir of readdirSync(usr)) {
+            if (dir.startsWith("llvm-"))
+                add(path.join(usr, dir, "bin", name));
+        }
+    }
+    add(path.join("/usr/bin", name));
+    return out;
 }

package/dist/mutate/fail-reporter.mjs

@@ -1,12 +1,21 @@
 // node --test custom reporter. One JSON object on stdout: failures with the file
 // the runner itself recorded. TAP headings are not used.
+import { staticLoadFailure } from "./load-failure.mjs"
+
 export default async function* reporter(source) {
+  // A file that fails to load prints its error on stderr before the file-level failure.
+  const stderr = new Map()
   const failed = []
   const names = []
   let tests = 0
   let pass = 0
   let fail = 0
   for await (const event of source) {
+    if (event.type === "test:stderr") {
+      const key = String(event.data?.file ?? "")
+      stderr.set(key, `${stderr.get(key) ?? ""}${String(event.data?.message ?? "")}`)
+      continue
+    }
     if (event.type === "test:pass" || event.type === "test:fail") {
       const data = event.data ?? {}
       if (data.details && data.details.type === "suite") continue
@@ -18,6 +27,12 @@ export default async function* reporter(source) {
         // An open handle is reported as a timeout whose name is the file path.
         // failureType lives on the error; the text is on error.cause, not message.
         if (fileTimedOut(name, file, data.details?.error)) failure.fileTimeout = true
+        // The file itself failed and no test in it ran: it did not load.
+        else if (isFileName(name, file)) {
+          failure.fileLoad = true
+          // It did not link (removed export, missing module, no parse): no test code ran.
+          if (staticLoadFailure(stderr.get(file) ?? "")) failure.staticLoad = true
+        }
         failed.push(failure)
       }
     } else if (event.type === "test:diagnostic") {
@@ -33,6 +48,12 @@ export default async function* reporter(source) {
   yield JSON.stringify({ tests, pass, fail, failed, names })
 }
 
+function isFileName(name, file) {
+  const base = String(file).split(/[/\\]/).pop()
+  if (!base) return false
+  return name === file || name === base || name.endsWith(`/${base}`) || name.endsWith(`\\${base}`)
+}
+
 function fileTimedOut(name, file, error) {
   if (!error || typeof error !== "object") return false
   const cause = typeof error.cause === "string" ? error.cause : ""

package/dist/mutate/generate.js

@@ -8,8 +8,9 @@ export function generateMutants(options) {
     const started = Date.now();
     const limit = options.maxMinutes;
     const over = () => limit != null && Date.now() - started >= limit * 60_000;
+    const sourceDefault = options.workingTree ? "working-tree" : "commit";
     if (over())
-        return { mutants: [], violations: [], filesVisited: 0, budgetHit: true };
+        return { mutants: [], violations: [], filesVisited: 0, budgetHit: true, source: sourceDefault };
     const srcRoot = path.resolve(options.packageDir, options.srcDir);
     const walked = walkSources(srcRoot, over);
     const files = walked.files.filter((file) => !options.skipFiles.includes(path.basename(file)));
@@ -18,13 +19,18 @@ export function generateMutants(options) {
     const chosen = [];
     let filesVisited = 0;
     let budgetHit = walked.stopped;
+    let source = sourceDefault;
     for (const file of files) {
         if (over()) {
             budgetHit = true;
             break;
         }
         filesVisited += 1;
-        const text = readFileSync(file, "utf8");
+        const loaded = fileText(options.packageDir, file, options);
+        if ("error" in loaded)
+            return { mutants: [], violations: [], filesVisited, budgetHit, source: "commit", error: loaded.error };
+        source = loaded.source;
+        const text = loaded.text;
         const rel = posix(path.relative(options.packageDir, file));
         const found = findInSource(rel, text);
         violations.push(...found.violations);
@@ -34,7 +40,7 @@ export function generateMutants(options) {
             chosen.push({ rel, text, point });
     }
     if (violations.length > 0)
-        return { mutants: [], violations, filesVisited, budgetHit };
+        return { mutants: [], violations, filesVisited, budgetHit, source };
     const capped = options.maxMutants === null ? chosen : chosen.slice(0, options.maxMutants);
     const packageReal = real(options.packageDir);
     const patchRoot = gitRoot(packageReal);
@@ -55,7 +61,22 @@ export function generateMutants(options) {
         mutants.push({ id, file: item.rel, line: item.point.line, op: item.point.op, patch });
     }
     writeFileSync(path.join(mutantsDir, "mutants.json"), `${JSON.stringify({ schemaVersion: 1, seed: options.seed, mutants }, null, 2)}\n`);
-    return { mutants, violations, filesVisited, budgetHit };
+    return { mutants, violations, filesVisited, budgetHit, source };
+}
+function fileText(packageDir, absFile, options) {
+    if (options.workingTree)
+        return { text: readFileSync(absFile, "utf8"), source: "working-tree" };
+    const root = gitRoot(packageDir);
+    if (!root)
+        return { text: readFileSync(absFile, "utf8"), source: "working-tree" };
+    const rel = posix(path.relative(root, real(absFile)));
+    const commit = options.commit && options.commit.length > 0 ? options.commit : "HEAD";
+    const shown = spawnSync("git", ["-C", root, "show", `${commit}:${rel}`], { encoding: "utf8" });
+    if (shown.status !== 0) {
+        const shownRel = posix(path.relative(packageDir, absFile));
+        return { error: `${shownRel} is not in ${commit}. Pass --working-tree to read the checkout.` };
+    }
+    return { text: shown.stdout, source: "commit" };
 }
 function walkSources(dir, over) {
     if (!statExists(dir))

package/dist/mutate/load-failure.mjs

@@ -0,0 +1,40 @@
+// A test file that failed before any test ran either did not link or threw while loading.
+// Did not link: a removed export, a missing module, a file that does not parse. No code ran.
+// A compiled language rejects the same change at compile time, so that mutant is unviable.
+// Threw while loading: the file's own top-level code ran the program and it broke. That is a kill.
+
+const STATIC_ERROR = /^(?:SyntaxError|(?:Type)?Error \[ERR_(?:MODULE_NOT_FOUND|UNKNOWN_FILE_EXTENSION|UNSUPPORTED_DIR_IMPORT|PACKAGE_PATH_NOT_EXPORTED)\]):/
+
+/**
+ * `text` is the error as printed: a `Name: message` line, then `at` frames.
+ * Static when the error is a link or parse error and no frame is in the project's own code.
+ * A SyntaxError thrown by running code (JSON.parse at the top of a test) has a frame in that file.
+ */
+export function staticLoadFailure(text) {
+  const lines = String(text).split(/\r?\n/)
+  const head = lines.findIndex((line) => STATIC_ERROR.test(line.trim()))
+  if (head === -1) return false
+  for (const raw of lines.slice(head + 1)) {
+    const line = raw.trim()
+    if (!line.startsWith("at ")) break
+    if (projectFrame(line)) return false
+  }
+  return true
+}
+
+/** The error object a node:test event carries, printed the way node prints it. */
+export function errorText(error) {
+  if (!error || typeof error !== "object") return ""
+  const name = typeof error.name === "string" ? error.name : "Error"
+  const code = typeof error.code === "string" ? ` [${error.code}]` : ""
+  const message = typeof error.message === "string" ? error.message : ""
+  const stack = typeof error.stack === "string" ? error.stack.split("\n").slice(1).join("\n") : ""
+  return `${name}${code}: ${message}\n${stack}`
+}
+
+function projectFrame(frame) {
+  if (/\bnode:/.test(frame) || frame.includes("<anonymous>")) return false
+  const location = /(?:file:\/\/)?(\/[^\s)]+|[A-Za-z]:\\[^\s)]+):\d+:\d+\)?$/.exec(frame)
+  if (!location) return false
+  return !location[1].split(/[\\/]/).includes("node_modules")
+}

package/dist/mutate/mocha-coverage.cjs

@@ -12,6 +12,8 @@ if (mapPath) {
   let prelude = []
   /** @type {{ positiveLines: () => Promise<Array<{ file: string, line: number }>> } | null} */
   let sampler = null
+  /** @type {((mapPath: string) => Array<{ file: string, line: number }>) | null} */
+  let takeChildHits = null
 
   function add(hits, name) {
     if (!name) return
@@ -40,7 +42,10 @@ if (mapPath) {
     async beforeAll() {
       try {
         const opened = await import("./precise-lines.mjs")
-        sampler = await opened.openLineSampler((rel) => rel.endsWith("mocha-coverage.cjs") || rel.endsWith("precise-lines.mjs") || rel.endsWith("node-coverage.mjs"))
+        sampler = await opened.openLineSampler((rel) => rel.endsWith("mocha-coverage.cjs") || rel.endsWith("precise-lines.mjs") || rel.endsWith("node-coverage.mjs") || rel.endsWith("child-lines.mjs"))
+        // A node child of a Mocha test is mapped under that test, the same as under node:test.
+        opened.exposeChildLines()
+        takeChildHits = opened.takeChildHits
         prelude = await sampler.positiveLines()
       } catch (err) {
         writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
@@ -52,6 +57,7 @@ if (mapPath) {
       try {
         add(prelude, title)
         add(await sampler.positiveLines(), title)
+        if (takeChildHits) add(takeChildHits(mapPath), title)
       } catch (err) {
         writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
       }

package/dist/mutate/node-batch-loader.mjs

@@ -40,8 +40,22 @@ function underPackage(abs) {
   return real === root || real.startsWith(root.endsWith(path.sep) ? root : root + path.sep)
 }
 
+/** TypeScript with NodeNext writes `./limit.js` for `limit.ts`. tsx resolves that, so this loader does too. */
+function typeScriptSibling(specifier) {
+  if (!/^(\.{1,2}\/|\/|file:)/.test(specifier)) return null
+  const swapped = specifier.replace(/\.js$/, ".ts").replace(/\.mjs$/, ".mts").replace(/\.cjs$/, ".cts").replace(/\.jsx$/, ".tsx")
+  return swapped === specifier ? null : swapped
+}
+
 export async function resolve(specifier, context, nextResolve) {
-  const resolved = await nextResolve(specifier, context)
+  let resolved
+  try {
+    resolved = await nextResolve(specifier, context)
+  } catch (error) {
+    const sibling = typeScriptSibling(specifier)
+    if (!sibling) throw error
+    resolved = await nextResolve(sibling, context)
+  }
   const gen = generation()
   if (!gen) return resolved
   let abs

package/dist/mutate/node-batch.mjs

@@ -1,10 +1,11 @@
 // One suite process for covered mutants of one file. Jobs arrive on stdin.
 // A mutant that exits this process ends it. The parent starts a clean process for the next mutant.
 import { createInterface } from "node:readline"
-import { realpathSync, writeFileSync } from "node:fs"
+import { readFileSync, realpathSync, writeFileSync } from "node:fs"
 import { register } from "node:module"
 import { run } from "node:test"
 import path from "node:path"
+import { errorText, staticLoadFailure } from "./load-failure.mjs"
 import { openLineSampler } from "./precise-lines.mjs"
 
 const loader = new URL("./node-batch-loader.mjs", import.meta.url).href
@@ -33,6 +34,12 @@ async function runJob(job) {
   const patterns = (job.names || []).filter((name) => name.length > 0).map((name) => new RegExp(`^${escapeRegExp(name)}$`))
   const failed = []
   const names = []
+  // run() with isolation "none" ignores testNamePatterns on Node 22, so every test in the file runs.
+  // Only a requested test, or a subtest of one, is a result. Another test failing is not a kill.
+  const wanted = new Set((job.names || []).filter((name) => name.length > 0))
+  const stack = []
+  const requested = (name, nesting) =>
+    wanted.size === 0 || wanted.has(name) || stack.slice(0, nesting).some((parent) => wanted.has(parent))
   const testTimeout = job.testTimeoutMs > 0 ? job.testTimeoutMs : job.timeoutMs
   const stream = run({
     files,
@@ -47,10 +54,28 @@ async function runJob(job) {
   const collected = (async () => {
     for await (const event of stream) {
       lastEvent = Date.now()
-      if (event.type !== "test:pass" && event.type !== "test:fail") continue
       const data = event.data || {}
+      const nesting = Number.isInteger(data.nesting) ? data.nesting : 0
+      if (event.type === "test:start") {
+        stack.length = nesting
+        stack[nesting] = typeof data.name === "string" ? data.name : ""
+        continue
+      }
+      if (event.type !== "test:pass" && event.type !== "test:fail") continue
       const name = typeof data.name === "string" ? data.name : ""
-      if (!name || files.includes(name)) continue
+      if (!name) continue
+      if (files.includes(name)) {
+        // The file itself failed: it did not load (a removed export, a throw at import time).
+        // No test in it ran. That is a failure of the file, the same as node --test reports it.
+        if (event.type === "test:fail" && fileHasRequested(name, wanted)) {
+          const posix = path.relative(job.root, name).split(path.sep).join("/")
+          const linked = !staticLoadFailure(errorText(data.details && data.details.error))
+          failed.push({ name: posix, file: name, line: 1, fileLoad: true, ...(linked ? {} : { staticLoad: true }) })
+          if (!names.includes(posix)) names.push(posix)
+        }
+        continue
+      }
+      if (!requested(name, nesting)) continue
       if (!names.includes(name)) names.push(name)
       if (event.type === "test:fail") {
         const error = data.details && data.details.error
@@ -101,6 +126,18 @@ async function runJob(job) {
   return { id: job.id, pid: process.pid, failed, names, command, dirty }
 }
 
+/** A file with no requested test in it does not speak for the requested tests. */
+function fileHasRequested(file, wanted) {
+  if (wanted.size === 0) return true
+  let text = ""
+  try {
+    text = readFileSync(file, "utf8")
+  } catch {
+    return true
+  }
+  return [...wanted].some((name) => text.includes(name))
+}
+
 function fileNameIs(name, file) {
   const base = String(file).split(/[/\\]/).pop()
   if (!base) return false

package/dist/mutate/node-coverage.mjs

@@ -1,13 +1,14 @@
 // Baseline-only per-test line map for node --test. Loaded with --import in each
 // test-file process. Writes one shard; the runner merges shards after the suite.
 // takePreciseCoverage resets counters, so each sample is only what ran since the last take.
-// A plain-script child does not run these hooks. It may leave a nameless hit list in
-// ${map}.children. afterEach stores that list under the parent test before the shard is written.
+// A node child of a test does not run these hooks. exposeChildLines puts child-lines.mjs in
+// its NODE_OPTIONS, so it leaves a nameless hit list in ${map}.children when it exits.
+// afterEach stores that list under the parent test before the shard is written.
 import { randomBytes } from "node:crypto"
-import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"
+import { mkdirSync, writeFileSync } from "node:fs"
 import { after, afterEach, before } from "node:test"
 import path from "node:path"
-import { openLineSampler } from "./precise-lines.mjs"
+import { exposeChildLines, openLineSampler, takeChildHits } from "./precise-lines.mjs"
 
 const mapPath = process.env.PROBATIO_COVERAGE_MAP
 if (mapPath) {
@@ -19,6 +20,7 @@ if (mapPath) {
   }
 
   if (sampler) {
+    exposeChildLines()
     /** @type {Map<string, Map<number, Set<string>>>} */
     const byFile = new Map()
     /** @type {Array<{ file: string, line: number }>} */
@@ -41,33 +43,6 @@ if (mapPath) {
       }
     }
 
-    function takeChildHits() {
-      const dir = `${mapPath}.children`
-      if (!existsSync(dir)) return []
-      /** @type {Array<{ file: string, line: number }>} */
-      const hits = []
-      for (const name of readdirSync(dir)) {
-        if (!name.endsWith(".json")) continue
-        const full = path.join(dir, name)
-        try {
-          const parsed = JSON.parse(readFileSync(full, "utf8"))
-          const list = parsed && Array.isArray(parsed.hits) ? parsed.hits : []
-          for (const hit of list) {
-            if (!hit || typeof hit.file !== "string" || !Number.isInteger(hit.line)) continue
-            hits.push({ file: hit.file, line: hit.line })
-          }
-        } catch {
-          // A torn dump is ignored. The parent does not invent a line.
-        }
-        try {
-          unlinkSync(full)
-        } catch {
-          // The next test must not inherit this dump.
-        }
-      }
-      return hits
-    }
-
     before(async () => {
       try {
         prelude = await sampler.positiveLines()
@@ -80,7 +55,7 @@ if (mapPath) {
       try {
         add(prelude, t.name)
         add(await sampler.positiveLines(), t.name)
-        add(takeChildHits(), t.name)
+        add(takeChildHits(mapPath), t.name)
       } catch (err) {
         writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
       }

package/dist/mutate/patch.js

@@ -131,5 +131,6 @@ export function applyPatch(repo, raw, direction) {
 /** Put a throwaway worktree back. A diff label is not trusted for the path list. */
 export function restoreTree(repo) {
     git(repo, ["checkout", "--", "."]);
-    git(repo, ["clean", "-fd", "-e", "node_modules"]);
+    // .probatio.json may be the user's uncommitted copy, put there when the worktree was made.
+    git(repo, ["clean", "-fd", "-e", "node_modules", "-e", ".probatio.json"]);
 }