probatio 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,8 @@ import { writeJavaCoverage } from "./java-coverage.js";
12
12
  import { writeRustCoverage } from "./rust-coverage.js";
13
13
  import { applyPatch, changedLines, chooseDirection, filesInDiff, git, restoreTree } from "./patch.js";
14
14
  import { killLabel } from "./kill-label.js";
15
- import { TIMEOUT_FLOOR_MS, TIMEOUT_MULTIPLE, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST, WHOLE_SUITE_NEXT, campaignSummary, decideSuite, hasLineMap, noCoverageNext, resumeNext, scaleStop, timeoutLimitMs, } from "./suite-decision.js";
15
+ import { CONFIG_FILE, readProjectConfig } from "./project-config.js";
16
+ import { UNVIABLE_NEXT, TIMEOUT_FLOOR_MS, TIMEOUT_MULTIPLE, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST, WHOLE_SUITE_NEXT, campaignSummary, decideSuite, hasLineMap, noCoverageNext, resumeNext, scaleStop, timeoutLimitMs, } from "./suite-decision.js";
16
17
  import { commandSuite, discoverSuite, nodeTestTimeoutMs, runDiscoveredSuite, sourceCandidates } from "./suites.js";
17
18
  import { duplicateTitles } from "./titles.js";
18
19
  export const reporterPath = fileURLToPath(new URL("./fail-reporter.mjs", import.meta.url));
@@ -41,7 +42,7 @@ export async function runMutants(options) {
41
42
  return failReport(options.outDir, "", options.commit, "that commit does not resolve");
42
43
  const commit = sha.stdout.trim();
43
44
  mkdirSync(options.outDir, { recursive: true });
44
- const run = loadRun(options.outDir, commit, options.agent);
45
+ const run = loadRun(options, commit);
45
46
  if ("error" in run)
46
47
  return failReport(options.outDir, run.id, commit, run.error);
47
48
  const listed = listPatches(options.patchDirs);
@@ -52,6 +53,9 @@ export async function runMutants(options) {
52
53
  : listed.files;
53
54
  if (patchFiles.length === 0)
54
55
  return failReport(options.outDir, run.id, commit, "no patches");
56
+ const config = readProjectConfig(packageDir);
57
+ if ("error" in config)
58
+ return failReport(options.outDir, run.id, commit, config.error, undefined, `Fix ${CONFIG_FILE} and run the same command again.`);
55
59
  const gate = resolveSuite(packageDir, options);
56
60
  if (gate.action === "stop")
57
61
  return failReport(options.outDir, run.id, commit, gate.summary, undefined, gate.next, gate.suiteCommand);
@@ -67,6 +71,13 @@ export async function runMutants(options) {
67
71
  if ("error" in worktrees)
68
72
  return failReport(options.outDir, run.id, commit, worktrees.error);
69
73
  hidePaths(worktrees.dirs, repo, packageDir, options.hide ?? []);
74
+ // A cancelled run (MCP cancel, Ctrl-C) still removes its worktrees from the user's repo.
75
+ const onSignal = (signal) => {
76
+ removeWorktrees(repo, worktrees.dirs, options.outDir);
77
+ process.exit(signal === "SIGINT" ? 130 : 143);
78
+ };
79
+ process.once("SIGTERM", onSignal);
80
+ process.once("SIGINT", onSignal);
70
81
  const budgetMs = options.budgetMs ?? (options.maxMinutes == null ? null : options.maxMinutes * 60_000);
71
82
  let budgetStarted = 0;
72
83
  const overBudget = () => budgetMs !== null && budgetStarted !== 0 && Date.now() - budgetStarted > budgetMs;
@@ -113,7 +124,7 @@ export async function runMutants(options) {
113
124
  const prior = history.get(id);
114
125
  if (prior && reusable(prior, raw, treePkg)) {
115
126
  startedMutants += 1;
116
- const reused = reusedResult(prior, commit, run.id, options.agent);
127
+ const reused = normalizeResult(reusedResult(prior, commit, run.id, options.agent));
117
128
  writeResult(options.outDir, reused);
118
129
  progress(`reused ${id}`);
119
130
  continue;
@@ -165,6 +176,8 @@ export async function runMutants(options) {
165
176
  });
166
177
  }
167
178
  finally {
179
+ process.removeListener("SIGTERM", onSignal);
180
+ process.removeListener("SIGINT", onSignal);
168
181
  removeWorktrees(repo, worktrees.dirs, options.outDir);
169
182
  }
170
183
  }
@@ -211,7 +224,7 @@ async function prepare(pkg, options, progress) {
211
224
  if (baseline.compileToken)
212
225
  return { ok: false, error: `baseline build failed (${baseline.compileToken})` };
213
226
  if (!baseline.report)
214
- return { ok: false, error: baselineReportError(baseline.detail) };
227
+ return { ok: false, error: baselineReportError(baseline.detail, baseline.command || spec.command) };
215
228
  baselineFailed = baseline.report.failed.map((item) => item.name);
216
229
  if (collect && !existsSync(mapPath)) {
217
230
  const binary = binaryFromCommand(baseline.command || spec.command, pkg);
@@ -224,6 +237,10 @@ async function prepare(pkg, options, progress) {
224
237
  progress(`suite report tests=${baseline.report.tests}`);
225
238
  progress(baselineFailed.length > 0 ? `baseline failing ${baselineFailed.length}` : "baseline pass");
226
239
  if (baselineFailed.length > 0) {
240
+ const tool = missingToolFrom(baseline.detail, baseline.command || spec.command);
241
+ const unnamed = baselineFailed.every((name) => name === "command" || name === "::command");
242
+ if (tool && unnamed)
243
+ return { ok: false, error: `missing tool: ${tool}. baseline suite did not return a test report` };
227
244
  return { ok: false, error: "baseline already failing", baselineFailed, suiteCommand: spec.command };
228
245
  }
229
246
  if (collect && !existsSync(mapPath) && languageMapKind(spec.kind)) {
@@ -250,12 +267,13 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
250
267
  const raw = readFileSync(patchPath, "utf8");
251
268
  const files = filesInDiff(raw);
252
269
  const edited = changedLines(raw);
270
+ const located = edited.length > 0 ? edited : files;
253
271
  const mutantHash = sha256(raw);
254
272
  const base = () => ({
255
273
  id,
256
274
  outcome: "error",
257
275
  direction: null,
258
- files,
276
+ files: located,
259
277
  killedBy: [],
260
278
  cause: null,
261
279
  next: "",
@@ -287,10 +305,12 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
287
305
  return { ...base(), direction, outcome: "timeout", error: "build timed out", next: TIMEOUT_NEXT };
288
306
  // The baseline build already succeeded. A mutant that no longer builds cannot pass the suite.
289
307
  if (built.code !== 0)
290
- return finishMutant(killed(base, direction, ["build"]), pkg, "", false, []);
308
+ return finishMutant(unviable(base, direction, "build"), pkg, "", false, []);
291
309
  }
292
310
  const picked = selectionFor(prep.coverage, edited.length > 0 ? edited : files);
293
- const forced = options.onlyNames && options.onlyNames.length > 0 ? options.onlyNames : null;
311
+ const forced = options.onlyNames && options.onlyNames.length > 0
312
+ ? prep.spec.kind === "node" ? options.onlyNames.map(nodeTitle) : options.onlyNames
313
+ : null;
294
314
  if (!forced && picked.state === "uncovered") {
295
315
  return {
296
316
  ...base(),
@@ -318,16 +338,34 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
318
338
  }
319
339
  // A compile error can leave the previous run's report on disk. That report is not this mutant.
320
340
  if (main.compileToken)
321
- return finishMutant(killed(base, direction, [main.compileToken]), pkg, commands.join(" | "), false, []);
322
- if (!main.report)
323
- return { ...base(), direction, error: "test reporter returned nothing", command: commands.join(" | ") };
341
+ return finishMutant(unviable(base, direction, main.compileToken), pkg, commands.join(" | "), false, []);
342
+ if (!main.report) {
343
+ const detail = main.detail ? clip(main.detail) : "";
344
+ const tool = missingToolFrom(detail, main.command);
345
+ const error = tool
346
+ ? `missing tool: ${tool}`
347
+ : detail
348
+ ? `test reporter returned nothing (${detail})`
349
+ : "test reporter returned nothing";
350
+ return { ...base(), direction, error, command: commands.join(" | ") };
351
+ }
352
+ // A node name pattern that matches nothing passes the file. That is not a survivor.
353
+ // A requested describe block reports its leaf tests, so any test that ran counts.
354
+ if (forced && prep.spec.kind === "node" && main.report.names.every((name) => isNodeTestFile(name))) {
355
+ return { ...base(), direction, error: `no requested test ran (${forced.join(", ")})`, command: commands.join(" | ") };
356
+ }
324
357
  const whole = main.wholeSuite || forceWhole;
325
358
  const chosen = main.selectedTests;
326
359
  if (main.report.failed.length === 0)
327
360
  return finishMutant({ ...base(), direction, outcome: "survived" }, pkg, commands.join(" | "), whole, chosen);
361
+ // A test file that no longer links ran no test. It is the build failing, the same as javac rejecting it.
362
+ // Only a failure where code ran can kill. When none did, the mutant did not build.
363
+ const ran = main.report.failed.filter((item) => !item.staticLoad);
364
+ if (ran.length === 0)
365
+ return finishMutant(unviable(base, direction, "compiler"), pkg, commands.join(" | "), false, []);
328
366
  if (!options.confirm)
329
- return finishMutant(killed(base, direction, main.report.failed.map((item) => failureId(pkg, item))), pkg, commands.join(" | "), whole, chosen);
330
- const confirmed = await confirmFailures(pkg, options, main.report.failed, prep, limit, commands);
367
+ return finishMutant(killed(base, direction, ran.map((item) => failureId(pkg, item))), pkg, commands.join(" | "), whole, chosen);
368
+ const confirmed = await confirmFailures(pkg, options, ran, prep, limit, commands);
331
369
  const killedBy = confirmed.killed.map((item) => failureId(pkg, item));
332
370
  const flaky = confirmed.flaky.map((item) => failureId(pkg, item));
333
371
  const outcome = killedBy.length > 0 ? "killed" : "flaky";
@@ -355,16 +393,31 @@ function killed(base, direction, killedBy) {
355
393
  const label = killLabel(killedBy);
356
394
  return { ...base(), direction, outcome: "killed", killedBy, cause: label?.cause ?? "test", next: label?.next ?? "" };
357
395
  }
396
+ /** The compiler rejected the mutant. `killedBy` names the compiler so the record says which one. */
397
+ function unviable(base, direction, compiler) {
398
+ return { ...base(), direction, outcome: "unviable", killedBy: [compiler], cause: "build", next: UNVIABLE_NEXT };
399
+ }
400
+ /**
401
+ * Results written before `unviable` existed stored a compile failure as killed with cause build.
402
+ * Read them the new way so an old out dir does not count a compiler as a test.
403
+ */
404
+ export function normalizeResult(result) {
405
+ if (result.outcome === "killed" && result.cause === "build")
406
+ return { ...result, outcome: "unviable", next: UNVIABLE_NEXT };
407
+ return result;
408
+ }
358
409
  async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands) {
359
410
  const counts = new Map();
360
411
  let wholeSuite = false;
361
412
  const named = new Map();
362
413
  const hungFiles = new Map();
414
+ const unloaded = new Map();
363
415
  for (const failure of failed) {
364
416
  const key = failure.file || "*";
365
417
  // A name filter skips the tests that left the handle open, so the file
366
418
  // timeout never comes back. Rerun that file with no name pattern.
367
- const bucket = failure.fileTimeout ? hungFiles : named;
419
+ // A file that did not load has no test name to filter by either.
420
+ const bucket = failure.fileTimeout ? hungFiles : failure.fileLoad ? unloaded : named;
368
421
  const list = bucket.get(key) ?? [];
369
422
  list.push(failure);
370
423
  bucket.set(key, list);
@@ -378,13 +431,17 @@ async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands)
378
431
  if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands))
379
432
  wholeSuite = true;
380
433
  }
434
+ for (const [file, failures] of unloaded) {
435
+ if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands, "load"))
436
+ wholeSuite = true;
437
+ }
381
438
  const killed = [];
382
439
  const flaky = [];
383
440
  for (const failure of failed)
384
441
  (counts.get(failureId(pkg, failure)) === 2 ? killed : flaky).push(failure);
385
442
  return { killed, flaky, wholeSuite };
386
443
  }
387
- async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands) {
444
+ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands, fileFailure = "timeout") {
388
445
  const files = file === "*" ? listTestFiles(pkg, options.testsDir, prep.spec) : [testArg(pkg, file, options.testsDir)];
389
446
  const isolated = { ...options, concurrency: 1 };
390
447
  let wholeSuite = false;
@@ -396,20 +453,23 @@ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, t
396
453
  wholeSuite = true;
397
454
  if (rerun.command)
398
455
  commands.push(rerun.command);
399
- if (pattern === null && rerun.timedOut) {
400
- for (const failure of failures)
401
- bump(counts, failureId(pkg, failure));
456
+ if (pattern === null && fileFailure === "timeout" && rerun.timedOut) {
457
+ for (const id of new Set(failures.map((item) => failureId(pkg, item))))
458
+ bump(counts, id);
402
459
  continue;
403
460
  }
404
461
  if (!rerun.report)
405
462
  continue;
463
+ const seen = new Set();
406
464
  for (const hit of rerun.report.failed) {
407
- if (pattern === null && !hit.fileTimeout)
465
+ if (pattern === null && !(fileFailure === "timeout" ? hit.fileTimeout : hit.fileLoad))
408
466
  continue;
409
467
  const id = failureId(pkg, hit);
410
468
  if (failures.some((item) => failureId(pkg, item) === id))
411
- bump(counts, id);
469
+ seen.add(id);
412
470
  }
471
+ for (const id of seen)
472
+ bump(counts, id);
413
473
  }
414
474
  return wholeSuite;
415
475
  }
@@ -536,7 +596,7 @@ class NodeBatch {
536
596
  return {
537
597
  report: null,
538
598
  timedOut: false,
539
- detail: waited.error || "test reporter returned nothing",
599
+ detail: waited.error || (selected.length > 0 ? `no requested test ran (${selected.join(", ")})` : "test reporter returned nothing"),
540
600
  compileToken: null,
541
601
  command: waited.command || "",
542
602
  wholeSuite: false,
@@ -599,6 +659,11 @@ function isNodeTestFile(file) {
599
659
  const base = path.posix.basename(file);
600
660
  return /\.(test|spec)\.(ts|tsx|mts|js|mjs|cjs)$/.test(base) || /_test\.(ts|tsx|js|mjs)$/.test(base);
601
661
  }
662
+ /** A node keep id may carry its file (`tests/gate.test.ts::zero stays shut`). node:test matches the title. */
663
+ export function nodeTitle(id) {
664
+ const match = /^[^\s:][^:]*\.(?:[cm]?[jt]s|[jt]sx)::(.+)$/.exec(id);
665
+ return match ? match[1] : id;
666
+ }
602
667
  function narrowNodeFiles(pkg, files, names, kind) {
603
668
  if (kind !== "node" || !names || names.length === 0)
604
669
  return files;
@@ -636,6 +701,18 @@ function childEnv(unset) {
636
701
  if (key === "NODE_TEST_CONTEXT" || key.startsWith("NODE_TEST_"))
637
702
  delete env[key];
638
703
  }
704
+ // This run may itself be a child of a suite that another Probatio run is mapping.
705
+ // That outer map is not this run's. Only the baseline sets a map, through runDiscoveredSuite.
706
+ delete env.PROBATIO_COVERAGE_MAP;
707
+ delete env.PROBATIO_PACKAGE_ROOT;
708
+ delete env.PROBATIO_CHILD_DIR;
709
+ if (env.NODE_OPTIONS) {
710
+ const kept = env.NODE_OPTIONS.split(/\s+/).filter((part) => part.length > 0 && !/child-lines\.mjs$/.test(part)).join(" ");
711
+ if (kept)
712
+ env.NODE_OPTIONS = kept;
713
+ else
714
+ delete env.NODE_OPTIONS;
715
+ }
639
716
  for (const key of unset)
640
717
  delete env[key];
641
718
  return env;
@@ -708,8 +785,8 @@ function loadMeta(dirs) {
708
785
  }
709
786
  return meta;
710
787
  }
711
- function loadRun(outDir, commit, agent) {
712
- const file = path.join(outDir, "run.json");
788
+ function loadRun(options, commit) {
789
+ const file = path.join(options.outDir, "run.json");
713
790
  if (existsSync(file)) {
714
791
  const stored = JSON.parse(readFileSync(file, "utf8"));
715
792
  if (stored.commit !== commit)
@@ -717,7 +794,24 @@ function loadRun(outDir, commit, agent) {
717
794
  return { id: stored.id };
718
795
  }
719
796
  const id = randomBytes(4).toString("hex");
720
- writeFileSync(file, `${JSON.stringify({ id, commit, agent, started: new Date().toISOString() }, null, 2)}\n`);
797
+ const record = {
798
+ id,
799
+ commit,
800
+ agent: options.agent,
801
+ started: new Date().toISOString(),
802
+ package: options.packageDir,
803
+ patches: options.patchDirs,
804
+ out: options.outDir,
805
+ repo: options.repoDir,
806
+ workers: options.workers,
807
+ confirm: options.confirm,
808
+ testsDir: options.testsDir,
809
+ suiteTimeoutMs: options.suiteTimeoutMs,
810
+ testTimeoutMs: options.testTimeoutMs,
811
+ direction: options.direction,
812
+ suiteCommand: options.suiteCommand ?? null,
813
+ };
814
+ writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`);
721
815
  return { id };
722
816
  }
723
817
  async function createWorktrees(repo, packageDir, runId, commit, workers, outDir) {
@@ -725,6 +819,10 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
725
819
  const dirs = [];
726
820
  for (let index = 0; index < workers; index++) {
727
821
  const dir = path.join(tmpdir(), `probatio-${runId}-${index}`);
822
+ if (!probatioWorktreeDir(dir)) {
823
+ removeWorktrees(repo, dirs, outDir);
824
+ return { error: "worktree path is outside the probatio temp prefix" };
825
+ }
728
826
  rmSync(dir, { recursive: true, force: true });
729
827
  const added = git(repo, ["worktree", "add", "--detach", dir, commit]);
730
828
  if (added.status !== 0) {
@@ -735,8 +833,12 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
735
833
  }
736
834
  writeFileSync(path.join(outDir, "worktrees.json"), `${JSON.stringify(dirs)}\n`);
737
835
  const sources = nodeModuleSources(repo, packageDir);
836
+ const config = path.join(packageDir, CONFIG_FILE);
738
837
  for (const dir of dirs) {
739
838
  const pkgRoot = packageIn(dir, repo, packageDir);
839
+ // How to run the suite is not the code under test. An uncommitted .probatio.json still applies.
840
+ if (existsSync(config) && !existsSync(path.join(pkgRoot, CONFIG_FILE)))
841
+ copyFileSync(config, path.join(pkgRoot, CONFIG_FILE));
740
842
  for (const source of sources) {
741
843
  const dest = path.join(pkgRoot, source.rel);
742
844
  if (existsSync(dest))
@@ -750,15 +852,42 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
750
852
  function hidePaths(dirs, repo, packageDir, hide) {
751
853
  for (const dir of dirs) {
752
854
  const pkg = packageIn(dir, repo, packageDir);
855
+ const root = path.resolve(pkg);
753
856
  for (const rel of hide) {
754
- if (!rel || path.isAbsolute(rel))
857
+ if (!rel || path.isAbsolute(rel) || rel.split(/[\\/]/).includes(".."))
858
+ continue;
859
+ const target = path.resolve(root, rel);
860
+ if (target !== root && !target.startsWith(root + path.sep))
755
861
  continue;
756
- rmSync(path.join(pkg, rel), { force: true });
862
+ rmSync(target, { force: true });
757
863
  }
758
864
  }
759
865
  }
866
+ /** A worktree Probatio created: one directory under the OS temp dir, named probatio- plus hex and an index. */
867
+ export function probatioWorktreeDir(dir) {
868
+ let resolved = path.resolve(dir);
869
+ try {
870
+ resolved = realpathSync(dir);
871
+ }
872
+ catch {
873
+ // The directory is created on the next line. The unresolved path is still checked.
874
+ }
875
+ const roots = new Set([path.resolve(tmpdir())]);
876
+ try {
877
+ roots.add(realpathSync(tmpdir()));
878
+ }
879
+ catch {
880
+ // tmpdir() itself is enough when realpath is unavailable.
881
+ }
882
+ const inside = [...roots].some((root) => resolved === root || resolved.startsWith(root + path.sep));
883
+ if (!inside)
884
+ return false;
885
+ return /^probatio-[0-9a-f]+-\d+$/.test(path.basename(resolved));
886
+ }
760
887
  function removeWorktrees(repo, dirs, outDir) {
761
888
  for (const dir of dirs) {
889
+ if (!probatioWorktreeDir(dir))
890
+ continue;
762
891
  git(repo, ["worktree", "remove", "--force", dir]);
763
892
  rmSync(dir, { recursive: true, force: true });
764
893
  }
@@ -868,11 +997,12 @@ function readResult(outDir, id) {
868
997
  const file = resultPath(outDir, id);
869
998
  if (!existsSync(file))
870
999
  return null;
871
- return JSON.parse(readFileSync(file, "utf8"));
1000
+ return normalizeResult(JSON.parse(readFileSync(file, "utf8")));
872
1001
  }
873
1002
  function reportOf(results, context) {
874
1003
  const count = (outcome) => results.filter((result) => result.outcome === outcome).length;
875
1004
  const killed = count("killed");
1005
+ const unviableCount = count("unviable");
876
1006
  const survived = count("survived");
877
1007
  const flaky = count("flaky");
878
1008
  const timeouts = count("timeout");
@@ -886,7 +1016,7 @@ function reportOf(results, context) {
886
1016
  return { id: result.id, file, line, ...(known?.op ? { op: known.op } : {}) };
887
1017
  });
888
1018
  const budgetHit = context.budgetHit || context.pendingLeft > 0;
889
- const summary = campaignSummary({ noCoverage, survived, killed, flaky, timeouts, errors, finished: results.length });
1019
+ const summary = campaignSummary({ noCoverage, survived, killed, unviable: unviableCount, flaky, timeouts, errors, finished: results.length });
890
1020
  const notes = [];
891
1021
  const uncovered = results.find((result) => result.outcome === "no coverage");
892
1022
  if (uncovered?.next)
@@ -899,6 +1029,9 @@ function reportOf(results, context) {
899
1029
  notes.push(resumeNext(context.notStarted));
900
1030
  if (results.some((result) => result.wholeSuite))
901
1031
  notes.push(WHOLE_SUITE_NEXT);
1032
+ const broken = results.find((result) => (result.outcome === "error" || result.outcome === "build-failed") && result.error);
1033
+ if (broken?.error)
1034
+ notes.push(`${broken.id}: ${broken.error}`);
902
1035
  if (notes.length === 0)
903
1036
  notes.push("No survivor in this batch.");
904
1037
  return {
@@ -909,6 +1042,8 @@ function reportOf(results, context) {
909
1042
  commit: context.commit,
910
1043
  runId: context.runId,
911
1044
  killed,
1045
+ unviable: unviableCount,
1046
+ unviableIds: results.filter((result) => result.outcome === "unviable").slice(0, 10).map((result) => result.id),
912
1047
  survived,
913
1048
  flaky,
914
1049
  timeouts,
@@ -968,6 +1103,8 @@ function failReport(outDir, runId, commit, error, baselineFailed, next, suiteCom
968
1103
  commit,
969
1104
  runId,
970
1105
  killed: 0,
1106
+ unviable: 0,
1107
+ unviableIds: [],
971
1108
  survived: 0,
972
1109
  flaky: 0,
973
1110
  timeouts: 0,
@@ -1132,10 +1269,19 @@ function failureId(pkg, failure) {
1132
1269
  return BARE_SUITE.has(failure.name) ? `::${failure.name}` : failure.name;
1133
1270
  if (!testFileOnDisk(pkg, failure.file))
1134
1271
  return failure.name;
1135
- const rel = path.relative(pkg, failure.file);
1272
+ // macOS temp dirs are /var and /private/var. A report can name either, so compare real paths.
1273
+ const rel = path.relative(realOrSame(pkg), realOrSame(path.resolve(pkg, failure.file)));
1136
1274
  const file = !rel || rel.startsWith("..") ? path.basename(failure.file) : rel.split(path.sep).join("/");
1137
1275
  return `${file}::${failure.name}`;
1138
1276
  }
1277
+ function realOrSame(file) {
1278
+ try {
1279
+ return realpathSync(file);
1280
+ }
1281
+ catch {
1282
+ return file;
1283
+ }
1284
+ }
1139
1285
  function testFileOnDisk(pkg, file) {
1140
1286
  if (path.isAbsolute(file))
1141
1287
  return existsSync(file);
@@ -1156,13 +1302,27 @@ function testArg(pkg, file, testsDir) {
1156
1302
  function escapeRegExp(value) {
1157
1303
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1158
1304
  }
1159
- function baselineReportError(detail) {
1305
+ function missingToolFrom(detail, command) {
1306
+ const spawnMiss = /spawn ([^\s]+) ENOENT/.exec(detail);
1307
+ if (spawnMiss)
1308
+ return path.basename(spawnMiss[1]);
1309
+ const shellMiss = /(?:^|\s)([A-Za-z0-9_./+-]+): (?:command not found|not found)\b/.exec(detail);
1310
+ if (shellMiss)
1311
+ return path.basename(shellMiss[1]);
1312
+ if (/No such file or directory|ENOENT/.test(detail)) {
1313
+ const bin = command.trim().split(/\s+/)[0];
1314
+ if (bin)
1315
+ return path.basename(bin);
1316
+ }
1317
+ return null;
1318
+ }
1319
+ function baselineReportError(detail, command = "") {
1320
+ const tool = missingToolFrom(detail, command);
1160
1321
  const line = detail.trim();
1161
1322
  // The first line of a Python or Node crash is a stack fragment. The summary
1162
1323
  // states the limit and does not carry that fragment.
1163
- if (!line || isStackFragment(line))
1164
- return "baseline suite did not return a test report";
1165
- return `baseline suite did not return a test report (${line})`;
1324
+ const base = !line || isStackFragment(line) ? "baseline suite did not return a test report" : `baseline suite did not return a test report (${line})`;
1325
+ return tool ? `missing tool: ${tool}. ${base}` : base;
1166
1326
  }
1167
1327
  function isStackFragment(line) {
1168
1328
  return line.startsWith("Traceback") || /^\s+at \S/.test(line) || /File ".+", line \d+/.test(line);
@@ -1,6 +1,7 @@
1
1
  import { spawnSync } from "node:child_process";
2
2
  import { existsSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
3
3
  import path from "node:path";
4
+ import { exportLlvmProfile, llvmToolsPresent } from "./coverage-map.js";
4
5
  /**
5
6
  * One instrumented test binary, then one profile per test name.
6
7
  * A single cargo test profile does not say which test hit the line.
@@ -9,9 +10,7 @@ export function writeRustCoverage(pkg, testNames, dest) {
9
10
  const names = [...new Set(testNames.map((name) => name.trim()).filter((name) => name.length > 0 && !/\s/.test(name)))];
10
11
  if (names.length === 0)
11
12
  return { ok: false, detail: "rust coverage: no test names" };
12
- const llvmCov = findTool("llvm-cov");
13
- const llvmProf = findTool("llvm-profdata");
14
- if (!llvmCov || !llvmProf)
13
+ if (!llvmToolsPresent())
15
14
  return { ok: false, detail: "rust coverage: llvm tool missing" };
16
15
  const env = { ...process.env, RUSTFLAGS: "-C instrument-coverage", CARGO_INCREMENTAL: "0" };
17
16
  const built = spawnSync("cargo", ["test", "--no-run", "--message-format=json"], {
@@ -43,11 +42,8 @@ export function writeRustCoverage(pkg, testNames, dest) {
43
42
  if (!new RegExp(`^test ${escapeRegExp(name)} \\.\\.\\. ok`, "m").test(`${ran.stdout}\n${ran.stderr}`))
44
43
  continue;
45
44
  const merged = path.join(work, `${safe(name)}-${safe(path.basename(exe))}.profdata`);
46
- const merge = spawnSync(llvmProf, ["merge", "-sparse", "-o", merged, raw], { encoding: "utf8" });
47
- if (merge.status !== 0)
48
- continue;
49
- const exported = spawnSync(llvmCov, ["export", exe, `-instr-profile=${merged}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
50
- if (exported.status !== 0)
45
+ const exported = exportLlvmProfile(exe, [raw], merged);
46
+ if (!exported.ok)
51
47
  continue;
52
48
  for (const hit of llvmHits(exported.stdout, pkg))
53
49
  addHit(files, hit.file, hit.line, name);
@@ -136,17 +132,6 @@ function addHit(files, file, line, test) {
136
132
  lines[key] = names;
137
133
  files[file] = lines;
138
134
  }
139
- function findTool(name) {
140
- const fromXcrun = spawnSync("xcrun", ["--find", name], { encoding: "utf8" });
141
- const xcrunPath = fromXcrun.stdout?.trim() ?? "";
142
- if (fromXcrun.status === 0 && xcrunPath && existsSync(xcrunPath))
143
- return xcrunPath;
144
- const which = spawnSync("sh", ["-c", `command -v ${name}`], { encoding: "utf8" });
145
- const found = which.stdout?.trim() ?? "";
146
- if (which.status === 0 && found)
147
- return found;
148
- return null;
149
- }
150
135
  function safe(value) {
151
136
  return value.replace(/[^A-Za-z0-9._-]+/g, "_");
152
137
  }
@@ -1,4 +1,6 @@
1
- import { readFileSync } from "node:fs";
1
+ import { spawnSync } from "node:child_process";
2
+ import { readdirSync, readFileSync, statSync } from "node:fs";
3
+ import path from "node:path";
2
4
  /** The label stays in its own file. The run report must not repeat these three values. */
3
5
  export function readSealedLabel(file) {
4
6
  let text = "";
@@ -18,8 +20,99 @@ export function readSealedLabel(file) {
18
20
  export function labelLeaked(body, label) {
19
21
  return body.includes(label.bugId) || body.includes(label.issueId) || body.includes(label.expectedFail);
20
22
  }
23
+ /** The envelope can be clean while a result file on disk still repeats the label. */
24
+ export function sealedTreeLeaked(dir, label) {
25
+ const visit = (current) => {
26
+ let names;
27
+ try {
28
+ names = readdirSync(current);
29
+ }
30
+ catch {
31
+ return false;
32
+ }
33
+ for (const name of names) {
34
+ const full = path.join(current, name);
35
+ let info;
36
+ try {
37
+ info = statSync(full);
38
+ }
39
+ catch {
40
+ continue;
41
+ }
42
+ if (info.isDirectory()) {
43
+ if (visit(full))
44
+ return true;
45
+ continue;
46
+ }
47
+ if (!info.isFile() || info.size > 2_000_000)
48
+ continue;
49
+ try {
50
+ if (labelLeaked(readFileSync(full, "utf8"), label))
51
+ return true;
52
+ }
53
+ catch {
54
+ // A binary artifact is not a report.
55
+ }
56
+ }
57
+ return false;
58
+ };
59
+ return visit(dir);
60
+ }
21
61
  export const LABEL_LEAK_SUMMARY = "The label was copied into the report.";
22
62
  function field(text, key) {
23
63
  const match = text.match(new RegExp(`^${key}=(.*)$`, "m"));
24
64
  return match?.[1]?.trim() ?? "";
25
65
  }
66
+ /**
67
+ * A sealed run hides the revealing test, but a fix commit often edits an older test too.
68
+ * A kill by that edited test is the fix's own test, not an older test that caught the bug.
69
+ * Returns the edited test files that killed, and whether any kill came from a test the fix left alone.
70
+ */
71
+ export function fixEditedKillers(repo, packageDir, commit, hide, kills) {
72
+ if (kills.length === 0)
73
+ return { files: [], olderTestCatch: false };
74
+ const diff = spawnSync("git", ["-C", repo, "diff", "--name-only", `${commit}^`, commit], { encoding: "utf8" });
75
+ // No parent, or git failed: nothing is known to be edited, so every kill stays an older-test catch.
76
+ if (diff.status !== 0)
77
+ return { files: [], olderTestCatch: true };
78
+ const hidden = new Set(hide.map((rel) => rel.split(path.sep).join("/")));
79
+ const edited = diff.stdout
80
+ .split("\n")
81
+ .filter((rel) => rel.length > 0)
82
+ .map((rel) => ({ repoRel: rel, pkgRel: path.relative(packageDir, path.join(repo, rel)).split(path.sep).join("/") }))
83
+ .filter((item) => !item.pkgRel.startsWith("..") && !hidden.has(item.pkgRel));
84
+ const text = new Map();
85
+ const contents = (repoRel) => {
86
+ if (!text.has(repoRel)) {
87
+ const shown = spawnSync("git", ["-C", repo, "show", `${commit}:${repoRel}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
88
+ text.set(repoRel, shown.status === 0 ? shown.stdout : "");
89
+ }
90
+ return text.get(repoRel) ?? "";
91
+ };
92
+ const fileFor = (id) => edited.find((item) => killerInFile(id, item.pkgRel, () => contents(item.repoRel)));
93
+ const files = new Set();
94
+ let olderTestCatch = false;
95
+ for (const kill of kills) {
96
+ const owners = kill.killedBy.map(fileFor);
97
+ for (const owner of owners)
98
+ if (owner)
99
+ files.add(owner.pkgRel);
100
+ if (owners.some((owner) => !owner))
101
+ olderTestCatch = true;
102
+ }
103
+ return { files: [...files].sort(), olderTestCatch };
104
+ }
105
+ /** True when a kill id names this file, its class, or a test written in it. */
106
+ function killerInFile(id, rel, read) {
107
+ if (id.includes(rel) || id.startsWith(`${path.posix.basename(rel)}::`))
108
+ return true;
109
+ const stem = path.posix.basename(rel).replace(/\.[^.]+$/, "");
110
+ // JUnit `pkg.ClassTest.method` or `ClassTest#method`, Go and Rust paths.
111
+ if (stem.length > 0 && id.split(/[.#:/]+/).includes(stem))
112
+ return true;
113
+ const leaf = id.split(/::|#/).pop()?.split(/\.(?=[^.]+$)/).pop()?.replace(/\[.*\]$/, "").trim() ?? "";
114
+ if (leaf.length < 4)
115
+ return false;
116
+ const body = read();
117
+ return [`"${leaf}"`, `'${leaf}'`, `\`${leaf}\``, `${leaf}(`, `def ${leaf}`, `func ${leaf}`, `fn ${leaf}`].some((needle) => body.includes(needle));
118
+ }
@@ -1,6 +1,7 @@
1
1
  /** Pure suite decisions. The process spawn stays in the runner. */
2
2
  export const UNKNOWN_NEXT = "The suite command is unknown. Pass --suite-command with the command that runs this package's tests.";
3
3
  export const TIMEOUT_NEXT = "Timed out. The mutant ran longer than the baseline allows.";
4
+ export const UNVIABLE_NEXT = "The mutant did not build. No test saw it, so it is not a kill and not a gap. No test is needed.";
4
5
  export const WHOLE_SUITE_NEXT = "The suite command cannot take a test name, so the whole suite ran.";
5
6
  export const TIMEOUT_MULTIPLE = 5;
6
7
  /** cargo-mutants uses 20s. A fixture can pass a smaller positive floor. */
@@ -22,7 +23,7 @@ export function leadSummary(noCoverage, survivors, survivorWord, rest) {
22
23
  return `${noCoverage} no coverage, ${survivors} ${survivorWord}, ${rest}`;
23
24
  }
24
25
  export function campaignSummary(counts) {
25
- return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
26
+ return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.unviable ? `${counts.unviable} did not build, ` : ""}${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
26
27
  }
27
28
  export function timeoutLimitMs(baselineMs, multiple, floorMs) {
28
29
  const floor = Math.max(1, floorMs);