probatio 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -17
- package/dist/cli.js +122 -55
- package/dist/contract.js +7 -5
- package/dist/ledger/build.js +14 -0
- package/dist/mcp.js +98 -26
- package/dist/mutate/child-lines.mjs +23 -18
- package/dist/mutate/coverage-map.js +75 -18
- package/dist/mutate/fail-reporter.mjs +21 -0
- package/dist/mutate/generate.js +25 -4
- package/dist/mutate/load-failure.mjs +40 -0
- package/dist/mutate/mocha-coverage.cjs +7 -1
- package/dist/mutate/node-batch-loader.mjs +15 -1
- package/dist/mutate/node-batch.mjs +40 -3
- package/dist/mutate/node-coverage.mjs +7 -32
- package/dist/mutate/patch.js +2 -1
- package/dist/mutate/precise-lines.mjs +147 -31
- package/dist/mutate/probatio-jacoco-run.java +2 -2
- package/dist/mutate/project-config.js +58 -0
- package/dist/mutate/run.js +192 -32
- package/dist/mutate/rust-coverage.js +4 -19
- package/dist/mutate/sealed.js +94 -1
- package/dist/mutate/suite-decision.js +2 -1
- package/dist/mutate/suites.js +136 -163
- package/dist/mutate/tally.js +85 -20
- package/dist/node-version.js +13 -0
- package/dist/swarm/check-kill.js +14 -7
- package/dist/swarm/queue.js +13 -7
- package/dist/verify/change.js +5 -0
- package/package.json +6 -4
package/dist/mutate/run.js
CHANGED
|
@@ -12,7 +12,8 @@ import { writeJavaCoverage } from "./java-coverage.js";
|
|
|
12
12
|
import { writeRustCoverage } from "./rust-coverage.js";
|
|
13
13
|
import { applyPatch, changedLines, chooseDirection, filesInDiff, git, restoreTree } from "./patch.js";
|
|
14
14
|
import { killLabel } from "./kill-label.js";
|
|
15
|
-
import {
|
|
15
|
+
import { CONFIG_FILE, readProjectConfig } from "./project-config.js";
|
|
16
|
+
import { UNVIABLE_NEXT, TIMEOUT_FLOOR_MS, TIMEOUT_MULTIPLE, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST, WHOLE_SUITE_NEXT, campaignSummary, decideSuite, hasLineMap, noCoverageNext, resumeNext, scaleStop, timeoutLimitMs, } from "./suite-decision.js";
|
|
16
17
|
import { commandSuite, discoverSuite, nodeTestTimeoutMs, runDiscoveredSuite, sourceCandidates } from "./suites.js";
|
|
17
18
|
import { duplicateTitles } from "./titles.js";
|
|
18
19
|
export const reporterPath = fileURLToPath(new URL("./fail-reporter.mjs", import.meta.url));
|
|
@@ -41,7 +42,7 @@ export async function runMutants(options) {
|
|
|
41
42
|
return failReport(options.outDir, "", options.commit, "that commit does not resolve");
|
|
42
43
|
const commit = sha.stdout.trim();
|
|
43
44
|
mkdirSync(options.outDir, { recursive: true });
|
|
44
|
-
const run = loadRun(options
|
|
45
|
+
const run = loadRun(options, commit);
|
|
45
46
|
if ("error" in run)
|
|
46
47
|
return failReport(options.outDir, run.id, commit, run.error);
|
|
47
48
|
const listed = listPatches(options.patchDirs);
|
|
@@ -52,6 +53,9 @@ export async function runMutants(options) {
|
|
|
52
53
|
: listed.files;
|
|
53
54
|
if (patchFiles.length === 0)
|
|
54
55
|
return failReport(options.outDir, run.id, commit, "no patches");
|
|
56
|
+
const config = readProjectConfig(packageDir);
|
|
57
|
+
if ("error" in config)
|
|
58
|
+
return failReport(options.outDir, run.id, commit, config.error, undefined, `Fix ${CONFIG_FILE} and run the same command again.`);
|
|
55
59
|
const gate = resolveSuite(packageDir, options);
|
|
56
60
|
if (gate.action === "stop")
|
|
57
61
|
return failReport(options.outDir, run.id, commit, gate.summary, undefined, gate.next, gate.suiteCommand);
|
|
@@ -67,6 +71,13 @@ export async function runMutants(options) {
|
|
|
67
71
|
if ("error" in worktrees)
|
|
68
72
|
return failReport(options.outDir, run.id, commit, worktrees.error);
|
|
69
73
|
hidePaths(worktrees.dirs, repo, packageDir, options.hide ?? []);
|
|
74
|
+
// A cancelled run (MCP cancel, Ctrl-C) still removes its worktrees from the user's repo.
|
|
75
|
+
const onSignal = (signal) => {
|
|
76
|
+
removeWorktrees(repo, worktrees.dirs, options.outDir);
|
|
77
|
+
process.exit(signal === "SIGINT" ? 130 : 143);
|
|
78
|
+
};
|
|
79
|
+
process.once("SIGTERM", onSignal);
|
|
80
|
+
process.once("SIGINT", onSignal);
|
|
70
81
|
const budgetMs = options.budgetMs ?? (options.maxMinutes == null ? null : options.maxMinutes * 60_000);
|
|
71
82
|
let budgetStarted = 0;
|
|
72
83
|
const overBudget = () => budgetMs !== null && budgetStarted !== 0 && Date.now() - budgetStarted > budgetMs;
|
|
@@ -113,7 +124,7 @@ export async function runMutants(options) {
|
|
|
113
124
|
const prior = history.get(id);
|
|
114
125
|
if (prior && reusable(prior, raw, treePkg)) {
|
|
115
126
|
startedMutants += 1;
|
|
116
|
-
const reused = reusedResult(prior, commit, run.id, options.agent);
|
|
127
|
+
const reused = normalizeResult(reusedResult(prior, commit, run.id, options.agent));
|
|
117
128
|
writeResult(options.outDir, reused);
|
|
118
129
|
progress(`reused ${id}`);
|
|
119
130
|
continue;
|
|
@@ -165,6 +176,8 @@ export async function runMutants(options) {
|
|
|
165
176
|
});
|
|
166
177
|
}
|
|
167
178
|
finally {
|
|
179
|
+
process.removeListener("SIGTERM", onSignal);
|
|
180
|
+
process.removeListener("SIGINT", onSignal);
|
|
168
181
|
removeWorktrees(repo, worktrees.dirs, options.outDir);
|
|
169
182
|
}
|
|
170
183
|
}
|
|
@@ -211,7 +224,7 @@ async function prepare(pkg, options, progress) {
|
|
|
211
224
|
if (baseline.compileToken)
|
|
212
225
|
return { ok: false, error: `baseline build failed (${baseline.compileToken})` };
|
|
213
226
|
if (!baseline.report)
|
|
214
|
-
return { ok: false, error: baselineReportError(baseline.detail) };
|
|
227
|
+
return { ok: false, error: baselineReportError(baseline.detail, baseline.command || spec.command) };
|
|
215
228
|
baselineFailed = baseline.report.failed.map((item) => item.name);
|
|
216
229
|
if (collect && !existsSync(mapPath)) {
|
|
217
230
|
const binary = binaryFromCommand(baseline.command || spec.command, pkg);
|
|
@@ -224,6 +237,10 @@ async function prepare(pkg, options, progress) {
|
|
|
224
237
|
progress(`suite report tests=${baseline.report.tests}`);
|
|
225
238
|
progress(baselineFailed.length > 0 ? `baseline failing ${baselineFailed.length}` : "baseline pass");
|
|
226
239
|
if (baselineFailed.length > 0) {
|
|
240
|
+
const tool = missingToolFrom(baseline.detail, baseline.command || spec.command);
|
|
241
|
+
const unnamed = baselineFailed.every((name) => name === "command" || name === "::command");
|
|
242
|
+
if (tool && unnamed)
|
|
243
|
+
return { ok: false, error: `missing tool: ${tool}. baseline suite did not return a test report` };
|
|
227
244
|
return { ok: false, error: "baseline already failing", baselineFailed, suiteCommand: spec.command };
|
|
228
245
|
}
|
|
229
246
|
if (collect && !existsSync(mapPath) && languageMapKind(spec.kind)) {
|
|
@@ -250,12 +267,13 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
250
267
|
const raw = readFileSync(patchPath, "utf8");
|
|
251
268
|
const files = filesInDiff(raw);
|
|
252
269
|
const edited = changedLines(raw);
|
|
270
|
+
const located = edited.length > 0 ? edited : files;
|
|
253
271
|
const mutantHash = sha256(raw);
|
|
254
272
|
const base = () => ({
|
|
255
273
|
id,
|
|
256
274
|
outcome: "error",
|
|
257
275
|
direction: null,
|
|
258
|
-
files,
|
|
276
|
+
files: located,
|
|
259
277
|
killedBy: [],
|
|
260
278
|
cause: null,
|
|
261
279
|
next: "",
|
|
@@ -287,10 +305,12 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
287
305
|
return { ...base(), direction, outcome: "timeout", error: "build timed out", next: TIMEOUT_NEXT };
|
|
288
306
|
// The baseline build already succeeded. A mutant that no longer builds cannot pass the suite.
|
|
289
307
|
if (built.code !== 0)
|
|
290
|
-
return finishMutant(
|
|
308
|
+
return finishMutant(unviable(base, direction, "build"), pkg, "", false, []);
|
|
291
309
|
}
|
|
292
310
|
const picked = selectionFor(prep.coverage, edited.length > 0 ? edited : files);
|
|
293
|
-
const forced = options.onlyNames && options.onlyNames.length > 0
|
|
311
|
+
const forced = options.onlyNames && options.onlyNames.length > 0
|
|
312
|
+
? prep.spec.kind === "node" ? options.onlyNames.map(nodeTitle) : options.onlyNames
|
|
313
|
+
: null;
|
|
294
314
|
if (!forced && picked.state === "uncovered") {
|
|
295
315
|
return {
|
|
296
316
|
...base(),
|
|
@@ -318,16 +338,34 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
318
338
|
}
|
|
319
339
|
// A compile error can leave the previous run's report on disk. That report is not this mutant.
|
|
320
340
|
if (main.compileToken)
|
|
321
|
-
return finishMutant(
|
|
322
|
-
if (!main.report)
|
|
323
|
-
|
|
341
|
+
return finishMutant(unviable(base, direction, main.compileToken), pkg, commands.join(" | "), false, []);
|
|
342
|
+
if (!main.report) {
|
|
343
|
+
const detail = main.detail ? clip(main.detail) : "";
|
|
344
|
+
const tool = missingToolFrom(detail, main.command);
|
|
345
|
+
const error = tool
|
|
346
|
+
? `missing tool: ${tool}`
|
|
347
|
+
: detail
|
|
348
|
+
? `test reporter returned nothing (${detail})`
|
|
349
|
+
: "test reporter returned nothing";
|
|
350
|
+
return { ...base(), direction, error, command: commands.join(" | ") };
|
|
351
|
+
}
|
|
352
|
+
// A node name pattern that matches nothing passes the file. That is not a survivor.
|
|
353
|
+
// A requested describe block reports its leaf tests, so any test that ran counts.
|
|
354
|
+
if (forced && prep.spec.kind === "node" && main.report.names.every((name) => isNodeTestFile(name))) {
|
|
355
|
+
return { ...base(), direction, error: `no requested test ran (${forced.join(", ")})`, command: commands.join(" | ") };
|
|
356
|
+
}
|
|
324
357
|
const whole = main.wholeSuite || forceWhole;
|
|
325
358
|
const chosen = main.selectedTests;
|
|
326
359
|
if (main.report.failed.length === 0)
|
|
327
360
|
return finishMutant({ ...base(), direction, outcome: "survived" }, pkg, commands.join(" | "), whole, chosen);
|
|
361
|
+
// A test file that no longer links ran no test. It is the build failing, the same as javac rejecting it.
|
|
362
|
+
// Only a failure where code ran can kill. When none did, the mutant did not build.
|
|
363
|
+
const ran = main.report.failed.filter((item) => !item.staticLoad);
|
|
364
|
+
if (ran.length === 0)
|
|
365
|
+
return finishMutant(unviable(base, direction, "compiler"), pkg, commands.join(" | "), false, []);
|
|
328
366
|
if (!options.confirm)
|
|
329
|
-
return finishMutant(killed(base, direction,
|
|
330
|
-
const confirmed = await confirmFailures(pkg, options,
|
|
367
|
+
return finishMutant(killed(base, direction, ran.map((item) => failureId(pkg, item))), pkg, commands.join(" | "), whole, chosen);
|
|
368
|
+
const confirmed = await confirmFailures(pkg, options, ran, prep, limit, commands);
|
|
331
369
|
const killedBy = confirmed.killed.map((item) => failureId(pkg, item));
|
|
332
370
|
const flaky = confirmed.flaky.map((item) => failureId(pkg, item));
|
|
333
371
|
const outcome = killedBy.length > 0 ? "killed" : "flaky";
|
|
@@ -355,16 +393,31 @@ function killed(base, direction, killedBy) {
|
|
|
355
393
|
const label = killLabel(killedBy);
|
|
356
394
|
return { ...base(), direction, outcome: "killed", killedBy, cause: label?.cause ?? "test", next: label?.next ?? "" };
|
|
357
395
|
}
|
|
396
|
+
/** The compiler rejected the mutant. `killedBy` names the compiler so the record says which one. */
|
|
397
|
+
function unviable(base, direction, compiler) {
|
|
398
|
+
return { ...base(), direction, outcome: "unviable", killedBy: [compiler], cause: "build", next: UNVIABLE_NEXT };
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Results written before `unviable` existed stored a compile failure as killed with cause build.
|
|
402
|
+
* Read them the new way so an old out dir does not count a compiler as a test.
|
|
403
|
+
*/
|
|
404
|
+
export function normalizeResult(result) {
|
|
405
|
+
if (result.outcome === "killed" && result.cause === "build")
|
|
406
|
+
return { ...result, outcome: "unviable", next: UNVIABLE_NEXT };
|
|
407
|
+
return result;
|
|
408
|
+
}
|
|
358
409
|
async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands) {
|
|
359
410
|
const counts = new Map();
|
|
360
411
|
let wholeSuite = false;
|
|
361
412
|
const named = new Map();
|
|
362
413
|
const hungFiles = new Map();
|
|
414
|
+
const unloaded = new Map();
|
|
363
415
|
for (const failure of failed) {
|
|
364
416
|
const key = failure.file || "*";
|
|
365
417
|
// A name filter skips the tests that left the handle open, so the file
|
|
366
418
|
// timeout never comes back. Rerun that file with no name pattern.
|
|
367
|
-
|
|
419
|
+
// A file that did not load has no test name to filter by either.
|
|
420
|
+
const bucket = failure.fileTimeout ? hungFiles : failure.fileLoad ? unloaded : named;
|
|
368
421
|
const list = bucket.get(key) ?? [];
|
|
369
422
|
list.push(failure);
|
|
370
423
|
bucket.set(key, list);
|
|
@@ -378,13 +431,17 @@ async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands)
|
|
|
378
431
|
if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands))
|
|
379
432
|
wholeSuite = true;
|
|
380
433
|
}
|
|
434
|
+
for (const [file, failures] of unloaded) {
|
|
435
|
+
if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands, "load"))
|
|
436
|
+
wholeSuite = true;
|
|
437
|
+
}
|
|
381
438
|
const killed = [];
|
|
382
439
|
const flaky = [];
|
|
383
440
|
for (const failure of failed)
|
|
384
441
|
(counts.get(failureId(pkg, failure)) === 2 ? killed : flaky).push(failure);
|
|
385
442
|
return { killed, flaky, wholeSuite };
|
|
386
443
|
}
|
|
387
|
-
async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands) {
|
|
444
|
+
async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands, fileFailure = "timeout") {
|
|
388
445
|
const files = file === "*" ? listTestFiles(pkg, options.testsDir, prep.spec) : [testArg(pkg, file, options.testsDir)];
|
|
389
446
|
const isolated = { ...options, concurrency: 1 };
|
|
390
447
|
let wholeSuite = false;
|
|
@@ -396,20 +453,23 @@ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, t
|
|
|
396
453
|
wholeSuite = true;
|
|
397
454
|
if (rerun.command)
|
|
398
455
|
commands.push(rerun.command);
|
|
399
|
-
if (pattern === null && rerun.timedOut) {
|
|
400
|
-
for (const
|
|
401
|
-
bump(counts,
|
|
456
|
+
if (pattern === null && fileFailure === "timeout" && rerun.timedOut) {
|
|
457
|
+
for (const id of new Set(failures.map((item) => failureId(pkg, item))))
|
|
458
|
+
bump(counts, id);
|
|
402
459
|
continue;
|
|
403
460
|
}
|
|
404
461
|
if (!rerun.report)
|
|
405
462
|
continue;
|
|
463
|
+
const seen = new Set();
|
|
406
464
|
for (const hit of rerun.report.failed) {
|
|
407
|
-
if (pattern === null && !hit.fileTimeout)
|
|
465
|
+
if (pattern === null && !(fileFailure === "timeout" ? hit.fileTimeout : hit.fileLoad))
|
|
408
466
|
continue;
|
|
409
467
|
const id = failureId(pkg, hit);
|
|
410
468
|
if (failures.some((item) => failureId(pkg, item) === id))
|
|
411
|
-
|
|
469
|
+
seen.add(id);
|
|
412
470
|
}
|
|
471
|
+
for (const id of seen)
|
|
472
|
+
bump(counts, id);
|
|
413
473
|
}
|
|
414
474
|
return wholeSuite;
|
|
415
475
|
}
|
|
@@ -536,7 +596,7 @@ class NodeBatch {
|
|
|
536
596
|
return {
|
|
537
597
|
report: null,
|
|
538
598
|
timedOut: false,
|
|
539
|
-
detail: waited.error || "test reporter returned nothing",
|
|
599
|
+
detail: waited.error || (selected.length > 0 ? `no requested test ran (${selected.join(", ")})` : "test reporter returned nothing"),
|
|
540
600
|
compileToken: null,
|
|
541
601
|
command: waited.command || "",
|
|
542
602
|
wholeSuite: false,
|
|
@@ -599,6 +659,11 @@ function isNodeTestFile(file) {
|
|
|
599
659
|
const base = path.posix.basename(file);
|
|
600
660
|
return /\.(test|spec)\.(ts|tsx|mts|js|mjs|cjs)$/.test(base) || /_test\.(ts|tsx|js|mjs)$/.test(base);
|
|
601
661
|
}
|
|
662
|
+
/** A node keep id may carry its file (`tests/gate.test.ts::zero stays shut`). node:test matches the title. */
|
|
663
|
+
export function nodeTitle(id) {
|
|
664
|
+
const match = /^[^\s:][^:]*\.(?:[cm]?[jt]s|[jt]sx)::(.+)$/.exec(id);
|
|
665
|
+
return match ? match[1] : id;
|
|
666
|
+
}
|
|
602
667
|
function narrowNodeFiles(pkg, files, names, kind) {
|
|
603
668
|
if (kind !== "node" || !names || names.length === 0)
|
|
604
669
|
return files;
|
|
@@ -636,6 +701,18 @@ function childEnv(unset) {
|
|
|
636
701
|
if (key === "NODE_TEST_CONTEXT" || key.startsWith("NODE_TEST_"))
|
|
637
702
|
delete env[key];
|
|
638
703
|
}
|
|
704
|
+
// This run may itself be a child of a suite that another Probatio run is mapping.
|
|
705
|
+
// That outer map is not this run's. Only the baseline sets a map, through runDiscoveredSuite.
|
|
706
|
+
delete env.PROBATIO_COVERAGE_MAP;
|
|
707
|
+
delete env.PROBATIO_PACKAGE_ROOT;
|
|
708
|
+
delete env.PROBATIO_CHILD_DIR;
|
|
709
|
+
if (env.NODE_OPTIONS) {
|
|
710
|
+
const kept = env.NODE_OPTIONS.split(/\s+/).filter((part) => part.length > 0 && !/child-lines\.mjs$/.test(part)).join(" ");
|
|
711
|
+
if (kept)
|
|
712
|
+
env.NODE_OPTIONS = kept;
|
|
713
|
+
else
|
|
714
|
+
delete env.NODE_OPTIONS;
|
|
715
|
+
}
|
|
639
716
|
for (const key of unset)
|
|
640
717
|
delete env[key];
|
|
641
718
|
return env;
|
|
@@ -708,8 +785,8 @@ function loadMeta(dirs) {
|
|
|
708
785
|
}
|
|
709
786
|
return meta;
|
|
710
787
|
}
|
|
711
|
-
function loadRun(
|
|
712
|
-
const file = path.join(outDir, "run.json");
|
|
788
|
+
function loadRun(options, commit) {
|
|
789
|
+
const file = path.join(options.outDir, "run.json");
|
|
713
790
|
if (existsSync(file)) {
|
|
714
791
|
const stored = JSON.parse(readFileSync(file, "utf8"));
|
|
715
792
|
if (stored.commit !== commit)
|
|
@@ -717,7 +794,24 @@ function loadRun(outDir, commit, agent) {
|
|
|
717
794
|
return { id: stored.id };
|
|
718
795
|
}
|
|
719
796
|
const id = randomBytes(4).toString("hex");
|
|
720
|
-
|
|
797
|
+
const record = {
|
|
798
|
+
id,
|
|
799
|
+
commit,
|
|
800
|
+
agent: options.agent,
|
|
801
|
+
started: new Date().toISOString(),
|
|
802
|
+
package: options.packageDir,
|
|
803
|
+
patches: options.patchDirs,
|
|
804
|
+
out: options.outDir,
|
|
805
|
+
repo: options.repoDir,
|
|
806
|
+
workers: options.workers,
|
|
807
|
+
confirm: options.confirm,
|
|
808
|
+
testsDir: options.testsDir,
|
|
809
|
+
suiteTimeoutMs: options.suiteTimeoutMs,
|
|
810
|
+
testTimeoutMs: options.testTimeoutMs,
|
|
811
|
+
direction: options.direction,
|
|
812
|
+
suiteCommand: options.suiteCommand ?? null,
|
|
813
|
+
};
|
|
814
|
+
writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`);
|
|
721
815
|
return { id };
|
|
722
816
|
}
|
|
723
817
|
async function createWorktrees(repo, packageDir, runId, commit, workers, outDir) {
|
|
@@ -725,6 +819,10 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
725
819
|
const dirs = [];
|
|
726
820
|
for (let index = 0; index < workers; index++) {
|
|
727
821
|
const dir = path.join(tmpdir(), `probatio-${runId}-${index}`);
|
|
822
|
+
if (!probatioWorktreeDir(dir)) {
|
|
823
|
+
removeWorktrees(repo, dirs, outDir);
|
|
824
|
+
return { error: "worktree path is outside the probatio temp prefix" };
|
|
825
|
+
}
|
|
728
826
|
rmSync(dir, { recursive: true, force: true });
|
|
729
827
|
const added = git(repo, ["worktree", "add", "--detach", dir, commit]);
|
|
730
828
|
if (added.status !== 0) {
|
|
@@ -735,8 +833,12 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
735
833
|
}
|
|
736
834
|
writeFileSync(path.join(outDir, "worktrees.json"), `${JSON.stringify(dirs)}\n`);
|
|
737
835
|
const sources = nodeModuleSources(repo, packageDir);
|
|
836
|
+
const config = path.join(packageDir, CONFIG_FILE);
|
|
738
837
|
for (const dir of dirs) {
|
|
739
838
|
const pkgRoot = packageIn(dir, repo, packageDir);
|
|
839
|
+
// How to run the suite is not the code under test. An uncommitted .probatio.json still applies.
|
|
840
|
+
if (existsSync(config) && !existsSync(path.join(pkgRoot, CONFIG_FILE)))
|
|
841
|
+
copyFileSync(config, path.join(pkgRoot, CONFIG_FILE));
|
|
740
842
|
for (const source of sources) {
|
|
741
843
|
const dest = path.join(pkgRoot, source.rel);
|
|
742
844
|
if (existsSync(dest))
|
|
@@ -750,15 +852,42 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
750
852
|
function hidePaths(dirs, repo, packageDir, hide) {
|
|
751
853
|
for (const dir of dirs) {
|
|
752
854
|
const pkg = packageIn(dir, repo, packageDir);
|
|
855
|
+
const root = path.resolve(pkg);
|
|
753
856
|
for (const rel of hide) {
|
|
754
|
-
if (!rel || path.isAbsolute(rel))
|
|
857
|
+
if (!rel || path.isAbsolute(rel) || rel.split(/[\\/]/).includes(".."))
|
|
858
|
+
continue;
|
|
859
|
+
const target = path.resolve(root, rel);
|
|
860
|
+
if (target !== root && !target.startsWith(root + path.sep))
|
|
755
861
|
continue;
|
|
756
|
-
rmSync(
|
|
862
|
+
rmSync(target, { force: true });
|
|
757
863
|
}
|
|
758
864
|
}
|
|
759
865
|
}
|
|
866
|
+
/** A worktree Probatio created: one directory under the OS temp dir, named probatio- plus hex and an index. */
|
|
867
|
+
export function probatioWorktreeDir(dir) {
|
|
868
|
+
let resolved = path.resolve(dir);
|
|
869
|
+
try {
|
|
870
|
+
resolved = realpathSync(dir);
|
|
871
|
+
}
|
|
872
|
+
catch {
|
|
873
|
+
// The directory is created on the next line. The unresolved path is still checked.
|
|
874
|
+
}
|
|
875
|
+
const roots = new Set([path.resolve(tmpdir())]);
|
|
876
|
+
try {
|
|
877
|
+
roots.add(realpathSync(tmpdir()));
|
|
878
|
+
}
|
|
879
|
+
catch {
|
|
880
|
+
// tmpdir() itself is enough when realpath is unavailable.
|
|
881
|
+
}
|
|
882
|
+
const inside = [...roots].some((root) => resolved === root || resolved.startsWith(root + path.sep));
|
|
883
|
+
if (!inside)
|
|
884
|
+
return false;
|
|
885
|
+
return /^probatio-[0-9a-f]+-\d+$/.test(path.basename(resolved));
|
|
886
|
+
}
|
|
760
887
|
function removeWorktrees(repo, dirs, outDir) {
|
|
761
888
|
for (const dir of dirs) {
|
|
889
|
+
if (!probatioWorktreeDir(dir))
|
|
890
|
+
continue;
|
|
762
891
|
git(repo, ["worktree", "remove", "--force", dir]);
|
|
763
892
|
rmSync(dir, { recursive: true, force: true });
|
|
764
893
|
}
|
|
@@ -868,11 +997,12 @@ function readResult(outDir, id) {
|
|
|
868
997
|
const file = resultPath(outDir, id);
|
|
869
998
|
if (!existsSync(file))
|
|
870
999
|
return null;
|
|
871
|
-
return JSON.parse(readFileSync(file, "utf8"));
|
|
1000
|
+
return normalizeResult(JSON.parse(readFileSync(file, "utf8")));
|
|
872
1001
|
}
|
|
873
1002
|
function reportOf(results, context) {
|
|
874
1003
|
const count = (outcome) => results.filter((result) => result.outcome === outcome).length;
|
|
875
1004
|
const killed = count("killed");
|
|
1005
|
+
const unviableCount = count("unviable");
|
|
876
1006
|
const survived = count("survived");
|
|
877
1007
|
const flaky = count("flaky");
|
|
878
1008
|
const timeouts = count("timeout");
|
|
@@ -886,7 +1016,7 @@ function reportOf(results, context) {
|
|
|
886
1016
|
return { id: result.id, file, line, ...(known?.op ? { op: known.op } : {}) };
|
|
887
1017
|
});
|
|
888
1018
|
const budgetHit = context.budgetHit || context.pendingLeft > 0;
|
|
889
|
-
const summary = campaignSummary({ noCoverage, survived, killed, flaky, timeouts, errors, finished: results.length });
|
|
1019
|
+
const summary = campaignSummary({ noCoverage, survived, killed, unviable: unviableCount, flaky, timeouts, errors, finished: results.length });
|
|
890
1020
|
const notes = [];
|
|
891
1021
|
const uncovered = results.find((result) => result.outcome === "no coverage");
|
|
892
1022
|
if (uncovered?.next)
|
|
@@ -899,6 +1029,9 @@ function reportOf(results, context) {
|
|
|
899
1029
|
notes.push(resumeNext(context.notStarted));
|
|
900
1030
|
if (results.some((result) => result.wholeSuite))
|
|
901
1031
|
notes.push(WHOLE_SUITE_NEXT);
|
|
1032
|
+
const broken = results.find((result) => (result.outcome === "error" || result.outcome === "build-failed") && result.error);
|
|
1033
|
+
if (broken?.error)
|
|
1034
|
+
notes.push(`${broken.id}: ${broken.error}`);
|
|
902
1035
|
if (notes.length === 0)
|
|
903
1036
|
notes.push("No survivor in this batch.");
|
|
904
1037
|
return {
|
|
@@ -909,6 +1042,8 @@ function reportOf(results, context) {
|
|
|
909
1042
|
commit: context.commit,
|
|
910
1043
|
runId: context.runId,
|
|
911
1044
|
killed,
|
|
1045
|
+
unviable: unviableCount,
|
|
1046
|
+
unviableIds: results.filter((result) => result.outcome === "unviable").slice(0, 10).map((result) => result.id),
|
|
912
1047
|
survived,
|
|
913
1048
|
flaky,
|
|
914
1049
|
timeouts,
|
|
@@ -968,6 +1103,8 @@ function failReport(outDir, runId, commit, error, baselineFailed, next, suiteCom
|
|
|
968
1103
|
commit,
|
|
969
1104
|
runId,
|
|
970
1105
|
killed: 0,
|
|
1106
|
+
unviable: 0,
|
|
1107
|
+
unviableIds: [],
|
|
971
1108
|
survived: 0,
|
|
972
1109
|
flaky: 0,
|
|
973
1110
|
timeouts: 0,
|
|
@@ -1132,10 +1269,19 @@ function failureId(pkg, failure) {
|
|
|
1132
1269
|
return BARE_SUITE.has(failure.name) ? `::${failure.name}` : failure.name;
|
|
1133
1270
|
if (!testFileOnDisk(pkg, failure.file))
|
|
1134
1271
|
return failure.name;
|
|
1135
|
-
|
|
1272
|
+
// macOS temp dirs are /var and /private/var. A report can name either, so compare real paths.
|
|
1273
|
+
const rel = path.relative(realOrSame(pkg), realOrSame(path.resolve(pkg, failure.file)));
|
|
1136
1274
|
const file = !rel || rel.startsWith("..") ? path.basename(failure.file) : rel.split(path.sep).join("/");
|
|
1137
1275
|
return `${file}::${failure.name}`;
|
|
1138
1276
|
}
|
|
1277
|
+
function realOrSame(file) {
|
|
1278
|
+
try {
|
|
1279
|
+
return realpathSync(file);
|
|
1280
|
+
}
|
|
1281
|
+
catch {
|
|
1282
|
+
return file;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1139
1285
|
function testFileOnDisk(pkg, file) {
|
|
1140
1286
|
if (path.isAbsolute(file))
|
|
1141
1287
|
return existsSync(file);
|
|
@@ -1156,13 +1302,27 @@ function testArg(pkg, file, testsDir) {
|
|
|
1156
1302
|
function escapeRegExp(value) {
|
|
1157
1303
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1158
1304
|
}
|
|
1159
|
-
function
|
|
1305
|
+
function missingToolFrom(detail, command) {
|
|
1306
|
+
const spawnMiss = /spawn ([^\s]+) ENOENT/.exec(detail);
|
|
1307
|
+
if (spawnMiss)
|
|
1308
|
+
return path.basename(spawnMiss[1]);
|
|
1309
|
+
const shellMiss = /(?:^|\s)([A-Za-z0-9_./+-]+): (?:command not found|not found)\b/.exec(detail);
|
|
1310
|
+
if (shellMiss)
|
|
1311
|
+
return path.basename(shellMiss[1]);
|
|
1312
|
+
if (/No such file or directory|ENOENT/.test(detail)) {
|
|
1313
|
+
const bin = command.trim().split(/\s+/)[0];
|
|
1314
|
+
if (bin)
|
|
1315
|
+
return path.basename(bin);
|
|
1316
|
+
}
|
|
1317
|
+
return null;
|
|
1318
|
+
}
|
|
1319
|
+
function baselineReportError(detail, command = "") {
|
|
1320
|
+
const tool = missingToolFrom(detail, command);
|
|
1160
1321
|
const line = detail.trim();
|
|
1161
1322
|
// The first line of a Python or Node crash is a stack fragment. The summary
|
|
1162
1323
|
// states the limit and does not carry that fragment.
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
return `baseline suite did not return a test report (${line})`;
|
|
1324
|
+
const base = !line || isStackFragment(line) ? "baseline suite did not return a test report" : `baseline suite did not return a test report (${line})`;
|
|
1325
|
+
return tool ? `missing tool: ${tool}. ${base}` : base;
|
|
1166
1326
|
}
|
|
1167
1327
|
function isStackFragment(line) {
|
|
1168
1328
|
return line.startsWith("Traceback") || /^\s+at \S/.test(line) || /File ".+", line \d+/.test(line);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { existsSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { exportLlvmProfile, llvmToolsPresent } from "./coverage-map.js";
|
|
4
5
|
/**
|
|
5
6
|
* One instrumented test binary, then one profile per test name.
|
|
6
7
|
* A single cargo test profile does not say which test hit the line.
|
|
@@ -9,9 +10,7 @@ export function writeRustCoverage(pkg, testNames, dest) {
|
|
|
9
10
|
const names = [...new Set(testNames.map((name) => name.trim()).filter((name) => name.length > 0 && !/\s/.test(name)))];
|
|
10
11
|
if (names.length === 0)
|
|
11
12
|
return { ok: false, detail: "rust coverage: no test names" };
|
|
12
|
-
|
|
13
|
-
const llvmProf = findTool("llvm-profdata");
|
|
14
|
-
if (!llvmCov || !llvmProf)
|
|
13
|
+
if (!llvmToolsPresent())
|
|
15
14
|
return { ok: false, detail: "rust coverage: llvm tool missing" };
|
|
16
15
|
const env = { ...process.env, RUSTFLAGS: "-C instrument-coverage", CARGO_INCREMENTAL: "0" };
|
|
17
16
|
const built = spawnSync("cargo", ["test", "--no-run", "--message-format=json"], {
|
|
@@ -43,11 +42,8 @@ export function writeRustCoverage(pkg, testNames, dest) {
|
|
|
43
42
|
if (!new RegExp(`^test ${escapeRegExp(name)} \\.\\.\\. ok`, "m").test(`${ran.stdout}\n${ran.stderr}`))
|
|
44
43
|
continue;
|
|
45
44
|
const merged = path.join(work, `${safe(name)}-${safe(path.basename(exe))}.profdata`);
|
|
46
|
-
const
|
|
47
|
-
if (
|
|
48
|
-
continue;
|
|
49
|
-
const exported = spawnSync(llvmCov, ["export", exe, `-instr-profile=${merged}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
|
50
|
-
if (exported.status !== 0)
|
|
45
|
+
const exported = exportLlvmProfile(exe, [raw], merged);
|
|
46
|
+
if (!exported.ok)
|
|
51
47
|
continue;
|
|
52
48
|
for (const hit of llvmHits(exported.stdout, pkg))
|
|
53
49
|
addHit(files, hit.file, hit.line, name);
|
|
@@ -136,17 +132,6 @@ function addHit(files, file, line, test) {
|
|
|
136
132
|
lines[key] = names;
|
|
137
133
|
files[file] = lines;
|
|
138
134
|
}
|
|
139
|
-
function findTool(name) {
|
|
140
|
-
const fromXcrun = spawnSync("xcrun", ["--find", name], { encoding: "utf8" });
|
|
141
|
-
const xcrunPath = fromXcrun.stdout?.trim() ?? "";
|
|
142
|
-
if (fromXcrun.status === 0 && xcrunPath && existsSync(xcrunPath))
|
|
143
|
-
return xcrunPath;
|
|
144
|
-
const which = spawnSync("sh", ["-c", `command -v ${name}`], { encoding: "utf8" });
|
|
145
|
-
const found = which.stdout?.trim() ?? "";
|
|
146
|
-
if (which.status === 0 && found)
|
|
147
|
-
return found;
|
|
148
|
-
return null;
|
|
149
|
-
}
|
|
150
135
|
function safe(value) {
|
|
151
136
|
return value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
152
137
|
}
|
package/dist/mutate/sealed.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
2
4
|
/** The label stays in its own file. The run report must not repeat these three values. */
|
|
3
5
|
export function readSealedLabel(file) {
|
|
4
6
|
let text = "";
|
|
@@ -18,8 +20,99 @@ export function readSealedLabel(file) {
|
|
|
18
20
|
export function labelLeaked(body, label) {
|
|
19
21
|
return body.includes(label.bugId) || body.includes(label.issueId) || body.includes(label.expectedFail);
|
|
20
22
|
}
|
|
23
|
+
/** The envelope can be clean while a result file on disk still repeats the label. */
|
|
24
|
+
export function sealedTreeLeaked(dir, label) {
|
|
25
|
+
const visit = (current) => {
|
|
26
|
+
let names;
|
|
27
|
+
try {
|
|
28
|
+
names = readdirSync(current);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
for (const name of names) {
|
|
34
|
+
const full = path.join(current, name);
|
|
35
|
+
let info;
|
|
36
|
+
try {
|
|
37
|
+
info = statSync(full);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (info.isDirectory()) {
|
|
43
|
+
if (visit(full))
|
|
44
|
+
return true;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (!info.isFile() || info.size > 2_000_000)
|
|
48
|
+
continue;
|
|
49
|
+
try {
|
|
50
|
+
if (labelLeaked(readFileSync(full, "utf8"), label))
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// A binary artifact is not a report.
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return false;
|
|
58
|
+
};
|
|
59
|
+
return visit(dir);
|
|
60
|
+
}
|
|
21
61
|
export const LABEL_LEAK_SUMMARY = "The label was copied into the report.";
|
|
22
62
|
function field(text, key) {
|
|
23
63
|
const match = text.match(new RegExp(`^${key}=(.*)$`, "m"));
|
|
24
64
|
return match?.[1]?.trim() ?? "";
|
|
25
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* A sealed run hides the revealing test, but a fix commit often edits an older test too.
|
|
68
|
+
* A kill by that edited test is the fix's own test, not an older test that caught the bug.
|
|
69
|
+
* Returns the edited test files that killed, and whether any kill came from a test the fix left alone.
|
|
70
|
+
*/
|
|
71
|
+
export function fixEditedKillers(repo, packageDir, commit, hide, kills) {
|
|
72
|
+
if (kills.length === 0)
|
|
73
|
+
return { files: [], olderTestCatch: false };
|
|
74
|
+
const diff = spawnSync("git", ["-C", repo, "diff", "--name-only", `${commit}^`, commit], { encoding: "utf8" });
|
|
75
|
+
// No parent, or git failed: nothing is known to be edited, so every kill stays an older-test catch.
|
|
76
|
+
if (diff.status !== 0)
|
|
77
|
+
return { files: [], olderTestCatch: true };
|
|
78
|
+
const hidden = new Set(hide.map((rel) => rel.split(path.sep).join("/")));
|
|
79
|
+
const edited = diff.stdout
|
|
80
|
+
.split("\n")
|
|
81
|
+
.filter((rel) => rel.length > 0)
|
|
82
|
+
.map((rel) => ({ repoRel: rel, pkgRel: path.relative(packageDir, path.join(repo, rel)).split(path.sep).join("/") }))
|
|
83
|
+
.filter((item) => !item.pkgRel.startsWith("..") && !hidden.has(item.pkgRel));
|
|
84
|
+
const text = new Map();
|
|
85
|
+
const contents = (repoRel) => {
|
|
86
|
+
if (!text.has(repoRel)) {
|
|
87
|
+
const shown = spawnSync("git", ["-C", repo, "show", `${commit}:${repoRel}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
|
88
|
+
text.set(repoRel, shown.status === 0 ? shown.stdout : "");
|
|
89
|
+
}
|
|
90
|
+
return text.get(repoRel) ?? "";
|
|
91
|
+
};
|
|
92
|
+
const fileFor = (id) => edited.find((item) => killerInFile(id, item.pkgRel, () => contents(item.repoRel)));
|
|
93
|
+
const files = new Set();
|
|
94
|
+
let olderTestCatch = false;
|
|
95
|
+
for (const kill of kills) {
|
|
96
|
+
const owners = kill.killedBy.map(fileFor);
|
|
97
|
+
for (const owner of owners)
|
|
98
|
+
if (owner)
|
|
99
|
+
files.add(owner.pkgRel);
|
|
100
|
+
if (owners.some((owner) => !owner))
|
|
101
|
+
olderTestCatch = true;
|
|
102
|
+
}
|
|
103
|
+
return { files: [...files].sort(), olderTestCatch };
|
|
104
|
+
}
|
|
105
|
+
/** True when a kill id names this file, its class, or a test written in it. */
|
|
106
|
+
function killerInFile(id, rel, read) {
|
|
107
|
+
if (id.includes(rel) || id.startsWith(`${path.posix.basename(rel)}::`))
|
|
108
|
+
return true;
|
|
109
|
+
const stem = path.posix.basename(rel).replace(/\.[^.]+$/, "");
|
|
110
|
+
// JUnit `pkg.ClassTest.method` or `ClassTest#method`, Go and Rust paths.
|
|
111
|
+
if (stem.length > 0 && id.split(/[.#:/]+/).includes(stem))
|
|
112
|
+
return true;
|
|
113
|
+
const leaf = id.split(/::|#/).pop()?.split(/\.(?=[^.]+$)/).pop()?.replace(/\[.*\]$/, "").trim() ?? "";
|
|
114
|
+
if (leaf.length < 4)
|
|
115
|
+
return false;
|
|
116
|
+
const body = read();
|
|
117
|
+
return [`"${leaf}"`, `'${leaf}'`, `\`${leaf}\``, `${leaf}(`, `def ${leaf}`, `func ${leaf}`, `fn ${leaf}`].some((needle) => body.includes(needle));
|
|
118
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/** Pure suite decisions. The process spawn stays in the runner. */
|
|
2
2
|
export const UNKNOWN_NEXT = "The suite command is unknown. Pass --suite-command with the command that runs this package's tests.";
|
|
3
3
|
export const TIMEOUT_NEXT = "Timed out. The mutant ran longer than the baseline allows.";
|
|
4
|
+
export const UNVIABLE_NEXT = "The mutant did not build. No test saw it, so it is not a kill and not a gap. No test is needed.";
|
|
4
5
|
export const WHOLE_SUITE_NEXT = "The suite command cannot take a test name, so the whole suite ran.";
|
|
5
6
|
export const TIMEOUT_MULTIPLE = 5;
|
|
6
7
|
/** cargo-mutants uses 20s. A fixture can pass a smaller positive floor. */
|
|
@@ -22,7 +23,7 @@ export function leadSummary(noCoverage, survivors, survivorWord, rest) {
|
|
|
22
23
|
return `${noCoverage} no coverage, ${survivors} ${survivorWord}, ${rest}`;
|
|
23
24
|
}
|
|
24
25
|
export function campaignSummary(counts) {
|
|
25
|
-
return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
|
|
26
|
+
return leadSummary(counts.noCoverage, counts.survived, "survived", `${counts.killed} killed, ${counts.unviable ? `${counts.unviable} did not build, ` : ""}${counts.flaky} flaky, ${counts.timeouts} timed out, ${counts.errors} errored, of ${counts.finished} finished.`);
|
|
26
27
|
}
|
|
27
28
|
export function timeoutLimitMs(baselineMs, multiple, floorMs) {
|
|
28
29
|
const floor = Math.max(1, floorMs);
|