probatio 0.1.0 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -14
- package/dist/cli.js +101 -53
- package/dist/contract.js +7 -5
- package/dist/mcp.js +23 -0
- package/dist/mutate/child-lines.mjs +34 -0
- package/dist/mutate/coverage-map.js +75 -18
- package/dist/mutate/generate.js +25 -4
- package/dist/mutate/mocha-coverage.cjs +71 -0
- package/dist/mutate/node-batch-loader.mjs +85 -0
- package/dist/mutate/node-batch.mjs +130 -0
- package/dist/mutate/node-coverage.mjs +37 -180
- package/dist/mutate/precise-lines.mjs +188 -0
- package/dist/mutate/run.js +319 -47
- package/dist/mutate/rust-coverage.js +4 -19
- package/dist/mutate/sealed.js +40 -1
- package/dist/mutate/suite-decision.js +1 -1
- package/dist/mutate/suites.js +88 -12
- package/dist/mutate/tally.js +38 -1
- package/dist/node-version.js +13 -0
- package/dist/swarm/queue.js +13 -7
- package/dist/verify/change.js +71 -2
- package/package.json +6 -4
package/dist/mutate/run.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
-
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { appendFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
@@ -13,7 +13,7 @@ import { writeRustCoverage } from "./rust-coverage.js";
|
|
|
13
13
|
import { applyPatch, changedLines, chooseDirection, filesInDiff, git, restoreTree } from "./patch.js";
|
|
14
14
|
import { killLabel } from "./kill-label.js";
|
|
15
15
|
import { TIMEOUT_FLOOR_MS, TIMEOUT_MULTIPLE, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST, WHOLE_SUITE_NEXT, campaignSummary, decideSuite, hasLineMap, noCoverageNext, resumeNext, scaleStop, timeoutLimitMs, } from "./suite-decision.js";
|
|
16
|
-
import { commandSuite, discoverSuite, runDiscoveredSuite, sourceCandidates } from "./suites.js";
|
|
16
|
+
import { commandSuite, discoverSuite, nodeTestTimeoutMs, runDiscoveredSuite, sourceCandidates } from "./suites.js";
|
|
17
17
|
import { duplicateTitles } from "./titles.js";
|
|
18
18
|
export const reporterPath = fileURLToPath(new URL("./fail-reporter.mjs", import.meta.url));
|
|
19
19
|
export function parseReport(stdout) {
|
|
@@ -41,7 +41,7 @@ export async function runMutants(options) {
|
|
|
41
41
|
return failReport(options.outDir, "", options.commit, "that commit does not resolve");
|
|
42
42
|
const commit = sha.stdout.trim();
|
|
43
43
|
mkdirSync(options.outDir, { recursive: true });
|
|
44
|
-
const run = loadRun(options
|
|
44
|
+
const run = loadRun(options, commit);
|
|
45
45
|
if ("error" in run)
|
|
46
46
|
return failReport(options.outDir, run.id, commit, run.error);
|
|
47
47
|
const listed = listPatches(options.patchDirs);
|
|
@@ -89,34 +89,61 @@ export async function runMutants(options) {
|
|
|
89
89
|
budgetStarted = Date.now();
|
|
90
90
|
const worker = async (worktree) => {
|
|
91
91
|
const treePkg = packageIn(worktree, repo, packageDir);
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
92
|
+
let batch = null;
|
|
93
|
+
const dropBatch = () => {
|
|
94
|
+
batch?.stop();
|
|
95
|
+
batch = null;
|
|
96
|
+
};
|
|
97
|
+
try {
|
|
98
|
+
for (;;) {
|
|
99
|
+
if (startedMutants > 0 && overBudget()) {
|
|
100
|
+
budgetHit = true;
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
const patch = queue.shift();
|
|
104
|
+
if (!patch)
|
|
105
|
+
return;
|
|
106
|
+
if (startedMutants > 0 && overBudget()) {
|
|
107
|
+
queue.unshift(patch);
|
|
108
|
+
budgetHit = true;
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const id = path.basename(patch, ".patch");
|
|
112
|
+
const raw = readFileSync(patch, "utf8");
|
|
113
|
+
const prior = history.get(id);
|
|
114
|
+
if (prior && reusable(prior, raw, treePkg)) {
|
|
115
|
+
startedMutants += 1;
|
|
116
|
+
const reused = reusedResult(prior, commit, run.id, options.agent);
|
|
117
|
+
writeResult(options.outDir, reused);
|
|
118
|
+
progress(`reused ${id}`);
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
109
121
|
startedMutants += 1;
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
122
|
+
progress(`${id} start`);
|
|
123
|
+
const plan = batchPlan(raw, prep, options);
|
|
124
|
+
if (plan) {
|
|
125
|
+
if (!batch || batch.dead) {
|
|
126
|
+
dropBatch();
|
|
127
|
+
try {
|
|
128
|
+
batch = await NodeBatch.start(treePkg, options, options.outDir);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
batch = null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const launch = plan && batch
|
|
136
|
+
? (pkg, files, names, timeoutMs) => batch.run(pkg, files, names, timeoutMs, id, options.outDir)
|
|
137
|
+
: undefined;
|
|
138
|
+
const result = await runOne(worktree, treePkg, patch, options, commit, run.id, prep, launch);
|
|
139
|
+
writeResult(options.outDir, result);
|
|
140
|
+
if (batch && (batch.dead || result.error === "suite process crashed"))
|
|
141
|
+
dropBatch();
|
|
142
|
+
progress(`${id} ${result.outcome}`);
|
|
114
143
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
writeResult(options.outDir, result);
|
|
119
|
-
progress(`${id} ${result.outcome}`);
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
dropBatch();
|
|
120
147
|
}
|
|
121
148
|
};
|
|
122
149
|
await Promise.all(worktrees.dirs.map((dir) => worker(dir)));
|
|
@@ -184,7 +211,7 @@ async function prepare(pkg, options, progress) {
|
|
|
184
211
|
if (baseline.compileToken)
|
|
185
212
|
return { ok: false, error: `baseline build failed (${baseline.compileToken})` };
|
|
186
213
|
if (!baseline.report)
|
|
187
|
-
return { ok: false, error: baselineReportError(baseline.detail) };
|
|
214
|
+
return { ok: false, error: baselineReportError(baseline.detail, baseline.command || spec.command) };
|
|
188
215
|
baselineFailed = baseline.report.failed.map((item) => item.name);
|
|
189
216
|
if (collect && !existsSync(mapPath)) {
|
|
190
217
|
const binary = binaryFromCommand(baseline.command || spec.command, pkg);
|
|
@@ -197,6 +224,10 @@ async function prepare(pkg, options, progress) {
|
|
|
197
224
|
progress(`suite report tests=${baseline.report.tests}`);
|
|
198
225
|
progress(baselineFailed.length > 0 ? `baseline failing ${baselineFailed.length}` : "baseline pass");
|
|
199
226
|
if (baselineFailed.length > 0) {
|
|
227
|
+
const tool = missingToolFrom(baseline.detail, baseline.command || spec.command);
|
|
228
|
+
const unnamed = baselineFailed.every((name) => name === "command" || name === "::command");
|
|
229
|
+
if (tool && unnamed)
|
|
230
|
+
return { ok: false, error: `missing tool: ${tool}. baseline suite did not return a test report` };
|
|
200
231
|
return { ok: false, error: "baseline already failing", baselineFailed, suiteCommand: spec.command };
|
|
201
232
|
}
|
|
202
233
|
if (collect && !existsSync(mapPath) && languageMapKind(spec.kind)) {
|
|
@@ -217,18 +248,19 @@ async function prepare(pkg, options, progress) {
|
|
|
217
248
|
const affectedMap = options.affected ? affectedTests(pkg, { testsDir: options.testsDir }) : null;
|
|
218
249
|
return { ok: true, tests, affectedMap, baselineMs, spec, coverage: readCoverageMap(mapPath) };
|
|
219
250
|
}
|
|
220
|
-
async function runOne(worktree, pkg, patchPath, options, commit, runId, prep) {
|
|
251
|
+
async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, launch) {
|
|
221
252
|
const id = path.basename(patchPath, ".patch");
|
|
222
253
|
const started = Date.now();
|
|
223
254
|
const raw = readFileSync(patchPath, "utf8");
|
|
224
255
|
const files = filesInDiff(raw);
|
|
225
256
|
const edited = changedLines(raw);
|
|
257
|
+
const located = edited.length > 0 ? edited : files;
|
|
226
258
|
const mutantHash = sha256(raw);
|
|
227
259
|
const base = () => ({
|
|
228
260
|
id,
|
|
229
261
|
outcome: "error",
|
|
230
262
|
direction: null,
|
|
231
|
-
files,
|
|
263
|
+
files: located,
|
|
232
264
|
killedBy: [],
|
|
233
265
|
cause: null,
|
|
234
266
|
next: "",
|
|
@@ -277,17 +309,31 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep) {
|
|
|
277
309
|
// node --test loads every file it is given, so a name pattern still pays for the other files.
|
|
278
310
|
const selected = narrowNodeFiles(pkg, selectTests(files, prep.tests, prep.affectedMap), names, prep.spec.kind);
|
|
279
311
|
const commands = [];
|
|
280
|
-
const
|
|
312
|
+
const named = names && names.length > 0 ? names : null;
|
|
313
|
+
const main = launch
|
|
314
|
+
? await launch(pkg, selected, named, limit)
|
|
315
|
+
: await runSuite(pkg, selected, options, null, named, prep.spec, limit, forceWhole, null, null);
|
|
281
316
|
if (main.command)
|
|
282
317
|
commands.push(main.command);
|
|
318
|
+
if (main.crashed) {
|
|
319
|
+
return finishMutant({ ...base(), direction, outcome: "error", error: "suite process crashed" }, pkg, commands.join(" | "), false, main.selectedTests);
|
|
320
|
+
}
|
|
283
321
|
if (main.timedOut) {
|
|
284
322
|
return finishMutant({ ...base(), direction, outcome: "timeout", error: "suite timed out", next: TIMEOUT_NEXT }, pkg, commands.join(" | "), main.wholeSuite || forceWhole, main.selectedTests);
|
|
285
323
|
}
|
|
286
324
|
// A compile error can leave the previous run's report on disk. That report is not this mutant.
|
|
287
325
|
if (main.compileToken)
|
|
288
326
|
return finishMutant(killed(base, direction, [main.compileToken]), pkg, commands.join(" | "), false, []);
|
|
289
|
-
if (!main.report)
|
|
290
|
-
|
|
327
|
+
if (!main.report) {
|
|
328
|
+
const detail = main.detail ? clip(main.detail) : "";
|
|
329
|
+
const tool = missingToolFrom(detail, main.command);
|
|
330
|
+
const error = tool
|
|
331
|
+
? `missing tool: ${tool}`
|
|
332
|
+
: detail
|
|
333
|
+
? `test reporter returned nothing (${detail})`
|
|
334
|
+
: "test reporter returned nothing";
|
|
335
|
+
return { ...base(), direction, error, command: commands.join(" | ") };
|
|
336
|
+
}
|
|
291
337
|
const whole = main.wholeSuite || forceWhole;
|
|
292
338
|
const chosen = main.selectedTests;
|
|
293
339
|
if (main.report.failed.length === 0)
|
|
@@ -364,19 +410,22 @@ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, t
|
|
|
364
410
|
if (rerun.command)
|
|
365
411
|
commands.push(rerun.command);
|
|
366
412
|
if (pattern === null && rerun.timedOut) {
|
|
367
|
-
for (const
|
|
368
|
-
bump(counts,
|
|
413
|
+
for (const id of new Set(failures.map((item) => failureId(pkg, item))))
|
|
414
|
+
bump(counts, id);
|
|
369
415
|
continue;
|
|
370
416
|
}
|
|
371
417
|
if (!rerun.report)
|
|
372
418
|
continue;
|
|
419
|
+
const seen = new Set();
|
|
373
420
|
for (const hit of rerun.report.failed) {
|
|
374
421
|
if (pattern === null && !hit.fileTimeout)
|
|
375
422
|
continue;
|
|
376
423
|
const id = failureId(pkg, hit);
|
|
377
424
|
if (failures.some((item) => failureId(pkg, item) === id))
|
|
378
|
-
|
|
425
|
+
seen.add(id);
|
|
379
426
|
}
|
|
427
|
+
for (const id of seen)
|
|
428
|
+
bump(counts, id);
|
|
380
429
|
}
|
|
381
430
|
return wholeSuite;
|
|
382
431
|
}
|
|
@@ -404,6 +453,164 @@ async function runSuite(pkg, files, options, pattern, names, spec, timeoutMs, fo
|
|
|
404
453
|
selectedTests: result.selectedTests,
|
|
405
454
|
};
|
|
406
455
|
}
|
|
456
|
+
function batchPlan(raw, prep, options) {
|
|
457
|
+
if (options.workers !== 1 || prep.spec.kind !== "node")
|
|
458
|
+
return null;
|
|
459
|
+
if (!prep.coverage || Object.keys(prep.coverage.files).length === 0)
|
|
460
|
+
return null;
|
|
461
|
+
const edited = changedLines(raw);
|
|
462
|
+
const files = filesInDiff(raw);
|
|
463
|
+
if (new Set(files.map((item) => item.file)).size !== 1)
|
|
464
|
+
return null;
|
|
465
|
+
const picked = selectionFor(prep.coverage, edited.length > 0 ? edited : files);
|
|
466
|
+
if (picked.state !== "covered")
|
|
467
|
+
return null;
|
|
468
|
+
const names = picked.tests.filter((name) => name !== WHOLE_PROGRAM_TEST && name.length > 0);
|
|
469
|
+
if (names.length === 0)
|
|
470
|
+
return null;
|
|
471
|
+
return { names };
|
|
472
|
+
}
|
|
473
|
+
/** Covered node mutants of one file share this process. A crash sets dead and the next mutant starts another. */
|
|
474
|
+
class NodeBatch {
|
|
475
|
+
pid;
|
|
476
|
+
dead = false;
|
|
477
|
+
testTimeoutMs;
|
|
478
|
+
proc;
|
|
479
|
+
constructor(proc, pid, testTimeoutMs) {
|
|
480
|
+
this.proc = proc;
|
|
481
|
+
this.pid = pid;
|
|
482
|
+
this.testTimeoutMs = testTimeoutMs;
|
|
483
|
+
proc.on("close", () => {
|
|
484
|
+
this.dead = true;
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
static async start(pkg, options, outDir) {
|
|
488
|
+
const dir = path.join(outDir, "batch");
|
|
489
|
+
mkdirSync(dir, { recursive: true });
|
|
490
|
+
const ready = path.join(dir, `ready-${process.pid}-${Date.now()}.json`);
|
|
491
|
+
const gen = path.join(dir, `gen-${process.pid}-${Date.now()}.txt`);
|
|
492
|
+
writeFileSync(gen, "0\n");
|
|
493
|
+
const script = fileURLToPath(new URL("./node-batch.mjs", import.meta.url));
|
|
494
|
+
const env = childEnv(options.unset);
|
|
495
|
+
env.PB_READY = ready;
|
|
496
|
+
env.PB_GEN = gen;
|
|
497
|
+
env.PB_ROOT = pkg;
|
|
498
|
+
const proc = spawn(process.execPath, [script], { cwd: pkg, env, stdio: ["pipe", "ignore", "pipe"] });
|
|
499
|
+
const started = Date.now();
|
|
500
|
+
while (!existsSync(ready)) {
|
|
501
|
+
if (proc.exitCode !== null)
|
|
502
|
+
throw new Error("batch process exited before ready");
|
|
503
|
+
if (Date.now() - started > 10_000) {
|
|
504
|
+
proc.kill("SIGKILL");
|
|
505
|
+
throw new Error("batch process did not start");
|
|
506
|
+
}
|
|
507
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
508
|
+
}
|
|
509
|
+
const body = JSON.parse(readFileSync(ready, "utf8"));
|
|
510
|
+
return new NodeBatch(proc, body.pid || proc.pid || 0, options.testTimeoutMs);
|
|
511
|
+
}
|
|
512
|
+
stop() {
|
|
513
|
+
this.dead = true;
|
|
514
|
+
try {
|
|
515
|
+
this.proc.stdin?.end();
|
|
516
|
+
}
|
|
517
|
+
catch {
|
|
518
|
+
// The pipe is already closed.
|
|
519
|
+
}
|
|
520
|
+
try {
|
|
521
|
+
if (this.proc.pid)
|
|
522
|
+
process.kill(this.proc.pid, "SIGTERM");
|
|
523
|
+
}
|
|
524
|
+
catch {
|
|
525
|
+
// The process has already exited.
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
async run(pkg, files, names, timeoutMs, id, outDir) {
|
|
529
|
+
const selected = names ?? [];
|
|
530
|
+
const replyFile = path.join(outDir, "batch", `${id}.reply.json`);
|
|
531
|
+
rmSync(replyFile, { force: true });
|
|
532
|
+
appendFileSync(path.join(outDir, "batch-processes.jsonl"), `${JSON.stringify({ id, pid: this.pid })}\n`);
|
|
533
|
+
const job = { id, root: pkg, tests: files, names: selected, timeoutMs, testTimeoutMs: nodeTestTimeoutMs(pkg, files, this.testTimeoutMs, timeoutMs), replyFile };
|
|
534
|
+
try {
|
|
535
|
+
this.proc.stdin?.write(`${JSON.stringify(job)}\n`);
|
|
536
|
+
}
|
|
537
|
+
catch {
|
|
538
|
+
this.dead = true;
|
|
539
|
+
return crashedCall(selected);
|
|
540
|
+
}
|
|
541
|
+
const waited = await waitForReply(replyFile, this.proc, timeoutMs);
|
|
542
|
+
if (waited === "timeout") {
|
|
543
|
+
this.dead = true;
|
|
544
|
+
this.stop();
|
|
545
|
+
return { report: null, timedOut: true, detail: "suite timed out", compileToken: null, command: "", wholeSuite: false, selectedTests: selected };
|
|
546
|
+
}
|
|
547
|
+
if (!waited)
|
|
548
|
+
return crashedCall(selected);
|
|
549
|
+
const failed = waited.failed ?? [];
|
|
550
|
+
const seen = waited.names ?? [];
|
|
551
|
+
if (seen.length === 0 && failed.length === 0) {
|
|
552
|
+
return {
|
|
553
|
+
report: null,
|
|
554
|
+
timedOut: false,
|
|
555
|
+
detail: waited.error || "test reporter returned nothing",
|
|
556
|
+
compileToken: null,
|
|
557
|
+
command: waited.command || "",
|
|
558
|
+
wholeSuite: false,
|
|
559
|
+
selectedTests: selected,
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
return {
|
|
563
|
+
report: { tests: Math.max(seen.length, failed.length), pass: Math.max(0, seen.length - failed.length), fail: failed.length, failed, names: seen.length > 0 ? seen : selected },
|
|
564
|
+
timedOut: false,
|
|
565
|
+
detail: "",
|
|
566
|
+
compileToken: null,
|
|
567
|
+
command: waited.command || "",
|
|
568
|
+
wholeSuite: false,
|
|
569
|
+
selectedTests: selected,
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
function crashedCall(selected) {
|
|
574
|
+
return { report: null, timedOut: false, detail: "suite process crashed", compileToken: null, command: "", wholeSuite: false, selectedTests: selected, crashed: true };
|
|
575
|
+
}
|
|
576
|
+
function waitForReply(file, proc, timeoutMs) {
|
|
577
|
+
return new Promise((resolve) => {
|
|
578
|
+
let settled = false;
|
|
579
|
+
const finish = (value) => {
|
|
580
|
+
if (settled)
|
|
581
|
+
return;
|
|
582
|
+
settled = true;
|
|
583
|
+
clearTimeout(timer);
|
|
584
|
+
clearInterval(poll);
|
|
585
|
+
proc.removeListener("close", onClose);
|
|
586
|
+
resolve(value);
|
|
587
|
+
};
|
|
588
|
+
const read = () => {
|
|
589
|
+
try {
|
|
590
|
+
return JSON.parse(readFileSync(file, "utf8"));
|
|
591
|
+
}
|
|
592
|
+
catch {
|
|
593
|
+
return null;
|
|
594
|
+
}
|
|
595
|
+
};
|
|
596
|
+
const onClose = () => {
|
|
597
|
+
const reply = existsSync(file) ? read() : null;
|
|
598
|
+
finish(reply);
|
|
599
|
+
};
|
|
600
|
+
const timer = setTimeout(() => finish("timeout"), timeoutMs);
|
|
601
|
+
const poll = setInterval(() => {
|
|
602
|
+
if (!existsSync(file))
|
|
603
|
+
return;
|
|
604
|
+
const reply = read();
|
|
605
|
+
if (reply)
|
|
606
|
+
finish(reply);
|
|
607
|
+
}, 20);
|
|
608
|
+
if (proc.exitCode !== null || proc.signalCode !== null)
|
|
609
|
+
onClose();
|
|
610
|
+
else
|
|
611
|
+
proc.once("close", onClose);
|
|
612
|
+
});
|
|
613
|
+
}
|
|
407
614
|
function isNodeTestFile(file) {
|
|
408
615
|
const base = path.posix.basename(file);
|
|
409
616
|
return /\.(test|spec)\.(ts|tsx|mts|js|mjs|cjs)$/.test(base) || /_test\.(ts|tsx|js|mjs)$/.test(base);
|
|
@@ -517,8 +724,8 @@ function loadMeta(dirs) {
|
|
|
517
724
|
}
|
|
518
725
|
return meta;
|
|
519
726
|
}
|
|
520
|
-
function loadRun(
|
|
521
|
-
const file = path.join(outDir, "run.json");
|
|
727
|
+
function loadRun(options, commit) {
|
|
728
|
+
const file = path.join(options.outDir, "run.json");
|
|
522
729
|
if (existsSync(file)) {
|
|
523
730
|
const stored = JSON.parse(readFileSync(file, "utf8"));
|
|
524
731
|
if (stored.commit !== commit)
|
|
@@ -526,7 +733,24 @@ function loadRun(outDir, commit, agent) {
|
|
|
526
733
|
return { id: stored.id };
|
|
527
734
|
}
|
|
528
735
|
const id = randomBytes(4).toString("hex");
|
|
529
|
-
|
|
736
|
+
const record = {
|
|
737
|
+
id,
|
|
738
|
+
commit,
|
|
739
|
+
agent: options.agent,
|
|
740
|
+
started: new Date().toISOString(),
|
|
741
|
+
package: options.packageDir,
|
|
742
|
+
patches: options.patchDirs,
|
|
743
|
+
out: options.outDir,
|
|
744
|
+
repo: options.repoDir,
|
|
745
|
+
workers: options.workers,
|
|
746
|
+
confirm: options.confirm,
|
|
747
|
+
testsDir: options.testsDir,
|
|
748
|
+
suiteTimeoutMs: options.suiteTimeoutMs,
|
|
749
|
+
testTimeoutMs: options.testTimeoutMs,
|
|
750
|
+
direction: options.direction,
|
|
751
|
+
suiteCommand: options.suiteCommand ?? null,
|
|
752
|
+
};
|
|
753
|
+
writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`);
|
|
530
754
|
return { id };
|
|
531
755
|
}
|
|
532
756
|
async function createWorktrees(repo, packageDir, runId, commit, workers, outDir) {
|
|
@@ -534,6 +758,10 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
534
758
|
const dirs = [];
|
|
535
759
|
for (let index = 0; index < workers; index++) {
|
|
536
760
|
const dir = path.join(tmpdir(), `probatio-${runId}-${index}`);
|
|
761
|
+
if (!probatioWorktreeDir(dir)) {
|
|
762
|
+
removeWorktrees(repo, dirs, outDir);
|
|
763
|
+
return { error: "worktree path is outside the probatio temp prefix" };
|
|
764
|
+
}
|
|
537
765
|
rmSync(dir, { recursive: true, force: true });
|
|
538
766
|
const added = git(repo, ["worktree", "add", "--detach", dir, commit]);
|
|
539
767
|
if (added.status !== 0) {
|
|
@@ -559,15 +787,42 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
559
787
|
function hidePaths(dirs, repo, packageDir, hide) {
|
|
560
788
|
for (const dir of dirs) {
|
|
561
789
|
const pkg = packageIn(dir, repo, packageDir);
|
|
790
|
+
const root = path.resolve(pkg);
|
|
562
791
|
for (const rel of hide) {
|
|
563
|
-
if (!rel || path.isAbsolute(rel))
|
|
792
|
+
if (!rel || path.isAbsolute(rel) || rel.split(/[\\/]/).includes(".."))
|
|
793
|
+
continue;
|
|
794
|
+
const target = path.resolve(root, rel);
|
|
795
|
+
if (target !== root && !target.startsWith(root + path.sep))
|
|
564
796
|
continue;
|
|
565
|
-
rmSync(
|
|
797
|
+
rmSync(target, { force: true });
|
|
566
798
|
}
|
|
567
799
|
}
|
|
568
800
|
}
|
|
801
|
+
/** A worktree Probatio created: one directory under the OS temp dir, named probatio- plus hex and an index. */
|
|
802
|
+
export function probatioWorktreeDir(dir) {
|
|
803
|
+
let resolved = path.resolve(dir);
|
|
804
|
+
try {
|
|
805
|
+
resolved = realpathSync(dir);
|
|
806
|
+
}
|
|
807
|
+
catch {
|
|
808
|
+
// The directory is created on the next line. The unresolved path is still checked.
|
|
809
|
+
}
|
|
810
|
+
const roots = new Set([path.resolve(tmpdir())]);
|
|
811
|
+
try {
|
|
812
|
+
roots.add(realpathSync(tmpdir()));
|
|
813
|
+
}
|
|
814
|
+
catch {
|
|
815
|
+
// tmpdir() itself is enough when realpath is unavailable.
|
|
816
|
+
}
|
|
817
|
+
const inside = [...roots].some((root) => resolved === root || resolved.startsWith(root + path.sep));
|
|
818
|
+
if (!inside)
|
|
819
|
+
return false;
|
|
820
|
+
return /^probatio-[0-9a-f]+-\d+$/.test(path.basename(resolved));
|
|
821
|
+
}
|
|
569
822
|
function removeWorktrees(repo, dirs, outDir) {
|
|
570
823
|
for (const dir of dirs) {
|
|
824
|
+
if (!probatioWorktreeDir(dir))
|
|
825
|
+
continue;
|
|
571
826
|
git(repo, ["worktree", "remove", "--force", dir]);
|
|
572
827
|
rmSync(dir, { recursive: true, force: true });
|
|
573
828
|
}
|
|
@@ -708,6 +963,9 @@ function reportOf(results, context) {
|
|
|
708
963
|
notes.push(resumeNext(context.notStarted));
|
|
709
964
|
if (results.some((result) => result.wholeSuite))
|
|
710
965
|
notes.push(WHOLE_SUITE_NEXT);
|
|
966
|
+
const broken = results.find((result) => (result.outcome === "error" || result.outcome === "build-failed") && result.error);
|
|
967
|
+
if (broken?.error)
|
|
968
|
+
notes.push(`${broken.id}: ${broken.error}`);
|
|
711
969
|
if (notes.length === 0)
|
|
712
970
|
notes.push("No survivor in this batch.");
|
|
713
971
|
return {
|
|
@@ -965,13 +1223,27 @@ function testArg(pkg, file, testsDir) {
|
|
|
965
1223
|
function escapeRegExp(value) {
|
|
966
1224
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
967
1225
|
}
|
|
968
|
-
function
|
|
1226
|
+
function missingToolFrom(detail, command) {
|
|
1227
|
+
const spawnMiss = /spawn ([^\s]+) ENOENT/.exec(detail);
|
|
1228
|
+
if (spawnMiss)
|
|
1229
|
+
return path.basename(spawnMiss[1]);
|
|
1230
|
+
const shellMiss = /(?:^|\s)([A-Za-z0-9_./+-]+): (?:command not found|not found)\b/.exec(detail);
|
|
1231
|
+
if (shellMiss)
|
|
1232
|
+
return path.basename(shellMiss[1]);
|
|
1233
|
+
if (/No such file or directory|ENOENT/.test(detail)) {
|
|
1234
|
+
const bin = command.trim().split(/\s+/)[0];
|
|
1235
|
+
if (bin)
|
|
1236
|
+
return path.basename(bin);
|
|
1237
|
+
}
|
|
1238
|
+
return null;
|
|
1239
|
+
}
|
|
1240
|
+
function baselineReportError(detail, command = "") {
|
|
1241
|
+
const tool = missingToolFrom(detail, command);
|
|
969
1242
|
const line = detail.trim();
|
|
970
1243
|
// The first line of a Python or Node crash is a stack fragment. The summary
|
|
971
1244
|
// states the limit and does not carry that fragment.
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
return `baseline suite did not return a test report (${line})`;
|
|
1245
|
+
const base = !line || isStackFragment(line) ? "baseline suite did not return a test report" : `baseline suite did not return a test report (${line})`;
|
|
1246
|
+
return tool ? `missing tool: ${tool}. ${base}` : base;
|
|
975
1247
|
}
|
|
976
1248
|
function isStackFragment(line) {
|
|
977
1249
|
return line.startsWith("Traceback") || /^\s+at \S/.test(line) || /File ".+", line \d+/.test(line);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { existsSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { exportLlvmProfile, llvmToolsPresent } from "./coverage-map.js";
|
|
4
5
|
/**
|
|
5
6
|
* One instrumented test binary, then one profile per test name.
|
|
6
7
|
* A single cargo test profile does not say which test hit the line.
|
|
@@ -9,9 +10,7 @@ export function writeRustCoverage(pkg, testNames, dest) {
|
|
|
9
10
|
const names = [...new Set(testNames.map((name) => name.trim()).filter((name) => name.length > 0 && !/\s/.test(name)))];
|
|
10
11
|
if (names.length === 0)
|
|
11
12
|
return { ok: false, detail: "rust coverage: no test names" };
|
|
12
|
-
|
|
13
|
-
const llvmProf = findTool("llvm-profdata");
|
|
14
|
-
if (!llvmCov || !llvmProf)
|
|
13
|
+
if (!llvmToolsPresent())
|
|
15
14
|
return { ok: false, detail: "rust coverage: llvm tool missing" };
|
|
16
15
|
const env = { ...process.env, RUSTFLAGS: "-C instrument-coverage", CARGO_INCREMENTAL: "0" };
|
|
17
16
|
const built = spawnSync("cargo", ["test", "--no-run", "--message-format=json"], {
|
|
@@ -43,11 +42,8 @@ export function writeRustCoverage(pkg, testNames, dest) {
|
|
|
43
42
|
if (!new RegExp(`^test ${escapeRegExp(name)} \\.\\.\\. ok`, "m").test(`${ran.stdout}\n${ran.stderr}`))
|
|
44
43
|
continue;
|
|
45
44
|
const merged = path.join(work, `${safe(name)}-${safe(path.basename(exe))}.profdata`);
|
|
46
|
-
const
|
|
47
|
-
if (
|
|
48
|
-
continue;
|
|
49
|
-
const exported = spawnSync(llvmCov, ["export", exe, `-instr-profile=${merged}`], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
|
50
|
-
if (exported.status !== 0)
|
|
45
|
+
const exported = exportLlvmProfile(exe, [raw], merged);
|
|
46
|
+
if (!exported.ok)
|
|
51
47
|
continue;
|
|
52
48
|
for (const hit of llvmHits(exported.stdout, pkg))
|
|
53
49
|
addHit(files, hit.file, hit.line, name);
|
|
@@ -136,17 +132,6 @@ function addHit(files, file, line, test) {
|
|
|
136
132
|
lines[key] = names;
|
|
137
133
|
files[file] = lines;
|
|
138
134
|
}
|
|
139
|
-
function findTool(name) {
|
|
140
|
-
const fromXcrun = spawnSync("xcrun", ["--find", name], { encoding: "utf8" });
|
|
141
|
-
const xcrunPath = fromXcrun.stdout?.trim() ?? "";
|
|
142
|
-
if (fromXcrun.status === 0 && xcrunPath && existsSync(xcrunPath))
|
|
143
|
-
return xcrunPath;
|
|
144
|
-
const which = spawnSync("sh", ["-c", `command -v ${name}`], { encoding: "utf8" });
|
|
145
|
-
const found = which.stdout?.trim() ?? "";
|
|
146
|
-
if (which.status === 0 && found)
|
|
147
|
-
return found;
|
|
148
|
-
return null;
|
|
149
|
-
}
|
|
150
135
|
function safe(value) {
|
|
151
136
|
return value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
152
137
|
}
|
package/dist/mutate/sealed.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
2
3
|
/** The label stays in its own file. The run report must not repeat these three values. */
|
|
3
4
|
export function readSealedLabel(file) {
|
|
4
5
|
let text = "";
|
|
@@ -18,6 +19,44 @@ export function readSealedLabel(file) {
|
|
|
18
19
|
export function labelLeaked(body, label) {
|
|
19
20
|
return body.includes(label.bugId) || body.includes(label.issueId) || body.includes(label.expectedFail);
|
|
20
21
|
}
|
|
22
|
+
/** The envelope can be clean while a result file on disk still repeats the label. */
|
|
23
|
+
export function sealedTreeLeaked(dir, label) {
|
|
24
|
+
const visit = (current) => {
|
|
25
|
+
let names;
|
|
26
|
+
try {
|
|
27
|
+
names = readdirSync(current);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
for (const name of names) {
|
|
33
|
+
const full = path.join(current, name);
|
|
34
|
+
let info;
|
|
35
|
+
try {
|
|
36
|
+
info = statSync(full);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (info.isDirectory()) {
|
|
42
|
+
if (visit(full))
|
|
43
|
+
return true;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (!info.isFile() || info.size > 2_000_000)
|
|
47
|
+
continue;
|
|
48
|
+
try {
|
|
49
|
+
if (labelLeaked(readFileSync(full, "utf8"), label))
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// A binary artifact is not a report.
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return false;
|
|
57
|
+
};
|
|
58
|
+
return visit(dir);
|
|
59
|
+
}
|
|
21
60
|
export const LABEL_LEAK_SUMMARY = "The label was copied into the report.";
|
|
22
61
|
function field(text, key) {
|
|
23
62
|
const match = text.match(new RegExp(`^${key}=(.*)$`, "m"));
|
|
@@ -30,7 +30,7 @@ export function timeoutLimitMs(baselineMs, multiple, floorMs) {
|
|
|
30
30
|
return Math.max(floor, scaled);
|
|
31
31
|
}
|
|
32
32
|
/** Kinds that collect a per-line map on the baseline. Other kinds pay the whole suite. */
|
|
33
|
-
const LINE_MAP_KINDS = new Set(["node", "pytest", "c", "go", "maven", "cargo", "dotnet"]);
|
|
33
|
+
const LINE_MAP_KINDS = new Set(["node", "pytest", "c", "go", "maven", "cargo", "dotnet", "mocha"]);
|
|
34
34
|
/** A baseline this long, with no line map, is too expensive to repeat for a large batch. */
|
|
35
35
|
export const SLOW_BASELINE_MS = 5_000;
|
|
36
36
|
/** Pending mutants above this count, on a slow unmapped suite, are not started. */
|