probatio 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -7
- package/dist/cli.js +22 -3
- package/dist/ledger/build.js +14 -0
- package/dist/mcp.js +75 -26
- package/dist/mutate/child-lines.mjs +23 -18
- package/dist/mutate/fail-reporter.mjs +21 -0
- package/dist/mutate/load-failure.mjs +40 -0
- package/dist/mutate/mocha-coverage.cjs +7 -1
- package/dist/mutate/node-batch-loader.mjs +15 -1
- package/dist/mutate/node-batch.mjs +40 -3
- package/dist/mutate/node-coverage.mjs +7 -32
- package/dist/mutate/patch.js +2 -1
- package/dist/mutate/precise-lines.mjs +147 -31
- package/dist/mutate/probatio-jacoco-run.java +2 -2
- package/dist/mutate/project-config.js +58 -0
- package/dist/mutate/run.js +94 -15
- package/dist/mutate/sealed.js +54 -0
- package/dist/mutate/suite-decision.js +2 -1
- package/dist/mutate/suites.js +87 -155
- package/dist/mutate/tally.js +54 -26
- package/dist/swarm/check-kill.js +14 -7
- package/package.json +2 -2
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
// Baseline-only per-test line map for node --test. Loaded with --import in each
|
|
2
2
|
// test-file process. Writes one shard; the runner merges shards after the suite.
|
|
3
3
|
// takePreciseCoverage resets counters, so each sample is only what ran since the last take.
|
|
4
|
-
// A
|
|
5
|
-
//
|
|
4
|
+
// A node child of a test does not run these hooks. exposeChildLines puts child-lines.mjs in
|
|
5
|
+
// its NODE_OPTIONS, so it leaves a nameless hit list in ${map}.children when it exits.
|
|
6
|
+
// afterEach stores that list under the parent test before the shard is written.
|
|
6
7
|
import { randomBytes } from "node:crypto"
|
|
7
|
-
import {
|
|
8
|
+
import { mkdirSync, writeFileSync } from "node:fs"
|
|
8
9
|
import { after, afterEach, before } from "node:test"
|
|
9
10
|
import path from "node:path"
|
|
10
|
-
import { openLineSampler } from "./precise-lines.mjs"
|
|
11
|
+
import { exposeChildLines, openLineSampler, takeChildHits } from "./precise-lines.mjs"
|
|
11
12
|
|
|
12
13
|
const mapPath = process.env.PROBATIO_COVERAGE_MAP
|
|
13
14
|
if (mapPath) {
|
|
@@ -19,6 +20,7 @@ if (mapPath) {
|
|
|
19
20
|
}
|
|
20
21
|
|
|
21
22
|
if (sampler) {
|
|
23
|
+
exposeChildLines()
|
|
22
24
|
/** @type {Map<string, Map<number, Set<string>>>} */
|
|
23
25
|
const byFile = new Map()
|
|
24
26
|
/** @type {Array<{ file: string, line: number }>} */
|
|
@@ -41,33 +43,6 @@ if (mapPath) {
|
|
|
41
43
|
}
|
|
42
44
|
}
|
|
43
45
|
|
|
44
|
-
function takeChildHits() {
|
|
45
|
-
const dir = `${mapPath}.children`
|
|
46
|
-
if (!existsSync(dir)) return []
|
|
47
|
-
/** @type {Array<{ file: string, line: number }>} */
|
|
48
|
-
const hits = []
|
|
49
|
-
for (const name of readdirSync(dir)) {
|
|
50
|
-
if (!name.endsWith(".json")) continue
|
|
51
|
-
const full = path.join(dir, name)
|
|
52
|
-
try {
|
|
53
|
-
const parsed = JSON.parse(readFileSync(full, "utf8"))
|
|
54
|
-
const list = parsed && Array.isArray(parsed.hits) ? parsed.hits : []
|
|
55
|
-
for (const hit of list) {
|
|
56
|
-
if (!hit || typeof hit.file !== "string" || !Number.isInteger(hit.line)) continue
|
|
57
|
-
hits.push({ file: hit.file, line: hit.line })
|
|
58
|
-
}
|
|
59
|
-
} catch {
|
|
60
|
-
// A torn dump is ignored. The parent does not invent a line.
|
|
61
|
-
}
|
|
62
|
-
try {
|
|
63
|
-
unlinkSync(full)
|
|
64
|
-
} catch {
|
|
65
|
-
// The next test must not inherit this dump.
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
return hits
|
|
69
|
-
}
|
|
70
|
-
|
|
71
46
|
before(async () => {
|
|
72
47
|
try {
|
|
73
48
|
prelude = await sampler.positiveLines()
|
|
@@ -80,7 +55,7 @@ if (mapPath) {
|
|
|
80
55
|
try {
|
|
81
56
|
add(prelude, t.name)
|
|
82
57
|
add(await sampler.positiveLines(), t.name)
|
|
83
|
-
add(takeChildHits(), t.name)
|
|
58
|
+
add(takeChildHits(mapPath), t.name)
|
|
84
59
|
} catch (err) {
|
|
85
60
|
writeFileSync(`${mapPath}.error`, `${err instanceof Error ? err.message : String(err)}\n`)
|
|
86
61
|
}
|
package/dist/mutate/patch.js
CHANGED
|
@@ -131,5 +131,6 @@ export function applyPatch(repo, raw, direction) {
|
|
|
131
131
|
/** Put a throwaway worktree back. A diff label is not trusted for the path list. */
|
|
132
132
|
export function restoreTree(repo) {
|
|
133
133
|
git(repo, ["checkout", "--", "."]);
|
|
134
|
-
|
|
134
|
+
// .probatio.json may be the user's uncommitted copy, put there when the worktree was made.
|
|
135
|
+
git(repo, ["clean", "-fd", "-e", "node_modules", "-e", ".probatio.json"]);
|
|
135
136
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// Inspector line samples shared by the node:test collector, the Mocha collector,
|
|
2
2
|
// and a plain-script child. This file has no test hooks and writes no shard.
|
|
3
|
+
import { existsSync, readdirSync, readFileSync, unlinkSync } from "node:fs"
|
|
4
|
+
import { Session as SyncSession } from "node:inspector"
|
|
3
5
|
import { Session } from "node:inspector/promises"
|
|
4
6
|
import { findSourceMap } from "node:module"
|
|
5
7
|
import path from "node:path"
|
|
@@ -31,32 +33,14 @@ function decodeSegments(row) {
|
|
|
31
33
|
}
|
|
32
34
|
|
|
33
35
|
/**
|
|
36
|
+
* Offsets to source lines, shared by the async sampler (test processes) and the sync one (children).
|
|
37
|
+
* Paths are relative to `root`. node_modules and paths outside root are left out.
|
|
34
38
|
* @param {(rel: string) => boolean} [skipRel]
|
|
35
|
-
* @
|
|
39
|
+
* @param {string} [root]
|
|
36
40
|
*/
|
|
37
|
-
|
|
38
|
-
const session = new Session()
|
|
39
|
-
session.connect()
|
|
40
|
-
await session.post("Profiler.enable")
|
|
41
|
-
await session.post("Debugger.enable")
|
|
42
|
-
await session.post("Profiler.startPreciseCoverage", { callCount: true, detailed: true })
|
|
43
|
-
|
|
44
|
-
const textCache = new Map()
|
|
41
|
+
function lineConverter(skipRel, root = process.cwd()) {
|
|
45
42
|
const indexCache = new Map()
|
|
46
43
|
|
|
47
|
-
async function scriptText(script) {
|
|
48
|
-
if (textCache.has(script.scriptId)) return textCache.get(script.scriptId)
|
|
49
|
-
let text = ""
|
|
50
|
-
try {
|
|
51
|
-
const got = await session.post("Debugger.getScriptSource", { scriptId: script.scriptId })
|
|
52
|
-
text = got.scriptSource || ""
|
|
53
|
-
} catch {
|
|
54
|
-
text = ""
|
|
55
|
-
}
|
|
56
|
-
textCache.set(script.scriptId, text)
|
|
57
|
-
return text
|
|
58
|
-
}
|
|
59
|
-
|
|
60
44
|
function relPath(url) {
|
|
61
45
|
if (!url || url.startsWith("node:")) return null
|
|
62
46
|
let abs
|
|
@@ -65,7 +49,7 @@ export async function openLineSampler(skipRel) {
|
|
|
65
49
|
} catch {
|
|
66
50
|
return null
|
|
67
51
|
}
|
|
68
|
-
const rel = path.relative(
|
|
52
|
+
const rel = path.relative(root, abs).split(path.sep).join("/")
|
|
69
53
|
if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return null
|
|
70
54
|
if (rel.split("/").includes("node_modules")) return null
|
|
71
55
|
if (skipRel && skipRel(rel)) return null
|
|
@@ -163,18 +147,55 @@ export async function openLineSampler(skipRel) {
|
|
|
163
147
|
return counts
|
|
164
148
|
}
|
|
165
149
|
|
|
150
|
+
/** Lines with a positive count in one script, or null when the script is not ours. */
|
|
151
|
+
function scriptLines(script, text) {
|
|
152
|
+
const file = relPath(script.url)
|
|
153
|
+
if (!file || !text) return null
|
|
154
|
+
const counts = applyRanges(text, script, indexFor(script.scriptId, text, findSourceMap(script.url)))
|
|
155
|
+
const found = []
|
|
156
|
+
for (const [line, count] of counts) {
|
|
157
|
+
if (count > 0) found.push({ file, line })
|
|
158
|
+
}
|
|
159
|
+
return found
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return { relPath, scriptLines }
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* @param {(rel: string) => boolean} [skipRel]
|
|
167
|
+
* @returns {Promise<{ positiveLines: () => Promise<Array<{ file: string, line: number }>>, reset: () => Promise<void> }>}
|
|
168
|
+
*/
|
|
169
|
+
export async function openLineSampler(skipRel) {
|
|
170
|
+
const session = new Session()
|
|
171
|
+
session.connect()
|
|
172
|
+
await session.post("Profiler.enable")
|
|
173
|
+
await session.post("Debugger.enable")
|
|
174
|
+
await session.post("Profiler.startPreciseCoverage", { callCount: true, detailed: true })
|
|
175
|
+
|
|
176
|
+
const textCache = new Map()
|
|
177
|
+
const convert = lineConverter(skipRel)
|
|
178
|
+
|
|
179
|
+
async function scriptText(script) {
|
|
180
|
+
if (textCache.has(script.scriptId)) return textCache.get(script.scriptId)
|
|
181
|
+
let text = ""
|
|
182
|
+
try {
|
|
183
|
+
const got = await session.post("Debugger.getScriptSource", { scriptId: script.scriptId })
|
|
184
|
+
text = got.scriptSource || ""
|
|
185
|
+
} catch {
|
|
186
|
+
text = ""
|
|
187
|
+
}
|
|
188
|
+
textCache.set(script.scriptId, text)
|
|
189
|
+
return text
|
|
190
|
+
}
|
|
191
|
+
|
|
166
192
|
async function positiveLines() {
|
|
167
193
|
const cov = await session.post("Profiler.takePreciseCoverage")
|
|
168
194
|
const found = []
|
|
169
195
|
for (const script of cov.result || []) {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
if (!text) continue
|
|
174
|
-
const counts = applyRanges(text, script, indexFor(script.scriptId, text, findSourceMap(script.url)))
|
|
175
|
-
for (const [line, count] of counts) {
|
|
176
|
-
if (count > 0) found.push({ file, line })
|
|
177
|
-
}
|
|
196
|
+
if (!convert.relPath(script.url)) continue
|
|
197
|
+
const lines = convert.scriptLines(script, await scriptText(script))
|
|
198
|
+
if (lines) found.push(...lines)
|
|
178
199
|
}
|
|
179
200
|
return found
|
|
180
201
|
}
|
|
@@ -186,3 +207,98 @@ export async function openLineSampler(skipRel) {
|
|
|
186
207
|
},
|
|
187
208
|
}
|
|
188
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* The same sampler with synchronous calls, so an `exit` handler can read it.
|
|
213
|
+
* `exit` runs after process.exit(), where a promise never settles.
|
|
214
|
+
* @param {(rel: string) => boolean} [skipRel]
|
|
215
|
+
* @param {string} [root]
|
|
216
|
+
*/
|
|
217
|
+
export function openSyncLineSampler(skipRel, root) {
|
|
218
|
+
const session = new SyncSession()
|
|
219
|
+
session.connect()
|
|
220
|
+
const post = (method, params) => {
|
|
221
|
+
let result
|
|
222
|
+
let failure = null
|
|
223
|
+
let done = false
|
|
224
|
+
session.post(method, params, (err, value) => {
|
|
225
|
+
failure = err
|
|
226
|
+
result = value
|
|
227
|
+
done = true
|
|
228
|
+
})
|
|
229
|
+
if (!done) throw new Error(`inspector ${method} did not answer synchronously`)
|
|
230
|
+
if (failure) throw failure
|
|
231
|
+
return result
|
|
232
|
+
}
|
|
233
|
+
post("Profiler.enable")
|
|
234
|
+
post("Debugger.enable")
|
|
235
|
+
post("Profiler.startPreciseCoverage", { callCount: true, detailed: true })
|
|
236
|
+
const convert = lineConverter(skipRel, root)
|
|
237
|
+
return {
|
|
238
|
+
positiveLines() {
|
|
239
|
+
const cov = post("Profiler.takePreciseCoverage")
|
|
240
|
+
const found = []
|
|
241
|
+
for (const script of cov.result || []) {
|
|
242
|
+
if (!convert.relPath(script.url)) continue
|
|
243
|
+
let text = ""
|
|
244
|
+
try {
|
|
245
|
+
text = post("Debugger.getScriptSource", { scriptId: script.scriptId }).scriptSource || ""
|
|
246
|
+
} catch {
|
|
247
|
+
text = ""
|
|
248
|
+
}
|
|
249
|
+
const lines = convert.scriptLines(script, text)
|
|
250
|
+
if (lines) found.push(...lines)
|
|
251
|
+
}
|
|
252
|
+
return found
|
|
253
|
+
},
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Called inside a test process that collects a line map. Every node child it starts
|
|
259
|
+
* loads child-lines.mjs through NODE_OPTIONS, with no import in the child's own code.
|
|
260
|
+
* The marker tells child-lines that this process already has test hooks.
|
|
261
|
+
*/
|
|
262
|
+
export function exposeChildLines() {
|
|
263
|
+
globalThis.__probatioTestHooks = true
|
|
264
|
+
// One dump dir per test process. Test files can run in parallel, and a child's lines
|
|
265
|
+
// belong to the test in the process that started it, not to a test in another file.
|
|
266
|
+
const mapPath = process.env.PROBATIO_COVERAGE_MAP
|
|
267
|
+
if (mapPath) process.env.PROBATIO_CHILD_DIR = childDir(mapPath)
|
|
268
|
+
const childLines = new URL("./child-lines.mjs", import.meta.url).href
|
|
269
|
+
const flag = `--import=${childLines}`
|
|
270
|
+
const current = process.env.NODE_OPTIONS || ""
|
|
271
|
+
if (!current.includes(flag)) process.env.NODE_OPTIONS = current ? `${current} ${flag}` : flag
|
|
272
|
+
if (!process.env.PROBATIO_PACKAGE_ROOT) process.env.PROBATIO_PACKAGE_ROOT = process.cwd()
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
export function childDir(mapPath) {
|
|
276
|
+
return path.join(`${mapPath}.children`, String(process.pid))
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Line dumps the children of the current test left. Each one is read once. */
|
|
280
|
+
export function takeChildHits(mapPath) {
|
|
281
|
+
const dir = childDir(mapPath)
|
|
282
|
+
if (!existsSync(dir)) return []
|
|
283
|
+
const hits = []
|
|
284
|
+
for (const name of readdirSync(dir)) {
|
|
285
|
+
if (!name.endsWith(".json")) continue
|
|
286
|
+
const full = path.join(dir, name)
|
|
287
|
+
try {
|
|
288
|
+
const parsed = JSON.parse(readFileSync(full, "utf8"))
|
|
289
|
+
const list = parsed && Array.isArray(parsed.hits) ? parsed.hits : []
|
|
290
|
+
for (const hit of list) {
|
|
291
|
+
if (!hit || typeof hit.file !== "string" || !Number.isInteger(hit.line)) continue
|
|
292
|
+
hits.push({ file: hit.file, line: hit.line })
|
|
293
|
+
}
|
|
294
|
+
} catch {
|
|
295
|
+
// A torn dump is ignored. The parent does not invent a line.
|
|
296
|
+
}
|
|
297
|
+
try {
|
|
298
|
+
unlinkSync(full)
|
|
299
|
+
} catch {
|
|
300
|
+
// The next test must not inherit this dump.
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return hits
|
|
304
|
+
}
|
|
@@ -85,8 +85,8 @@ public class ProbatioJacocoRun {
|
|
|
85
85
|
System.err.println("java coverage: tests ran but JaCoCo recorded no classes");
|
|
86
86
|
System.exit(1);
|
|
87
87
|
}
|
|
88
|
-
//
|
|
89
|
-
// wait on those threads until the parent clock kills the dump.
|
|
88
|
+
// Integration tests can leave non-daemon threads running (jsoup leaves Netty loops).
|
|
89
|
+
// A normal return would wait on those threads until the parent clock kills the dump.
|
|
90
90
|
System.exit(0);
|
|
91
91
|
}
|
|
92
92
|
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* `.probatio.json` at the package root. How to run this project's suite, written by the
|
|
5
|
+
* project and visible in review. Probatio does not patch a project's files or environment
|
|
6
|
+
* to make one repository pass. A project that needs that says so here.
|
|
7
|
+
*/
|
|
8
|
+
export const CONFIG_FILE = ".probatio.json";
|
|
9
|
+
const EMPTY = { env: {}, testLine: null, passLine: null };
|
|
10
|
+
export function readProjectConfig(pkg) {
|
|
11
|
+
const file = path.join(pkg, CONFIG_FILE);
|
|
12
|
+
if (!existsSync(file))
|
|
13
|
+
return EMPTY;
|
|
14
|
+
let raw;
|
|
15
|
+
try {
|
|
16
|
+
raw = JSON.parse(readFileSync(file, "utf8"));
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
return { error: `${CONFIG_FILE} is not JSON (${error instanceof Error ? error.message : String(error)})` };
|
|
20
|
+
}
|
|
21
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
22
|
+
return { error: `${CONFIG_FILE} must be one JSON object` };
|
|
23
|
+
const body = raw;
|
|
24
|
+
const env = {};
|
|
25
|
+
if (body.env !== undefined) {
|
|
26
|
+
if (!body.env || typeof body.env !== "object" || Array.isArray(body.env))
|
|
27
|
+
return { error: `${CONFIG_FILE} env must be an object of strings` };
|
|
28
|
+
for (const [key, value] of Object.entries(body.env)) {
|
|
29
|
+
if (typeof value !== "string")
|
|
30
|
+
return { error: `${CONFIG_FILE} env.${key} must be a string` };
|
|
31
|
+
env[key] = value;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const testLine = pattern(body.testLine, "testLine");
|
|
35
|
+
if (typeof testLine === "string")
|
|
36
|
+
return { error: testLine };
|
|
37
|
+
const passLine = pattern(body.passLine, "passLine");
|
|
38
|
+
if (typeof passLine === "string")
|
|
39
|
+
return { error: passLine };
|
|
40
|
+
return { env, testLine, passLine };
|
|
41
|
+
}
|
|
42
|
+
/** The config, or the empty one when the file is unreadable. The runner refuses that case earlier. */
|
|
43
|
+
export function projectConfig(pkg) {
|
|
44
|
+
const read = readProjectConfig(pkg);
|
|
45
|
+
return "error" in read ? EMPTY : read;
|
|
46
|
+
}
|
|
47
|
+
function pattern(value, name) {
|
|
48
|
+
if (value === undefined || value === null)
|
|
49
|
+
return null;
|
|
50
|
+
if (typeof value !== "string" || value.length === 0)
|
|
51
|
+
return `${CONFIG_FILE} ${name} must be a non-empty string`;
|
|
52
|
+
try {
|
|
53
|
+
return new RegExp(value);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
return `${CONFIG_FILE} ${name} is not a regular expression (${error instanceof Error ? error.message : String(error)})`;
|
|
57
|
+
}
|
|
58
|
+
}
|
package/dist/mutate/run.js
CHANGED
|
@@ -12,7 +12,8 @@ import { writeJavaCoverage } from "./java-coverage.js";
|
|
|
12
12
|
import { writeRustCoverage } from "./rust-coverage.js";
|
|
13
13
|
import { applyPatch, changedLines, chooseDirection, filesInDiff, git, restoreTree } from "./patch.js";
|
|
14
14
|
import { killLabel } from "./kill-label.js";
|
|
15
|
-
import {
|
|
15
|
+
import { CONFIG_FILE, readProjectConfig } from "./project-config.js";
|
|
16
|
+
import { UNVIABLE_NEXT, TIMEOUT_FLOOR_MS, TIMEOUT_MULTIPLE, TIMEOUT_NEXT, WHOLE_PROGRAM_TEST, WHOLE_SUITE_NEXT, campaignSummary, decideSuite, hasLineMap, noCoverageNext, resumeNext, scaleStop, timeoutLimitMs, } from "./suite-decision.js";
|
|
16
17
|
import { commandSuite, discoverSuite, nodeTestTimeoutMs, runDiscoveredSuite, sourceCandidates } from "./suites.js";
|
|
17
18
|
import { duplicateTitles } from "./titles.js";
|
|
18
19
|
export const reporterPath = fileURLToPath(new URL("./fail-reporter.mjs", import.meta.url));
|
|
@@ -52,6 +53,9 @@ export async function runMutants(options) {
|
|
|
52
53
|
: listed.files;
|
|
53
54
|
if (patchFiles.length === 0)
|
|
54
55
|
return failReport(options.outDir, run.id, commit, "no patches");
|
|
56
|
+
const config = readProjectConfig(packageDir);
|
|
57
|
+
if ("error" in config)
|
|
58
|
+
return failReport(options.outDir, run.id, commit, config.error, undefined, `Fix ${CONFIG_FILE} and run the same command again.`);
|
|
55
59
|
const gate = resolveSuite(packageDir, options);
|
|
56
60
|
if (gate.action === "stop")
|
|
57
61
|
return failReport(options.outDir, run.id, commit, gate.summary, undefined, gate.next, gate.suiteCommand);
|
|
@@ -67,6 +71,13 @@ export async function runMutants(options) {
|
|
|
67
71
|
if ("error" in worktrees)
|
|
68
72
|
return failReport(options.outDir, run.id, commit, worktrees.error);
|
|
69
73
|
hidePaths(worktrees.dirs, repo, packageDir, options.hide ?? []);
|
|
74
|
+
// A cancelled run (MCP cancel, Ctrl-C) still removes its worktrees from the user's repo.
|
|
75
|
+
const onSignal = (signal) => {
|
|
76
|
+
removeWorktrees(repo, worktrees.dirs, options.outDir);
|
|
77
|
+
process.exit(signal === "SIGINT" ? 130 : 143);
|
|
78
|
+
};
|
|
79
|
+
process.once("SIGTERM", onSignal);
|
|
80
|
+
process.once("SIGINT", onSignal);
|
|
70
81
|
const budgetMs = options.budgetMs ?? (options.maxMinutes == null ? null : options.maxMinutes * 60_000);
|
|
71
82
|
let budgetStarted = 0;
|
|
72
83
|
const overBudget = () => budgetMs !== null && budgetStarted !== 0 && Date.now() - budgetStarted > budgetMs;
|
|
@@ -113,7 +124,7 @@ export async function runMutants(options) {
|
|
|
113
124
|
const prior = history.get(id);
|
|
114
125
|
if (prior && reusable(prior, raw, treePkg)) {
|
|
115
126
|
startedMutants += 1;
|
|
116
|
-
const reused = reusedResult(prior, commit, run.id, options.agent);
|
|
127
|
+
const reused = normalizeResult(reusedResult(prior, commit, run.id, options.agent));
|
|
117
128
|
writeResult(options.outDir, reused);
|
|
118
129
|
progress(`reused ${id}`);
|
|
119
130
|
continue;
|
|
@@ -165,6 +176,8 @@ export async function runMutants(options) {
|
|
|
165
176
|
});
|
|
166
177
|
}
|
|
167
178
|
finally {
|
|
179
|
+
process.removeListener("SIGTERM", onSignal);
|
|
180
|
+
process.removeListener("SIGINT", onSignal);
|
|
168
181
|
removeWorktrees(repo, worktrees.dirs, options.outDir);
|
|
169
182
|
}
|
|
170
183
|
}
|
|
@@ -292,10 +305,12 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
292
305
|
return { ...base(), direction, outcome: "timeout", error: "build timed out", next: TIMEOUT_NEXT };
|
|
293
306
|
// The baseline build already succeeded. A mutant that no longer builds cannot pass the suite.
|
|
294
307
|
if (built.code !== 0)
|
|
295
|
-
return finishMutant(
|
|
308
|
+
return finishMutant(unviable(base, direction, "build"), pkg, "", false, []);
|
|
296
309
|
}
|
|
297
310
|
const picked = selectionFor(prep.coverage, edited.length > 0 ? edited : files);
|
|
298
|
-
const forced = options.onlyNames && options.onlyNames.length > 0
|
|
311
|
+
const forced = options.onlyNames && options.onlyNames.length > 0
|
|
312
|
+
? prep.spec.kind === "node" ? options.onlyNames.map(nodeTitle) : options.onlyNames
|
|
313
|
+
: null;
|
|
299
314
|
if (!forced && picked.state === "uncovered") {
|
|
300
315
|
return {
|
|
301
316
|
...base(),
|
|
@@ -323,7 +338,7 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
323
338
|
}
|
|
324
339
|
// A compile error can leave the previous run's report on disk. That report is not this mutant.
|
|
325
340
|
if (main.compileToken)
|
|
326
|
-
return finishMutant(
|
|
341
|
+
return finishMutant(unviable(base, direction, main.compileToken), pkg, commands.join(" | "), false, []);
|
|
327
342
|
if (!main.report) {
|
|
328
343
|
const detail = main.detail ? clip(main.detail) : "";
|
|
329
344
|
const tool = missingToolFrom(detail, main.command);
|
|
@@ -334,13 +349,23 @@ async function runOne(worktree, pkg, patchPath, options, commit, runId, prep, la
|
|
|
334
349
|
: "test reporter returned nothing";
|
|
335
350
|
return { ...base(), direction, error, command: commands.join(" | ") };
|
|
336
351
|
}
|
|
352
|
+
// A node name pattern that matches nothing passes the file. That is not a survivor.
|
|
353
|
+
// A requested describe block reports its leaf tests, so any test that ran counts.
|
|
354
|
+
if (forced && prep.spec.kind === "node" && main.report.names.every((name) => isNodeTestFile(name))) {
|
|
355
|
+
return { ...base(), direction, error: `no requested test ran (${forced.join(", ")})`, command: commands.join(" | ") };
|
|
356
|
+
}
|
|
337
357
|
const whole = main.wholeSuite || forceWhole;
|
|
338
358
|
const chosen = main.selectedTests;
|
|
339
359
|
if (main.report.failed.length === 0)
|
|
340
360
|
return finishMutant({ ...base(), direction, outcome: "survived" }, pkg, commands.join(" | "), whole, chosen);
|
|
361
|
+
// A test file that no longer links ran no test. It is the build failing, the same as javac rejecting it.
|
|
362
|
+
// Only a failure where code ran can kill. When none did, the mutant did not build.
|
|
363
|
+
const ran = main.report.failed.filter((item) => !item.staticLoad);
|
|
364
|
+
if (ran.length === 0)
|
|
365
|
+
return finishMutant(unviable(base, direction, "compiler"), pkg, commands.join(" | "), false, []);
|
|
341
366
|
if (!options.confirm)
|
|
342
|
-
return finishMutant(killed(base, direction,
|
|
343
|
-
const confirmed = await confirmFailures(pkg, options,
|
|
367
|
+
return finishMutant(killed(base, direction, ran.map((item) => failureId(pkg, item))), pkg, commands.join(" | "), whole, chosen);
|
|
368
|
+
const confirmed = await confirmFailures(pkg, options, ran, prep, limit, commands);
|
|
344
369
|
const killedBy = confirmed.killed.map((item) => failureId(pkg, item));
|
|
345
370
|
const flaky = confirmed.flaky.map((item) => failureId(pkg, item));
|
|
346
371
|
const outcome = killedBy.length > 0 ? "killed" : "flaky";
|
|
@@ -368,16 +393,31 @@ function killed(base, direction, killedBy) {
|
|
|
368
393
|
const label = killLabel(killedBy);
|
|
369
394
|
return { ...base(), direction, outcome: "killed", killedBy, cause: label?.cause ?? "test", next: label?.next ?? "" };
|
|
370
395
|
}
|
|
396
|
+
/** The compiler rejected the mutant. `killedBy` names the compiler so the record says which one. */
|
|
397
|
+
function unviable(base, direction, compiler) {
|
|
398
|
+
return { ...base(), direction, outcome: "unviable", killedBy: [compiler], cause: "build", next: UNVIABLE_NEXT };
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Results written before `unviable` existed stored a compile failure as killed with cause build.
|
|
402
|
+
* Read them the new way so an old out dir does not count a compiler as a test.
|
|
403
|
+
*/
|
|
404
|
+
export function normalizeResult(result) {
|
|
405
|
+
if (result.outcome === "killed" && result.cause === "build")
|
|
406
|
+
return { ...result, outcome: "unviable", next: UNVIABLE_NEXT };
|
|
407
|
+
return result;
|
|
408
|
+
}
|
|
371
409
|
async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands) {
|
|
372
410
|
const counts = new Map();
|
|
373
411
|
let wholeSuite = false;
|
|
374
412
|
const named = new Map();
|
|
375
413
|
const hungFiles = new Map();
|
|
414
|
+
const unloaded = new Map();
|
|
376
415
|
for (const failure of failed) {
|
|
377
416
|
const key = failure.file || "*";
|
|
378
417
|
// A name filter skips the tests that left the handle open, so the file
|
|
379
418
|
// timeout never comes back. Rerun that file with no name pattern.
|
|
380
|
-
|
|
419
|
+
// A file that did not load has no test name to filter by either.
|
|
420
|
+
const bucket = failure.fileTimeout ? hungFiles : failure.fileLoad ? unloaded : named;
|
|
381
421
|
const list = bucket.get(key) ?? [];
|
|
382
422
|
list.push(failure);
|
|
383
423
|
bucket.set(key, list);
|
|
@@ -391,13 +431,17 @@ async function confirmFailures(pkg, options, failed, prep, timeoutMs, commands)
|
|
|
391
431
|
if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands))
|
|
392
432
|
wholeSuite = true;
|
|
393
433
|
}
|
|
434
|
+
for (const [file, failures] of unloaded) {
|
|
435
|
+
if (await rerunGroup(pkg, options, file, failures, counts, null, prep, timeoutMs, commands, "load"))
|
|
436
|
+
wholeSuite = true;
|
|
437
|
+
}
|
|
394
438
|
const killed = [];
|
|
395
439
|
const flaky = [];
|
|
396
440
|
for (const failure of failed)
|
|
397
441
|
(counts.get(failureId(pkg, failure)) === 2 ? killed : flaky).push(failure);
|
|
398
442
|
return { killed, flaky, wholeSuite };
|
|
399
443
|
}
|
|
400
|
-
async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands) {
|
|
444
|
+
async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, timeoutMs, commands, fileFailure = "timeout") {
|
|
401
445
|
const files = file === "*" ? listTestFiles(pkg, options.testsDir, prep.spec) : [testArg(pkg, file, options.testsDir)];
|
|
402
446
|
const isolated = { ...options, concurrency: 1 };
|
|
403
447
|
let wholeSuite = false;
|
|
@@ -409,7 +453,7 @@ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, t
|
|
|
409
453
|
wholeSuite = true;
|
|
410
454
|
if (rerun.command)
|
|
411
455
|
commands.push(rerun.command);
|
|
412
|
-
if (pattern === null && rerun.timedOut) {
|
|
456
|
+
if (pattern === null && fileFailure === "timeout" && rerun.timedOut) {
|
|
413
457
|
for (const id of new Set(failures.map((item) => failureId(pkg, item))))
|
|
414
458
|
bump(counts, id);
|
|
415
459
|
continue;
|
|
@@ -418,7 +462,7 @@ async function rerunGroup(pkg, options, file, failures, counts, pattern, prep, t
|
|
|
418
462
|
continue;
|
|
419
463
|
const seen = new Set();
|
|
420
464
|
for (const hit of rerun.report.failed) {
|
|
421
|
-
if (pattern === null && !hit.fileTimeout)
|
|
465
|
+
if (pattern === null && !(fileFailure === "timeout" ? hit.fileTimeout : hit.fileLoad))
|
|
422
466
|
continue;
|
|
423
467
|
const id = failureId(pkg, hit);
|
|
424
468
|
if (failures.some((item) => failureId(pkg, item) === id))
|
|
@@ -552,7 +596,7 @@ class NodeBatch {
|
|
|
552
596
|
return {
|
|
553
597
|
report: null,
|
|
554
598
|
timedOut: false,
|
|
555
|
-
detail: waited.error || "test reporter returned nothing",
|
|
599
|
+
detail: waited.error || (selected.length > 0 ? `no requested test ran (${selected.join(", ")})` : "test reporter returned nothing"),
|
|
556
600
|
compileToken: null,
|
|
557
601
|
command: waited.command || "",
|
|
558
602
|
wholeSuite: false,
|
|
@@ -615,6 +659,11 @@ function isNodeTestFile(file) {
|
|
|
615
659
|
const base = path.posix.basename(file);
|
|
616
660
|
return /\.(test|spec)\.(ts|tsx|mts|js|mjs|cjs)$/.test(base) || /_test\.(ts|tsx|js|mjs)$/.test(base);
|
|
617
661
|
}
|
|
662
|
+
/** A node keep id may carry its file (`tests/gate.test.ts::zero stays shut`). node:test matches the title. */
|
|
663
|
+
export function nodeTitle(id) {
|
|
664
|
+
const match = /^[^\s:][^:]*\.(?:[cm]?[jt]s|[jt]sx)::(.+)$/.exec(id);
|
|
665
|
+
return match ? match[1] : id;
|
|
666
|
+
}
|
|
618
667
|
function narrowNodeFiles(pkg, files, names, kind) {
|
|
619
668
|
if (kind !== "node" || !names || names.length === 0)
|
|
620
669
|
return files;
|
|
@@ -652,6 +701,18 @@ function childEnv(unset) {
|
|
|
652
701
|
if (key === "NODE_TEST_CONTEXT" || key.startsWith("NODE_TEST_"))
|
|
653
702
|
delete env[key];
|
|
654
703
|
}
|
|
704
|
+
// This run may itself be a child of a suite that another Probatio run is mapping.
|
|
705
|
+
// That outer map is not this run's. Only the baseline sets a map, through runDiscoveredSuite.
|
|
706
|
+
delete env.PROBATIO_COVERAGE_MAP;
|
|
707
|
+
delete env.PROBATIO_PACKAGE_ROOT;
|
|
708
|
+
delete env.PROBATIO_CHILD_DIR;
|
|
709
|
+
if (env.NODE_OPTIONS) {
|
|
710
|
+
const kept = env.NODE_OPTIONS.split(/\s+/).filter((part) => part.length > 0 && !/child-lines\.mjs$/.test(part)).join(" ");
|
|
711
|
+
if (kept)
|
|
712
|
+
env.NODE_OPTIONS = kept;
|
|
713
|
+
else
|
|
714
|
+
delete env.NODE_OPTIONS;
|
|
715
|
+
}
|
|
655
716
|
for (const key of unset)
|
|
656
717
|
delete env[key];
|
|
657
718
|
return env;
|
|
@@ -772,8 +833,12 @@ async function createWorktrees(repo, packageDir, runId, commit, workers, outDir)
|
|
|
772
833
|
}
|
|
773
834
|
writeFileSync(path.join(outDir, "worktrees.json"), `${JSON.stringify(dirs)}\n`);
|
|
774
835
|
const sources = nodeModuleSources(repo, packageDir);
|
|
836
|
+
const config = path.join(packageDir, CONFIG_FILE);
|
|
775
837
|
for (const dir of dirs) {
|
|
776
838
|
const pkgRoot = packageIn(dir, repo, packageDir);
|
|
839
|
+
// How to run the suite is not the code under test. An uncommitted .probatio.json still applies.
|
|
840
|
+
if (existsSync(config) && !existsSync(path.join(pkgRoot, CONFIG_FILE)))
|
|
841
|
+
copyFileSync(config, path.join(pkgRoot, CONFIG_FILE));
|
|
777
842
|
for (const source of sources) {
|
|
778
843
|
const dest = path.join(pkgRoot, source.rel);
|
|
779
844
|
if (existsSync(dest))
|
|
@@ -932,11 +997,12 @@ function readResult(outDir, id) {
|
|
|
932
997
|
const file = resultPath(outDir, id);
|
|
933
998
|
if (!existsSync(file))
|
|
934
999
|
return null;
|
|
935
|
-
return JSON.parse(readFileSync(file, "utf8"));
|
|
1000
|
+
return normalizeResult(JSON.parse(readFileSync(file, "utf8")));
|
|
936
1001
|
}
|
|
937
1002
|
function reportOf(results, context) {
|
|
938
1003
|
const count = (outcome) => results.filter((result) => result.outcome === outcome).length;
|
|
939
1004
|
const killed = count("killed");
|
|
1005
|
+
const unviableCount = count("unviable");
|
|
940
1006
|
const survived = count("survived");
|
|
941
1007
|
const flaky = count("flaky");
|
|
942
1008
|
const timeouts = count("timeout");
|
|
@@ -950,7 +1016,7 @@ function reportOf(results, context) {
|
|
|
950
1016
|
return { id: result.id, file, line, ...(known?.op ? { op: known.op } : {}) };
|
|
951
1017
|
});
|
|
952
1018
|
const budgetHit = context.budgetHit || context.pendingLeft > 0;
|
|
953
|
-
const summary = campaignSummary({ noCoverage, survived, killed, flaky, timeouts, errors, finished: results.length });
|
|
1019
|
+
const summary = campaignSummary({ noCoverage, survived, killed, unviable: unviableCount, flaky, timeouts, errors, finished: results.length });
|
|
954
1020
|
const notes = [];
|
|
955
1021
|
const uncovered = results.find((result) => result.outcome === "no coverage");
|
|
956
1022
|
if (uncovered?.next)
|
|
@@ -976,6 +1042,8 @@ function reportOf(results, context) {
|
|
|
976
1042
|
commit: context.commit,
|
|
977
1043
|
runId: context.runId,
|
|
978
1044
|
killed,
|
|
1045
|
+
unviable: unviableCount,
|
|
1046
|
+
unviableIds: results.filter((result) => result.outcome === "unviable").slice(0, 10).map((result) => result.id),
|
|
979
1047
|
survived,
|
|
980
1048
|
flaky,
|
|
981
1049
|
timeouts,
|
|
@@ -1035,6 +1103,8 @@ function failReport(outDir, runId, commit, error, baselineFailed, next, suiteCom
|
|
|
1035
1103
|
commit,
|
|
1036
1104
|
runId,
|
|
1037
1105
|
killed: 0,
|
|
1106
|
+
unviable: 0,
|
|
1107
|
+
unviableIds: [],
|
|
1038
1108
|
survived: 0,
|
|
1039
1109
|
flaky: 0,
|
|
1040
1110
|
timeouts: 0,
|
|
@@ -1199,10 +1269,19 @@ function failureId(pkg, failure) {
|
|
|
1199
1269
|
return BARE_SUITE.has(failure.name) ? `::${failure.name}` : failure.name;
|
|
1200
1270
|
if (!testFileOnDisk(pkg, failure.file))
|
|
1201
1271
|
return failure.name;
|
|
1202
|
-
|
|
1272
|
+
// macOS temp dirs are /var and /private/var. A report can name either, so compare real paths.
|
|
1273
|
+
const rel = path.relative(realOrSame(pkg), realOrSame(path.resolve(pkg, failure.file)));
|
|
1203
1274
|
const file = !rel || rel.startsWith("..") ? path.basename(failure.file) : rel.split(path.sep).join("/");
|
|
1204
1275
|
return `${file}::${failure.name}`;
|
|
1205
1276
|
}
|
|
1277
|
+
function realOrSame(file) {
|
|
1278
|
+
try {
|
|
1279
|
+
return realpathSync(file);
|
|
1280
|
+
}
|
|
1281
|
+
catch {
|
|
1282
|
+
return file;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1206
1285
|
function testFileOnDisk(pkg, file) {
|
|
1207
1286
|
if (path.isAbsolute(file))
|
|
1208
1287
|
return existsSync(file);
|