testguard-cli 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -1
- package/README.md +16 -4
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +65 -3
- package/spec/README.md +7 -0
- package/spec/schemas/evidence.schema.json +14 -0
- package/src/cli.mjs +4 -1
- package/src/command-budget.mjs +52 -0
- package/src/commands/probe.mjs +9 -0
- package/src/commands/replay.mjs +14 -1
- package/src/commands/sweep.mjs +15 -6
- package/src/git.mjs +4 -2
- package/src/probe/attribution.mjs +2 -0
- package/src/probe/discover.mjs +44 -8
- package/src/probe/imports.mjs +785 -0
- package/src/probe/mocks.mjs +13 -4
- package/src/probe/probe.mjs +176 -19
- package/src/probe/rank.mjs +28 -5
- package/src/probe/runners/discovery.mjs +634 -0
- package/src/probe/runners/index.mjs +51 -19
- package/src/probe/runners/jest.mjs +32 -1
- package/src/probe/runners/playwright.mjs +58 -2
- package/src/probe/runners/python.mjs +7 -7
- package/src/probe/runners/shared.mjs +55 -6
- package/src/probe/runners/vitest.mjs +21 -0
- package/src/replay/replay.mjs +108 -13
- package/src/sweep/sweep.mjs +5 -0
package/src/probe/mocks.mjs
CHANGED
|
@@ -67,7 +67,7 @@ function expectMentions(source, bindings) {
|
|
|
67
67
|
* an unasserted mock with a reason (`// unasserted: <why>` on or above the
|
|
68
68
|
* mock line).
|
|
69
69
|
*/
|
|
70
|
-
export function analyzeDefender(projectDir, testRel, targetRel) {
|
|
70
|
+
export function analyzeDefender(projectDir, testRel, targetRel, relation) {
|
|
71
71
|
const abs = join(projectDir, testRel);
|
|
72
72
|
let source;
|
|
73
73
|
try {
|
|
@@ -75,7 +75,16 @@ export function analyzeDefender(projectDir, testRel, targetRel) {
|
|
|
75
75
|
} catch {
|
|
76
76
|
return { file: testRel, imports: false, mocks: false, asserted: false, annotation: null };
|
|
77
77
|
}
|
|
78
|
-
|
|
78
|
+
// A symbol-level witness can pass through one or more barrels. Mocking any
|
|
79
|
+
// module on that witnessed path replaces the subject just as surely as
|
|
80
|
+
// mocking the leaf itself, so it must disqualify the candidate. Declared
|
|
81
|
+
// defenders do not supply a witness and retain the direct, author-chosen
|
|
82
|
+
// semantics.
|
|
83
|
+
const witnessedModules = relation?.status === 'matched'
|
|
84
|
+
? [...new Set(relation.paths.flatMap((path) => path.slice(1)))]
|
|
85
|
+
: [];
|
|
86
|
+
const matchesTarget = (spec) => specifierResolvesTo(projectDir, abs, spec, targetRel)
|
|
87
|
+
|| witnessedModules.some((moduleRel) => specifierResolvesTo(projectDir, abs, spec, moduleRel));
|
|
79
88
|
const mocks = mockSpecifiers(source).filter((m) => matchesTarget(m.specifier));
|
|
80
89
|
const bindings = bindingsFor(source, matchesTarget);
|
|
81
90
|
const asserted = expectMentions(source, bindings);
|
|
@@ -104,13 +113,13 @@ function matchesTargetAnyImport(source, matchesTarget) {
|
|
|
104
113
|
* the target carries `mocked-never-asserted`, unless annotated, in which case
|
|
105
114
|
* the annotation is recorded (`unasserted-annotated`) — silenced, never hidden.
|
|
106
115
|
*/
|
|
107
|
-
export function classifyDefenders(projectDir, targetRel, candidates) {
|
|
116
|
+
export function classifyDefenders(projectDir, targetRel, candidates, relations = new Map()) {
|
|
108
117
|
if (targetRel.endsWith('.py')) return classifyPythonDefenders(projectDir, targetRel, candidates);
|
|
109
118
|
const canDetect = [];
|
|
110
119
|
const mocking = [];
|
|
111
120
|
const signals = [];
|
|
112
121
|
for (const file of candidates) {
|
|
113
|
-
const a = analyzeDefender(projectDir, file, targetRel);
|
|
122
|
+
const a = analyzeDefender(projectDir, file, targetRel, relations.get(file));
|
|
114
123
|
if (a.mocks) {
|
|
115
124
|
mocking.push(file);
|
|
116
125
|
if (!a.asserted) signals.push(a.annotation ? { file, signal: 'unasserted-annotated', reason: a.annotation } : { file, signal: 'mocked-never-asserted' });
|
package/src/probe/probe.mjs
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
|
|
2
|
-
import {
|
|
2
|
+
import { gitRaw } from '../git.mjs';
|
|
3
3
|
import { createRequire } from 'node:module';
|
|
4
4
|
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
5
5
|
import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../git.mjs';
|
|
6
|
-
import {
|
|
6
|
+
import { discoverDefendersDetailed } from './discover.mjs';
|
|
7
7
|
import { classifyDefenders } from './mocks.mjs';
|
|
8
8
|
import { detectContention, contentionWarning } from './contention.mjs';
|
|
9
9
|
import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
|
|
10
10
|
import { applyContent, applyFault, locate } from './inject.mjs';
|
|
11
11
|
import { resolveDefenders, parseCommandTemplate } from './runners/shared.mjs';
|
|
12
|
-
import { selectRunner, RUNNERS, OWNED_RUNNERS, partitionByRunner, mergeRuns, runnerLabel, runnerFor } from './runners/index.mjs';
|
|
12
|
+
import { selectRunner, discoverRunnerManifest, RUNNERS, OWNED_RUNNERS, partitionByRunner, mergeRuns, runnerLabel, runnerFor } from './runners/index.mjs';
|
|
13
13
|
import { classify, shouldStopEarly } from './classify.mjs';
|
|
14
14
|
import { blastRadius, rank } from './rank.mjs';
|
|
15
15
|
import { classifyIndependence } from './independence.mjs';
|
|
@@ -66,6 +66,7 @@ export async function probe({
|
|
|
66
66
|
serial = false,
|
|
67
67
|
onWarn = () => {},
|
|
68
68
|
budgetMs = 120_000,
|
|
69
|
+
commandBudget,
|
|
69
70
|
runnerCommand,
|
|
70
71
|
runnerName = 'auto',
|
|
71
72
|
nodeModules,
|
|
@@ -79,6 +80,7 @@ export async function probe({
|
|
|
79
80
|
previous,
|
|
80
81
|
onProgress = () => {},
|
|
81
82
|
}) {
|
|
83
|
+
commandBudget?.assertOpen();
|
|
82
84
|
// realpath: git reports the repository root by its real path (/private/var
|
|
83
85
|
// on macOS, not /var); every relative() below must start from the same place.
|
|
84
86
|
projectDir = realpathSync(resolve(projectDir));
|
|
@@ -109,12 +111,61 @@ export async function probe({
|
|
|
109
111
|
if (isDirty(root)) snapshot = snapshotWorkingTree(root);
|
|
110
112
|
} else {
|
|
111
113
|
const watched = new Set(targets);
|
|
114
|
+
const records = gitRaw(['status', '--porcelain=v1', '-z'], root).split('\0');
|
|
115
|
+
const dirtyPaths = [];
|
|
116
|
+
for (let i = 0; i < records.length; i += 1) {
|
|
117
|
+
const record = records[i];
|
|
118
|
+
if (!record) continue;
|
|
119
|
+
const status = record.slice(0, 2);
|
|
120
|
+
dirtyPaths.push(record.slice(3));
|
|
121
|
+
// Under -z, rename/copy records carry the destination in this record
|
|
122
|
+
// and the source as the next NUL-delimited record. Watch both: either
|
|
123
|
+
// side may be a declared defender, fault target, or config input.
|
|
124
|
+
if (/[RC]/.test(status) && records[i + 1]) dirtyPaths.push(records[++i]);
|
|
125
|
+
}
|
|
126
|
+
let configuredFiles;
|
|
127
|
+
// Use the same exact primary + owned native universes as measurement.
|
|
128
|
+
// A dirty custom-named Playwright test or imported config helper must be
|
|
129
|
+
// refused before the scratch tree silently substitutes HEAD's version.
|
|
130
|
+
if (dirtyPaths.length && !runnerCommand) {
|
|
131
|
+
const budgetFor = () => commandBudget?.runBudget(budgetMs) ?? budgetMs;
|
|
132
|
+
const selected = await selectRunner({ projectDir, sourceDir: projectDir, python, name: runnerName, budgetMs: budgetFor(), budgetFor });
|
|
133
|
+
commandBudget?.assertOpen();
|
|
134
|
+
if (selected.error) throw new PreconditionError(`cannot verify dirty defender/config inputs because test discovery failed (${selected.error})`);
|
|
135
|
+
const manifests = new Map([[selected.runner, selected.manifest]]);
|
|
136
|
+
const primaryIsPython = ['python', 'pytest', 'unittest'].includes(selected.runner.name);
|
|
137
|
+
const owned = OWNED_RUNNERS.filter((candidate) => candidate !== selected.runner && !(candidate.name === 'python' && primaryIsPython));
|
|
138
|
+
for (const candidate of owned) {
|
|
139
|
+
const check = await candidate.check({ projectDir, sourceDir: projectDir, python, budgetMs: budgetFor(), budgetFor });
|
|
140
|
+
commandBudget?.assertOpen();
|
|
141
|
+
if (!check.ok) continue;
|
|
142
|
+
try {
|
|
143
|
+
manifests.set(candidate, await discoverRunnerManifest(candidate, check, projectDir, budgetFor()));
|
|
144
|
+
} catch (error) {
|
|
145
|
+
throw new PreconditionError(`cannot verify dirty defender/config inputs because ${candidate.name} discovery failed (${error.message})`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
configuredFiles = Object.freeze([...new Set([...manifests.values()].flatMap((manifest) => manifest.files))].sort());
|
|
149
|
+
for (const manifest of manifests.values()) {
|
|
150
|
+
for (const file of manifest.files) watched.add(relative(root, join(projectDir, file)));
|
|
151
|
+
for (const config of manifest.configFiles) {
|
|
152
|
+
if (!config.path.startsWith('@module/')) watched.add(relative(root, join(projectDir, config.path)));
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
112
156
|
for (const claim of scopedClaims) {
|
|
113
|
-
const
|
|
157
|
+
const discovered = claim.defendedBy?.length ? null : claim.faults.map((fault) => discoverDefendersDetailed(projectDir, fault.file, configuredFiles));
|
|
158
|
+
const declared = claim.defendedBy?.length ? resolveDefenders(projectDir, claim.defendedBy) : discovered.flatMap((detail) => detail.canDetect);
|
|
114
159
|
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
160
|
+
for (const dependency of discovered?.flatMap((detail) => detail.dependencies) ?? []) watched.add(relative(root, join(projectDir, dependency)));
|
|
115
161
|
for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
116
162
|
}
|
|
117
|
-
|
|
163
|
+
for (const config of [
|
|
164
|
+
'package.json', 'pnpm-workspace.yaml', 'pnpm-workspace.yml', 'lerna.json', 'nx.json', 'turbo.json', 'rush.json', 'workspace.json',
|
|
165
|
+
...['js', 'mjs', 'cjs', 'ts', 'mts', 'cts'].flatMap((ext) => [`vitest.config.${ext}`, `vite.config.${ext}`, `vitest.workspace.${ext}`, `jest.config.${ext}`, `playwright.config.${ext}`]),
|
|
166
|
+
'vitest.workspace.json',
|
|
167
|
+
]) watched.add(relative(root, join(projectDir, config)));
|
|
168
|
+
const dirty = dirtyPaths.filter((path) => watched.has(path));
|
|
118
169
|
if (dirty.length && !refExplicit && !ignoreDirty) {
|
|
119
170
|
throw new PreconditionError(`${dirty.length} defender/target file${dirty.length === 1 ? ' has' : 's have'} uncommitted changes (${dirty.join(', ')}); worktree mode probes HEAD (${head.slice(0, 7)}), so those changes would be silently ignored. Commit them, run with --include-dirty to probe the working tree, use --in-place, or --ignore-dirty if you mean HEAD as committed.`);
|
|
120
171
|
}
|
|
@@ -135,6 +186,7 @@ export async function probe({
|
|
|
135
186
|
const records = [];
|
|
136
187
|
let runnerVersion;
|
|
137
188
|
let runnerSource;
|
|
189
|
+
let primaryManifest;
|
|
138
190
|
let runner = RUNNERS[runnerName === 'auto' ? 'vitest' : runnerName];
|
|
139
191
|
// A runner with more than one engine (Python: pytest or unittest) is recorded
|
|
140
192
|
// by the engine that actually ran, never by the module's name. TestGuard does
|
|
@@ -146,7 +198,8 @@ export async function probe({
|
|
|
146
198
|
const commandTemplate = runnerCommand ? parseCommandTemplate(runnerCommand) : undefined;
|
|
147
199
|
if (!commandTemplate) {
|
|
148
200
|
// An unresolvable runner is a precondition failure, not a flaky defender.
|
|
149
|
-
const sel = await selectRunner({ projectDir: iso.projectDir, sourceDir: projectDir, python, name: runnerName });
|
|
201
|
+
const sel = await selectRunner({ projectDir: iso.projectDir, sourceDir: projectDir, python, name: runnerName, budgetMs, budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs });
|
|
202
|
+
commandBudget?.assertOpen();
|
|
150
203
|
if (sel.error) {
|
|
151
204
|
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
|
|
152
205
|
(mode === 'worktree' ? 'No usable node_modules was linked: pass --node-modules <path>, or run with --in-place.' : 'Install dependencies first.'));
|
|
@@ -154,22 +207,66 @@ export async function probe({
|
|
|
154
207
|
runner = sel.runner;
|
|
155
208
|
runnerVersion = sel.version;
|
|
156
209
|
runnerSource = sel.source;
|
|
210
|
+
primaryManifest = sel.manifest;
|
|
157
211
|
if (sel.engine) engines.set(sel.runner, sel.engine);
|
|
158
212
|
}
|
|
159
213
|
runnersUsed.set(labelOf(runner), runnerVersion ?? readRunnerVersion(projectDir, runner.name));
|
|
160
214
|
// Files an owning runner (Playwright) claims run under it, whatever the
|
|
161
215
|
// project runner is; it must resolve before the first such defender runs.
|
|
162
|
-
const owned = OWNED_RUNNERS.filter((r) => r !== runner
|
|
216
|
+
const owned = OWNED_RUNNERS.filter((r) => r !== runner
|
|
217
|
+
&& !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
|
|
163
218
|
const ownedChecked = new Map();
|
|
219
|
+
const manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
|
|
220
|
+
if (!commandTemplate) {
|
|
221
|
+
for (const r of owned) {
|
|
222
|
+
const check = await r.check({ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs });
|
|
223
|
+
commandBudget?.assertOpen();
|
|
224
|
+
ownedChecked.set(r, check);
|
|
225
|
+
if (!check.ok) continue;
|
|
226
|
+
try {
|
|
227
|
+
manifests.set(r, await discoverRunnerManifest(r, check, iso.projectDir, commandBudget?.runBudget(budgetMs) ?? budgetMs));
|
|
228
|
+
} catch (error) {
|
|
229
|
+
throw new PreconditionError(`${r.name}: test discovery failed: ${error.message}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
164
233
|
const ensureOwned = async (r, file) => {
|
|
165
|
-
if (!ownedChecked.has(r))
|
|
234
|
+
if (!ownedChecked.has(r)) {
|
|
235
|
+
ownedChecked.set(r, await r.check({ projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs }));
|
|
236
|
+
commandBudget?.assertOpen();
|
|
237
|
+
}
|
|
166
238
|
const c = ownedChecked.get(r);
|
|
167
239
|
if (!c.ok) throw new PreconditionError(`${file} is a ${r.name} test (${r.name === 'python' ? 'it is a .py file' : `it lives under ${r.name}'s testDir`}) but ${r.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${c.message}).`);
|
|
168
240
|
if (c.engine) engines.set(r, c.engine);
|
|
169
241
|
runnersUsed.set(labelOf(r), c.version);
|
|
170
242
|
};
|
|
171
|
-
const
|
|
243
|
+
const ownerByFile = new Map();
|
|
244
|
+
if (!commandTemplate) {
|
|
245
|
+
for (const file of manifests.get(runner)?.files ?? []) ownerByFile.set(file, runner);
|
|
246
|
+
for (const [owner, manifest] of manifests) {
|
|
247
|
+
if (owner === runner) continue;
|
|
248
|
+
for (const file of manifest.files) {
|
|
249
|
+
const previousOwner = ownerByFile.get(file);
|
|
250
|
+
if (previousOwner && previousOwner !== runner && previousOwner !== owner) {
|
|
251
|
+
throw new PreconditionError(`${file} was listed by both ${previousOwner.name} and ${owner.name}; TestGuard cannot choose which configured runner owns it.`);
|
|
252
|
+
}
|
|
253
|
+
// A per-file runner that natively lists a file owns it over a broad
|
|
254
|
+
// project runner that also collected it. Membership, never filename
|
|
255
|
+
// heuristics, makes that decision.
|
|
256
|
+
ownerByFile.set(file, owner);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const allTests = Object.freeze((commandTemplate
|
|
261
|
+
? [...new Set([...runner.tests(iso.projectDir).filter((t) => !owned.some((r) => r.owns(iso.projectDir, t))), ...owned.flatMap((r) => r.tests(iso.projectDir))])]
|
|
262
|
+
: [...ownerByFile.keys()]).sort());
|
|
263
|
+
const testUniverseHash = commandTemplate
|
|
264
|
+
? sha256(JSON.stringify({ schemaVersion: 1, source: 'custom-command-static', files: allTests }))
|
|
265
|
+
: sha256(JSON.stringify([...manifests]
|
|
266
|
+
.map(([r, manifest]) => ({ runner: r.name, hash: manifest.testUniverseHash }))
|
|
267
|
+
.sort((a, b) => a.runner.localeCompare(b.runner))));
|
|
172
268
|
const baselineCache = new Map();
|
|
269
|
+
const discoveryHashCache = new Map();
|
|
173
270
|
// The negative control is charged per (target file, defender set): two
|
|
174
271
|
// claims over the same file with the same defenders ask the same question.
|
|
175
272
|
const controlCache = new Map();
|
|
@@ -179,39 +276,73 @@ export async function probe({
|
|
|
179
276
|
const prior = previous && previous.run.confirmRuns === confirmRuns
|
|
180
277
|
? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
|
|
181
278
|
: new Map();
|
|
279
|
+
const partitionConfigured = (files) => {
|
|
280
|
+
if (commandTemplate) return partitionByRunner(iso.projectDir, files, runner);
|
|
281
|
+
const groups = new Map([[runner, []]]);
|
|
282
|
+
for (const file of files) {
|
|
283
|
+
const owner = ownerByFile.get(file);
|
|
284
|
+
if (!owner) throw new PreconditionError(`${file} is not in any resolved runner's configured test universe`);
|
|
285
|
+
if (!groups.has(owner)) groups.set(owner, []);
|
|
286
|
+
groups.get(owner).push(file);
|
|
287
|
+
}
|
|
288
|
+
if (groups.get(runner).length === 0 && groups.size > 1) groups.delete(runner);
|
|
289
|
+
return groups;
|
|
290
|
+
};
|
|
182
291
|
// `targets` are the fault's files, passed so a runner that can tell which
|
|
183
292
|
// file the interpreter actually loaded (Python) reports it back.
|
|
184
293
|
const runDefenders = async (files, targets = []) => {
|
|
185
|
-
if (commandTemplate)
|
|
186
|
-
|
|
294
|
+
if (commandTemplate) {
|
|
295
|
+
const result = await runner.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, targets });
|
|
296
|
+
commandBudget?.assertOpen();
|
|
297
|
+
return result;
|
|
298
|
+
}
|
|
299
|
+
const groups = partitionConfigured(files);
|
|
187
300
|
const parts = [];
|
|
188
301
|
for (const [r, group] of groups) {
|
|
302
|
+
commandBudget?.assertOpen();
|
|
189
303
|
if (r !== runner) await ensureOwned(r, group[0]);
|
|
190
|
-
parts.push(await r.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs, serial, targets }));
|
|
304
|
+
parts.push(await r.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, targets }));
|
|
305
|
+
commandBudget?.assertOpen();
|
|
191
306
|
}
|
|
192
307
|
return mergeRuns(parts);
|
|
193
308
|
};
|
|
194
309
|
/** Which runner each defender runs under, when more than the project runner is involved. */
|
|
195
310
|
const byRunner = (files) => {
|
|
196
|
-
const groups =
|
|
311
|
+
const groups = partitionConfigured(files);
|
|
197
312
|
if (groups.size <= 1) return undefined; // one runner ran them all, whichever it was
|
|
198
313
|
return Object.fromEntries([...groups].map(([r, group]) => [labelOf(r), group]));
|
|
199
314
|
};
|
|
200
315
|
|
|
201
316
|
for (const claim of scopedClaims) {
|
|
317
|
+
commandBudget?.assertOpen();
|
|
202
318
|
const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
319
|
+
if (declared && !commandTemplate) {
|
|
320
|
+
const excluded = declared.filter((file) => !ownerByFile.has(file));
|
|
321
|
+
for (const file of excluded) {
|
|
322
|
+
const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
|
|
323
|
+
const check = expectedOwner && ownedChecked.get(expectedOwner);
|
|
324
|
+
if (expectedOwner && check && !check.ok) {
|
|
325
|
+
throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
if (excluded.length) throw new PreconditionError(`${claim.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
|
|
329
|
+
}
|
|
203
330
|
for (const fault of claim.faults) {
|
|
331
|
+
commandBudget?.assertOpen();
|
|
204
332
|
// Mock-awareness: a discovered file that mocks the target is not a
|
|
205
333
|
// defender; a declared one that mocks it stays (the author named it)
|
|
206
334
|
// but is listed, because a declared defender that mocks the subject
|
|
207
335
|
// is a broken evidence chain the author should see.
|
|
208
|
-
const mockInfo = declared ? classifyDefenders(iso.projectDir, fault.file, declared) : discoverDefendersDetailed(iso.projectDir, fault.file);
|
|
336
|
+
const mockInfo = declared ? classifyDefenders(iso.projectDir, fault.file, declared) : discoverDefendersDetailed(iso.projectDir, fault.file, allTests);
|
|
209
337
|
const defenders = declared ?? mockInfo.canDetect;
|
|
338
|
+
const discoveryHashes = declared ? {} : hashDiscoveryDependencies(iso.projectDir, mockInfo.dependencies, iso.mode === 'worktree' ? discoveryHashCache : new Map());
|
|
210
339
|
const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
|
|
211
340
|
const common = { claim, fault, defenders, discovered: declared === null, byRunner: byRunner(defenders), mocking: mockInfo.mocking, signals: mockInfo.signals };
|
|
212
341
|
let record;
|
|
213
342
|
try {
|
|
214
|
-
|
|
343
|
+
if (!declared && mockInfo.indeterminate.length > 0) {
|
|
344
|
+
record = discoveryIndeterminateRecord({ ...common, inputs: safeInputs(iso.projectDir, fault, defenders, { testUniverseHash, discoveryHashes }), indeterminate: mockInfo.indeterminate });
|
|
345
|
+
} else record = await probeOne({ ...common, allTests, iso, isoReal, onWarn, confirmRuns, escalate, baselineCache, controlCache, fatalEditFor, runDefenders, stage, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id, testUniverseHash, discoveryHashes,
|
|
215
346
|
historyRef: snapshot ?? (mode === 'worktree' ? head : 'HEAD'), historyDir: root,
|
|
216
347
|
// A path from the runner is absolute inside the SCRATCH worktree, or
|
|
217
348
|
// project-relative. Either way history is read from the real repository,
|
|
@@ -225,7 +356,7 @@ export async function probe({
|
|
|
225
356
|
// is not resolvable — and continuing would produce verdicts that are
|
|
226
357
|
// all false. Everything else is this one fault's problem.
|
|
227
358
|
if (err instanceof PreconditionError) throw err;
|
|
228
|
-
record = errorRecord({ ...common, error: err, inputs: safeInputs(iso.projectDir, fault, defenders) });
|
|
359
|
+
record = errorRecord({ ...common, error: err, inputs: safeInputs(iso.projectDir, fault, defenders, { testUniverseHash, discoveryHashes }) });
|
|
229
360
|
onWarn(`${claim.id}/${fault.id}: ${record.detail.message} — reported as unverifiable so the rest of the run still produces evidence.`);
|
|
230
361
|
}
|
|
231
362
|
records.push(record);
|
|
@@ -236,6 +367,8 @@ export async function probe({
|
|
|
236
367
|
iso.cleanup();
|
|
237
368
|
}
|
|
238
369
|
|
|
370
|
+
commandBudget?.assertOpen();
|
|
371
|
+
|
|
239
372
|
return {
|
|
240
373
|
schemaVersion: 1,
|
|
241
374
|
tool: { name: 'testguard', version: toolVersion },
|
|
@@ -285,13 +418,28 @@ export function errorRecord({ claim, fault, defenders, discovered, byRunner, moc
|
|
|
285
418
|
};
|
|
286
419
|
}
|
|
287
420
|
|
|
421
|
+
/** Automatic discovery failed closed: uncertainty can never become `nocover`. */
|
|
422
|
+
export function discoveryIndeterminateRecord({ claim, fault, defenders, discovered, byRunner, mocking = [], signals = [], inputs, indeterminate }) {
|
|
423
|
+
const first = indeterminate[0];
|
|
424
|
+
const message = `${indeterminate.length} candidate test file${indeterminate.length === 1 ? '' : 's'} could not be resolved safely; first: ${first.file} (${first.reason ?? 'unknown'})`.slice(0, 1024);
|
|
425
|
+
return {
|
|
426
|
+
fingerprint: fingerprint({ claimId: claim.id, subjectId: fault.id, file: fault.file, verdict: 'unverifiable' }),
|
|
427
|
+
claim: { id: claim.id, statement: claim.statement, severity: claim.severity, source: claim.source, producedBy: claim.producedBy },
|
|
428
|
+
subject: subjectOf(fault, sha256),
|
|
429
|
+
verdict: 'unverifiable',
|
|
430
|
+
detail: { baselineRuns: [], probeRuns: [], reason: 'defender-discovery-indeterminate', message },
|
|
431
|
+
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: false, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
|
|
432
|
+
inputs,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
|
|
288
436
|
/**
|
|
289
437
|
* `inputs` for a record built after something threw — which is exactly when a
|
|
290
438
|
* file may no longer be readable. Hashing must not be the second failure, so an
|
|
291
439
|
* unreadable file hashes as empty: never equal to a real file's hash, so the
|
|
292
440
|
* record can only ever fail a reuse check, never pass one by accident.
|
|
293
441
|
*/
|
|
294
|
-
function safeInputs(projectDir, fault, defenders) {
|
|
442
|
+
function safeInputs(projectDir, fault, defenders, extra = {}) {
|
|
295
443
|
const h = (f) => {
|
|
296
444
|
try {
|
|
297
445
|
return hashFile(join(projectDir, f));
|
|
@@ -299,7 +447,14 @@ function safeInputs(projectDir, fault, defenders) {
|
|
|
299
447
|
return sha256('');
|
|
300
448
|
}
|
|
301
449
|
};
|
|
302
|
-
return { targetHash: h(fault.file), defenderHashes: Object.fromEntries(defenders.map((f) => [f, h(f)])) };
|
|
450
|
+
return { targetHash: h(fault.file), defenderHashes: Object.fromEntries(defenders.map((f) => [f, h(f)])), ...extra };
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function hashDiscoveryDependencies(projectDir, dependencies = [], cache = new Map()) {
|
|
454
|
+
return Object.fromEntries(dependencies.map((file) => {
|
|
455
|
+
if (!cache.has(file)) cache.set(file, hashFile(join(projectDir, file)));
|
|
456
|
+
return [file, cache.get(file)];
|
|
457
|
+
}));
|
|
303
458
|
}
|
|
304
459
|
|
|
305
460
|
/**
|
|
@@ -348,12 +503,14 @@ async function negativeControl({ file, defenders, iso, runDefenders, fatalEditFo
|
|
|
348
503
|
return reached;
|
|
349
504
|
}
|
|
350
505
|
|
|
351
|
-
async function probeOne({ claim, fault, defenders, discovered, allTests, iso, isoReal, onWarn = () => {}, confirmRuns, escalate, baselineCache, controlCache, fatalEditFor, runDefenders, stage, prior, priorRunId, byRunner, mocking = [], signals = [], historyRef, historyDir, toRepoPath }) {
|
|
506
|
+
async function probeOne({ claim, fault, defenders, discovered, allTests, iso, isoReal, onWarn = () => {}, confirmRuns, escalate, baselineCache, controlCache, fatalEditFor, runDefenders, stage, prior, priorRunId, byRunner, mocking = [], signals = [], historyRef, historyDir, toRepoPath, testUniverseHash, discoveryHashes = {} }) {
|
|
352
507
|
const targetPath = join(iso.projectDir, fault.file);
|
|
353
508
|
const targetExists = existsSync(targetPath);
|
|
354
509
|
const inputs = {
|
|
355
510
|
targetHash: targetExists ? hashFile(targetPath) : sha256(''),
|
|
356
511
|
defenderHashes: Object.fromEntries(defenders.map((f) => [f, hashFile(join(iso.projectDir, f))])),
|
|
512
|
+
testUniverseHash,
|
|
513
|
+
discoveryHashes,
|
|
357
514
|
};
|
|
358
515
|
|
|
359
516
|
const subject = subjectOf(fault, sha256);
|
package/src/probe/rank.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { readFileSync, existsSync } from 'node:fs';
|
|
1
|
+
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
|
2
2
|
import { dirname, join, resolve, extname } from 'node:path';
|
|
3
3
|
import { walk } from '../util/glob.mjs';
|
|
4
|
+
import { createImportResolver } from './imports.mjs';
|
|
4
5
|
import { pyBlastRadius } from './pyimports.mjs';
|
|
5
6
|
|
|
6
7
|
const aliasCache = new Map();
|
|
@@ -17,9 +18,10 @@ function parseJsonc(text) {
|
|
|
17
18
|
* vite/vitest `resolve.alias` read as text, and package.json `imports`. Bare
|
|
18
19
|
* specifiers that match no rule are ignored, as documented.
|
|
19
20
|
*/
|
|
20
|
-
|
|
21
|
+
function loadAliasContext(projectDir) {
|
|
21
22
|
if (aliasCache.has(projectDir)) return aliasCache.get(projectDir);
|
|
22
23
|
const rules = [];
|
|
24
|
+
const configFiles = new Set();
|
|
23
25
|
const toRule = (pattern, targets, base) => {
|
|
24
26
|
const [prefix, suffix = ''] = pattern.split('*');
|
|
25
27
|
rules.push({ prefix, suffix, wildcard: pattern.includes('*'), targets: targets.map((t) => resolve(base, t)) });
|
|
@@ -32,6 +34,7 @@ export function loadAliases(projectDir) {
|
|
|
32
34
|
const cfgPath = queue.shift();
|
|
33
35
|
if (seen.has(cfgPath) || !existsSync(cfgPath)) continue;
|
|
34
36
|
seen.add(cfgPath);
|
|
37
|
+
configFiles.add(cfgPath);
|
|
35
38
|
let cfg;
|
|
36
39
|
try {
|
|
37
40
|
cfg = parseJsonc(readFileSync(cfgPath, 'utf8'));
|
|
@@ -58,6 +61,7 @@ export function loadAliases(projectDir) {
|
|
|
58
61
|
for (const f of ['vitest.config.ts', 'vitest.config.mts', 'vitest.config.js', 'vitest.config.mjs', 'vite.config.ts', 'vite.config.mts', 'vite.config.js', 'vite.config.mjs']) {
|
|
59
62
|
const path = join(projectDir, f);
|
|
60
63
|
if (!existsSync(path)) continue;
|
|
64
|
+
configFiles.add(path);
|
|
61
65
|
const text = readFileSync(path, 'utf8');
|
|
62
66
|
const start = /\balias\s*:\s*([\[{])/.exec(text);
|
|
63
67
|
if (!start) continue;
|
|
@@ -92,6 +96,7 @@ export function loadAliases(projectDir) {
|
|
|
92
96
|
|
|
93
97
|
const pkgPath = join(projectDir, 'package.json');
|
|
94
98
|
if (existsSync(pkgPath)) {
|
|
99
|
+
configFiles.add(pkgPath);
|
|
95
100
|
try {
|
|
96
101
|
const imports = JSON.parse(readFileSync(pkgPath, 'utf8')).imports ?? {};
|
|
97
102
|
for (const [pattern, target] of Object.entries(imports)) {
|
|
@@ -100,8 +105,13 @@ export function loadAliases(projectDir) {
|
|
|
100
105
|
}
|
|
101
106
|
} catch {}
|
|
102
107
|
}
|
|
103
|
-
|
|
104
|
-
|
|
108
|
+
const context = { rules, configFiles: [...configFiles].map((file) => realpathSync(file)) };
|
|
109
|
+
aliasCache.set(projectDir, context);
|
|
110
|
+
return context;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function loadAliases(projectDir) {
|
|
114
|
+
return loadAliasContext(projectDir).rules;
|
|
105
115
|
}
|
|
106
116
|
|
|
107
117
|
/** Absolute candidate bases an aliased specifier could mean, or [] if it matches no rule. */
|
|
@@ -163,6 +173,20 @@ export function fileImports(projectDir, absFile, targetRel) {
|
|
|
163
173
|
return false;
|
|
164
174
|
}
|
|
165
175
|
|
|
176
|
+
/**
|
|
177
|
+
* Symbol-aware import relation for discovery and proof callers. Unlike the
|
|
178
|
+
* compatibility boolean above, uncertainty stays explicit and the result
|
|
179
|
+
* carries the barrel witnesses and configuration/source dependencies used.
|
|
180
|
+
*/
|
|
181
|
+
export function fileImportRelation(projectDir, absFile, targetRel, { limits } = {}) {
|
|
182
|
+
const { rules, configFiles } = loadAliasContext(projectDir);
|
|
183
|
+
return createImportResolver(projectDir, {
|
|
184
|
+
aliases: rules,
|
|
185
|
+
configFiles,
|
|
186
|
+
...(limits ? { limits } : {}),
|
|
187
|
+
}).relation(absFile, targetRel);
|
|
188
|
+
}
|
|
189
|
+
|
|
166
190
|
/**
|
|
167
191
|
* Number of non-test source files that import `targetRel`: relative specifiers,
|
|
168
192
|
* tsconfig/jsconfig `paths` aliases and package.json `imports` are resolved;
|
|
@@ -201,4 +225,3 @@ export function rank({ severity, sourceKind, blast, independence }) {
|
|
|
201
225
|
const score = SEVERITY_WEIGHT[severity] * (SOURCE_WEIGHT[sourceKind] ?? 0.9) * (1 + Math.log2(1 + blast)) * (INDEPENDENCE_WEIGHT[independence] ?? 1);
|
|
202
226
|
return { score: Number(score.toFixed(3)), blastRadius: blast, tier: severity };
|
|
203
227
|
}
|
|
204
|
-
|