testguard-cli 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +169 -14
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +316 -0
- package/spec/README.md +27 -2
- package/spec/lib/authoring-input.mjs +40 -0
- package/spec/lib/origins.mjs +56 -0
- package/spec/lib/validate.mjs +123 -1
- package/spec/schemas/annotations.schema.json +65 -0
- package/spec/schemas/authoring-input.schema.json +67 -0
- package/spec/schemas/brief.schema.json +2 -0
- package/spec/schemas/claims.schema.json +2 -8
- package/spec/schemas/common.schema.json +73 -0
- package/spec/schemas/evidence.schema.json +40 -10
- package/spec/schemas/status.schema.json +6 -2
- package/src/brief/brief.mjs +11 -4
- package/src/claims/annotation-placement.mjs +357 -0
- package/src/claims/annotations.mjs +61 -2
- package/src/cli.mjs +41 -6
- package/src/commands/admit.mjs +13 -0
- package/src/commands/annotations.mjs +67 -0
- package/src/commands/claims.mjs +8 -4
- package/src/commands/probe.mjs +50 -7
- package/src/commands/replay.mjs +7 -0
- package/src/commands/scaffold.mjs +49 -2
- package/src/commands/sweep.mjs +7 -0
- package/src/evidence/writer.mjs +12 -4
- package/src/git.mjs +4 -4
- package/src/init/templates/SKILL.md +35 -0
- package/src/probe/attribution.mjs +10 -0
- package/src/probe/contention.mjs +20 -2
- package/src/probe/cost.mjs +6 -1
- package/src/probe/inject.mjs +2 -6
- package/src/probe/origin-policy.mjs +69 -0
- package/src/probe/policy-freshness.mjs +74 -0
- package/src/probe/probe.mjs +49 -24
- package/src/probe/runners/discovery.mjs +4 -0
- package/src/probe/runners/index.mjs +3 -1
- package/src/probe/runners/jest.mjs +2 -2
- package/src/probe/runners/lifecycle.mjs +58 -0
- package/src/probe/runners/node/entry.mjs +11 -0
- package/src/probe/runners/node/report.mjs +66 -0
- package/src/probe/runners/node/runner.mjs +42 -0
- package/src/probe/runners/node-test.mjs +130 -0
- package/src/probe/runners/playwright.mjs +6 -2
- package/src/probe/runners/python.mjs +10 -4
- package/src/probe/runners/shared.mjs +14 -7
- package/src/probe/runners/vitest.mjs +3 -3
- package/src/probe/universe.mjs +47 -0
- package/src/probe/worktree.mjs +3 -1
- package/src/render.mjs +17 -1
- package/src/replay/replay.mjs +8 -4
- package/src/scaffold/admission.mjs +117 -0
- package/src/scaffold/append.mjs +41 -0
- package/src/scaffold/apply-append.mjs +71 -0
- package/src/scaffold/fix-input.mjs +119 -0
- package/src/scaffold/fix-paths.mjs +90 -0
- package/src/scaffold/intent.mjs +2 -0
- package/src/scaffold/open-append.mjs +44 -0
- package/src/scaffold/plan-append.mjs +37 -0
- package/src/scaffold/plan-intent-input.mjs +38 -0
- package/src/scaffold/prepare-append.mjs +46 -0
- package/src/scaffold/request.mjs +30 -0
- package/src/scaffold/scaffold.mjs +23 -10
- package/src/status/status.mjs +34 -9
- package/src/sweep/pool.mjs +33 -0
- package/src/sweep/sweep.mjs +9 -4
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { CLAIM_SOURCE_KINDS } from '../../spec/lib/origins.mjs';
|
|
4
|
+
import { sameClaimMetadata, subjectOf } from './attribution.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
|
|
7
|
+
/** Admit the planned probe option before loading claims or executing/writing. */
|
|
8
|
+
export function originPolicyRequest(values, command = 'probe') {
|
|
9
|
+
const raw = values['require-origin'];
|
|
10
|
+
if (raw === undefined) return undefined;
|
|
11
|
+
if (command !== 'probe') throw new TypeError('--require-origin is only valid on probe');
|
|
12
|
+
if (!Array.isArray(raw) || !raw.length || raw.some((v) => typeof v !== 'string')) {
|
|
13
|
+
throw new TypeError('--require-origin must name declared source kinds');
|
|
14
|
+
}
|
|
15
|
+
const kinds = raw.flatMap((v) => v.split(',').map((k) => k.trim()));
|
|
16
|
+
if (kinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) throw new TypeError('--require-origin contains an empty or unsupported kind');
|
|
17
|
+
if (values.claim !== undefined || values['allow-empty'] || values.ref !== undefined || values['ignore-dirty']) {
|
|
18
|
+
throw new TypeError('--require-origin needs a complete current-project run without --claim, --allow-empty, --ref or --ignore-dirty');
|
|
19
|
+
}
|
|
20
|
+
const n = Number(values.confirm ?? 3);
|
|
21
|
+
if (!Number.isInteger(n) || n < 3) throw new TypeError('--require-origin requires --confirm of at least 3');
|
|
22
|
+
return [...new Set(kinds)].sort();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Internal policy only: fresh must come from the existing input-binding checker. */
|
|
26
|
+
export function evaluateOriginPolicy({ claims, evidence, eligibleKinds, fresh }) {
|
|
27
|
+
if (eligibleKinds === undefined) return undefined;
|
|
28
|
+
if (!Array.isArray(eligibleKinds) || !eligibleKinds.length || eligibleKinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) {
|
|
29
|
+
throw new TypeError('origin policy needs a nonempty set of declared source kinds');
|
|
30
|
+
}
|
|
31
|
+
const eligible = [...new Set(eligibleKinds)].sort();
|
|
32
|
+
const reasons = new Set();
|
|
33
|
+
const claimsValid = validate('claims', claims).ok;
|
|
34
|
+
const evidenceValid = validate('evidence', evidence).ok;
|
|
35
|
+
if (!claimsValid) reasons.add('invalid-claims');
|
|
36
|
+
if (!evidenceValid) reasons.add('invalid-evidence');
|
|
37
|
+
if (fresh !== true) reasons.add('stale-inputs');
|
|
38
|
+
const current = claimsValid ? claims.claims : [];
|
|
39
|
+
const records = evidenceValid ? evidence.records : [];
|
|
40
|
+
const key = (claimId, faultId) => JSON.stringify([claimId, faultId]);
|
|
41
|
+
const expected = new Map(current.flatMap((claim) => claim.faults.map((fault) => [key(claim.id, fault.id), { claim, fault }])));
|
|
42
|
+
if (!current.length || !expected.size) reasons.add('empty-universe');
|
|
43
|
+
if (evidenceValid && (evidence.run.provisional === true || evidence.run.confirmRuns < 3)) reasons.add('provisional-evidence');
|
|
44
|
+
if (evidenceValid && methodOf(evidence.run) !== 'fault-injection') reasons.add('incompatible-method');
|
|
45
|
+
if (current.some((c) => c.faults.some((f) => methodOf(f) !== 'fault-injection'))) reasons.add('incompatible-method');
|
|
46
|
+
const seen = new Set();
|
|
47
|
+
for (const record of records) {
|
|
48
|
+
const identity = key(record.claim.id, record.subject.id);
|
|
49
|
+
if (seen.has(identity)) reasons.add('duplicate-record');
|
|
50
|
+
seen.add(identity);
|
|
51
|
+
const pair = expected.get(identity);
|
|
52
|
+
if (!pair) { reasons.add('extra-record'); continue; }
|
|
53
|
+
if (!sameClaimMetadata(record.claim, pair.claim)) reasons.add('claim-metadata-mismatch');
|
|
54
|
+
if (methodOf(pair.fault) !== 'fault-injection') continue;
|
|
55
|
+
const subject = subjectOf(pair.fault, sha256);
|
|
56
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash'].some((field) => record.subject[field] !== subject[field])
|
|
57
|
+
|| !isDeepStrictEqual(record.subject.producedBy, subject.producedBy)) reasons.add('fault-mismatch');
|
|
58
|
+
}
|
|
59
|
+
if ([...expected.keys()].some((identity) => !seen.has(identity))) reasons.add('missing-record');
|
|
60
|
+
const ineligibleClaims = current.filter((c) => !eligible.includes(c.source.kind)).map((c) => c.id).sort();
|
|
61
|
+
const nonKilledFaults = [...new Map(records.filter((r) => r.verdict !== 'killed').map((r) =>
|
|
62
|
+
[key(r.claim.id, r.subject.id), { claimId: r.claim.id, faultId: r.subject.id }])).values()]
|
|
63
|
+
.sort((a, b) => a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId));
|
|
64
|
+
return {
|
|
65
|
+
state: reasons.size ? 'unavailable' : ineligibleClaims.length || nonKilledFaults.length ? 'failed' : 'passed',
|
|
66
|
+
eligibleKinds: eligible, claims: current.length, faults: expected.size,
|
|
67
|
+
ineligibleClaims, nonKilledFaults, unavailableReasons: [...reasons].sort(),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join, relative, sep } from 'node:path';
|
|
3
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
import { defenderSelection, isReusable, subjectOf } from './attribution.mjs';
|
|
7
|
+
import { discoverDefendersDetailed } from './discover.mjs';
|
|
8
|
+
import { resolveDefenders } from './runners/shared.mjs';
|
|
9
|
+
import { hashNativeTestUniverse } from './universe.mjs';
|
|
10
|
+
|
|
11
|
+
// Point-in-time reads, not a transaction over the whole project. Never turn a
|
|
12
|
+
// missing file into the empty-file hash that diagnostic probe records can use.
|
|
13
|
+
function currentHash(projectDir, file) {
|
|
14
|
+
if (typeof file !== 'string' || isAbsolute(file) || file.split(/[\\/]/).includes('..')) throw new Error('unsafe input path');
|
|
15
|
+
const path = join(projectDir, file);
|
|
16
|
+
const resolved = realpathSync(path);
|
|
17
|
+
const inside = relative(realpathSync(projectDir), resolved);
|
|
18
|
+
if (isAbsolute(inside) || inside === '..' || inside.startsWith(`..${sep}`)) throw new Error('input escaped project');
|
|
19
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
20
|
+
try {
|
|
21
|
+
const before = fstatSync(fd);
|
|
22
|
+
if (!before.isFile()) throw new Error('input is not a regular file');
|
|
23
|
+
const hash = sha256(readFileSync(fd));
|
|
24
|
+
const after = fstatSync(fd);
|
|
25
|
+
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || realpathSync(path) !== resolved) throw new Error('input changed during read');
|
|
26
|
+
return hash;
|
|
27
|
+
} finally { closeSync(fd); }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Internal policy boundary: native listing plus current per-fault bindings.
|
|
31
|
+
* Legacy/static universes and any uncertainty refuse; verdicts remain intact.
|
|
32
|
+
*/
|
|
33
|
+
export function originPolicyInputsFresh({ projectDir, claims, evidence, universe }) {
|
|
34
|
+
try {
|
|
35
|
+
if (!validate('claims', claims).ok || !validate('evidence', evidence).ok) return false;
|
|
36
|
+
if (methodOf(evidence.run) !== 'fault-injection' || evidence.run.provisional || evidence.run.confirmRuns < 3) return false;
|
|
37
|
+
const testUniverseHash = hashNativeTestUniverse(universe.manifests);
|
|
38
|
+
if (universe.testUniverseHash !== testUniverseHash) return false;
|
|
39
|
+
const matchesRunner = (recorded, current) => recorded?.name === current?.name && recorded?.version === current?.version;
|
|
40
|
+
if (!universe.primaryRunner || !matchesRunner(evidence.run.runner, universe.primaryRunner)) return false;
|
|
41
|
+
if ((evidence.run.runners ?? []).some((recorded) => !universe.runners?.some((current) => matchesRunner(recorded, current)))) return false;
|
|
42
|
+
const allTests = Object.freeze([...new Set([...universe.manifests.values()].flatMap((m) => m.files))].sort());
|
|
43
|
+
const expected = new Map();
|
|
44
|
+
for (const claim of claims.claims) for (const fault of claim.faults) {
|
|
45
|
+
if (methodOf(fault) !== 'fault-injection') return false;
|
|
46
|
+
expected.set(JSON.stringify([claim.id, fault.id]), { claim, fault });
|
|
47
|
+
}
|
|
48
|
+
if (!expected.size || evidence.records.length !== expected.size) return false;
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
for (const record of evidence.records) {
|
|
51
|
+
const key = JSON.stringify([record.claim.id, record.subject.id]);
|
|
52
|
+
const current = expected.get(key);
|
|
53
|
+
if (!current || seen.has(key)) return false;
|
|
54
|
+
seen.add(key);
|
|
55
|
+
const { claim, fault } = current;
|
|
56
|
+
const subject = subjectOf(fault, sha256);
|
|
57
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash', 'producedBy'].some((field) => !isDeepStrictEqual(record.subject[field], subject[field]))) return false;
|
|
58
|
+
const { requested, selectionSource } = defenderSelection(claim, fault);
|
|
59
|
+
const discovery = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, allTests);
|
|
60
|
+
if (discovery?.indeterminate.length) return false;
|
|
61
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discovery.canDetect;
|
|
62
|
+
if (resolved.some((file) => !allTests.includes(file))) return false;
|
|
63
|
+
const hashes = (files) => Object.fromEntries(files.map((file) => [file, currentHash(projectDir, file)]));
|
|
64
|
+
const inputs = {
|
|
65
|
+
targetHash: currentHash(projectDir, fault.file),
|
|
66
|
+
defenderHashes: hashes(resolved),
|
|
67
|
+
testUniverseHash,
|
|
68
|
+
discoveryHashes: hashes(discovery?.dependencies ?? []),
|
|
69
|
+
};
|
|
70
|
+
if (!isReusable(record, { claim, requested, resolved, selectionSource, contentHash: subject.contentHash, inputs })) return false;
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
} catch { return false; }
|
|
74
|
+
}
|
package/src/probe/probe.mjs
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { resetInterpreterCache } from './runners/python.mjs';
|
|
2
|
+
import { assertNotCancelled } from './runners/lifecycle.mjs';
|
|
3
|
+
import { performance } from 'node:perf_hooks';
|
|
1
4
|
import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
|
|
2
5
|
import { gitRaw } from '../git.mjs';
|
|
3
6
|
import { createRequire } from 'node:module';
|
|
@@ -16,7 +19,9 @@ import { classifyIndependence } from './independence.mjs';
|
|
|
16
19
|
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
|
|
17
20
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
18
21
|
import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
22
|
+
import { recordedOriginSummary } from '../../spec/lib/origins.mjs';
|
|
19
23
|
import { persistenceSignalsFor } from '../supply/persistence.mjs';
|
|
24
|
+
import { collectOwnedManifests, hashNativeTestUniverse } from './universe.mjs';
|
|
20
25
|
|
|
21
26
|
|
|
22
27
|
/** Is `child` the same file as `root`, or under it? Both must already be real paths. */
|
|
@@ -51,6 +56,12 @@ export function selectClaims(claims, only) {
|
|
|
51
56
|
return only.map((id) => byId.get(id));
|
|
52
57
|
}
|
|
53
58
|
|
|
59
|
+
/** Reuse cannot stand in for a measurement under a different worker policy. */
|
|
60
|
+
export function sameWorkerPolicy(previousRun, { workers = 1, serial = workers === 1, runnerCommand } = {}) {
|
|
61
|
+
if (runnerCommand) return previousRun?.workers === undefined && !previousRun?.serial;
|
|
62
|
+
return previousRun?.workers === (serial ? 1 : workers) && Boolean(previousRun?.serial) === serial;
|
|
63
|
+
}
|
|
64
|
+
|
|
54
65
|
/**
|
|
55
66
|
* Probe every fault of every claim. Returns a spec-conformant evidence
|
|
56
67
|
* document; writing it is the caller's job.
|
|
@@ -63,7 +74,8 @@ export async function probe({
|
|
|
63
74
|
ref = 'HEAD',
|
|
64
75
|
refExplicit = false,
|
|
65
76
|
ignoreDirty = false,
|
|
66
|
-
|
|
77
|
+
workers = 1,
|
|
78
|
+
serial = workers === 1,
|
|
67
79
|
onWarn = () => {},
|
|
68
80
|
budgetMs = 120_000,
|
|
69
81
|
commandBudget,
|
|
@@ -80,6 +92,18 @@ export async function probe({
|
|
|
80
92
|
previous,
|
|
81
93
|
onProgress = () => {},
|
|
82
94
|
}) {
|
|
95
|
+
if (!Number.isSafeInteger(workers) || workers < 1) throw new PreconditionError('workers must be a positive safe integer');
|
|
96
|
+
resetInterpreterCache(); // Cache only within this measurement, never across commands.
|
|
97
|
+
const measuredStart = performance.now();
|
|
98
|
+
const startedAt = new Date().toISOString();
|
|
99
|
+
const measurements = { elapsedMs: 0, runnerMs: 0, overheadMs: 0, runnerInvocations: 0 };
|
|
100
|
+
const timedRun = async (runner, opts) => {
|
|
101
|
+
assertNotCancelled();
|
|
102
|
+
const start = performance.now();
|
|
103
|
+
measurements.runnerInvocations++;
|
|
104
|
+
try { return await runner.run(opts); }
|
|
105
|
+
finally { measurements.runnerMs += performance.now() - start; }
|
|
106
|
+
};
|
|
83
107
|
commandBudget?.assertOpen();
|
|
84
108
|
// realpath: git reports the repository root by its real path (/private/var
|
|
85
109
|
// on macOS, not /var); every relative() below must start from the same place.
|
|
@@ -183,7 +207,6 @@ export async function probe({
|
|
|
183
207
|
const contention = detectContention();
|
|
184
208
|
if (contention.detected) onWarn(contentionWarning(contention));
|
|
185
209
|
|
|
186
|
-
const startedAt = new Date().toISOString();
|
|
187
210
|
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
188
211
|
const isoReal = realpathSync(iso.projectDir);
|
|
189
212
|
const records = [];
|
|
@@ -218,20 +241,16 @@ export async function probe({
|
|
|
218
241
|
// project runner is; it must resolve before the first such defender runs.
|
|
219
242
|
const owned = OWNED_RUNNERS.filter((r) => r !== runner
|
|
220
243
|
&& !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
|
|
221
|
-
|
|
222
|
-
|
|
244
|
+
let ownedChecked = new Map();
|
|
245
|
+
let manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
|
|
223
246
|
if (!commandTemplate) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
} catch (error) {
|
|
232
|
-
throw new PreconditionError(`${r.name}: test discovery failed: ${error.message}`);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
247
|
+
try {
|
|
248
|
+
({ ownedChecked, manifests } = await collectOwnedManifests(runner, primaryManifest, {
|
|
249
|
+
projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs,
|
|
250
|
+
budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
251
|
+
assertOpen: () => commandBudget?.assertOpen(),
|
|
252
|
+
}));
|
|
253
|
+
} catch (error) { throw new PreconditionError(`test discovery failed: ${error.message}`); }
|
|
235
254
|
}
|
|
236
255
|
const ensureOwned = async (r, file) => {
|
|
237
256
|
if (!ownedChecked.has(r)) {
|
|
@@ -265,9 +284,7 @@ export async function probe({
|
|
|
265
284
|
: [...ownerByFile.keys()]).sort());
|
|
266
285
|
const testUniverseHash = commandTemplate
|
|
267
286
|
? sha256(JSON.stringify({ schemaVersion: 1, source: 'custom-command-static', files: allTests }))
|
|
268
|
-
:
|
|
269
|
-
.map(([r, manifest]) => ({ runner: r.name, hash: manifest.testUniverseHash }))
|
|
270
|
-
.sort((a, b) => a.runner.localeCompare(b.runner))));
|
|
287
|
+
: hashNativeTestUniverse(manifests);
|
|
271
288
|
const baselineCache = new Map();
|
|
272
289
|
const discoveryHashCache = new Map();
|
|
273
290
|
// The negative control is charged per (target file, defender set): two
|
|
@@ -277,6 +294,7 @@ export async function probe({
|
|
|
277
294
|
// runner — a `.py` source file is Python's whatever the project runs.
|
|
278
295
|
const fatalEditFor = (file) => runnerFor(iso.projectDir, file, runner).fatalEdit?.();
|
|
279
296
|
const prior = previous && previous.run.confirmRuns === confirmRuns
|
|
297
|
+
&& sameWorkerPolicy(previous.run, { workers, serial, runnerCommand })
|
|
280
298
|
? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
|
|
281
299
|
: new Map();
|
|
282
300
|
const partitionConfigured = (files) => {
|
|
@@ -295,7 +313,7 @@ export async function probe({
|
|
|
295
313
|
// file the interpreter actually loaded (Python) reports it back.
|
|
296
314
|
const runDefenders = async (files, targets = []) => {
|
|
297
315
|
if (commandTemplate) {
|
|
298
|
-
const result = await runner
|
|
316
|
+
const result = await timedRun(runner, { projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, workers, targets });
|
|
299
317
|
commandBudget?.assertOpen();
|
|
300
318
|
return result;
|
|
301
319
|
}
|
|
@@ -304,7 +322,7 @@ export async function probe({
|
|
|
304
322
|
for (const [r, group] of groups) {
|
|
305
323
|
commandBudget?.assertOpen();
|
|
306
324
|
if (r !== runner) await ensureOwned(r, group[0]);
|
|
307
|
-
parts.push(await r
|
|
325
|
+
parts.push(await timedRun(r, { projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, workers, targets }));
|
|
308
326
|
commandBudget?.assertOpen();
|
|
309
327
|
}
|
|
310
328
|
return mergeRuns(parts);
|
|
@@ -373,6 +391,10 @@ export async function probe({
|
|
|
373
391
|
|
|
374
392
|
commandBudget?.assertOpen();
|
|
375
393
|
|
|
394
|
+
assertNotCancelled();
|
|
395
|
+
const dirty = isDirty(root);
|
|
396
|
+
measurements.elapsedMs = performance.now() - measuredStart;
|
|
397
|
+
measurements.overheadMs = measurements.elapsedMs - measurements.runnerMs;
|
|
376
398
|
return {
|
|
377
399
|
schemaVersion: 1,
|
|
378
400
|
tool: { name: 'testguard', version: toolVersion },
|
|
@@ -380,11 +402,13 @@ export async function probe({
|
|
|
380
402
|
id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
|
|
381
403
|
startedAt,
|
|
382
404
|
finishedAt: new Date().toISOString(),
|
|
383
|
-
repo: { head, dirty
|
|
384
|
-
runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
|
|
405
|
+
repo: { head, dirty, ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
|
|
406
|
+
runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}), ...(runnerSource === 'builtin' ? { source: runnerSource } : {}) },
|
|
385
407
|
...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
|
|
386
408
|
confirmRuns,
|
|
387
|
-
|
|
409
|
+
measurements,
|
|
410
|
+
...(!runnerCommand ? { workers: serial ? 1 : workers } : {}),
|
|
411
|
+
...(serial && !runnerCommand ? { serial: true } : {}),
|
|
388
412
|
...(contention.detected ? { contention } : {}),
|
|
389
413
|
...(confirmRuns < 3 ? { provisional: true } : {}),
|
|
390
414
|
// The isolation that was actually created, not the mode that was asked
|
|
@@ -394,6 +418,7 @@ export async function probe({
|
|
|
394
418
|
mode: iso.mode,
|
|
395
419
|
},
|
|
396
420
|
records,
|
|
421
|
+
origins: recordedOriginSummary(records),
|
|
397
422
|
};
|
|
398
423
|
}
|
|
399
424
|
|
|
@@ -520,7 +545,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
520
545
|
const subject = subjectOf(fault, sha256);
|
|
521
546
|
|
|
522
547
|
// Same source, same defenders, same N, same fault: the verdict cannot have changed.
|
|
523
|
-
if (isReusable(prior, { inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
548
|
+
if (isReusable(prior, { claim, inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
524
549
|
return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
525
550
|
}
|
|
526
551
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { registerChild } from './lifecycle.mjs';
|
|
1
2
|
import { spawn } from 'node:child_process';
|
|
2
3
|
import { randomBytes } from 'node:crypto';
|
|
3
4
|
import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, realpathSync, rmSync, statSync } from 'node:fs';
|
|
@@ -155,6 +156,7 @@ export function runDiscoveryProcess({
|
|
|
155
156
|
timeoutMs = DISCOVERY_TIMEOUT_MS,
|
|
156
157
|
maxOutputBytes = MAX_DISCOVERY_OUTPUT_BYTES,
|
|
157
158
|
env = {},
|
|
159
|
+
cleanupOnClose = false,
|
|
158
160
|
}) {
|
|
159
161
|
validateArgv(argv);
|
|
160
162
|
if (typeof projectDir !== 'string' || projectDir.length === 0 || projectDir.includes('\0')) throw new DiscoveryError('projectDir must be a non-empty path');
|
|
@@ -177,6 +179,7 @@ export function runDiscoveryProcess({
|
|
|
177
179
|
return;
|
|
178
180
|
}
|
|
179
181
|
|
|
182
|
+
registerChild(child, () => terminateProcessTree(child));
|
|
180
183
|
const started = Date.now();
|
|
181
184
|
const stdout = [];
|
|
182
185
|
const stderr = [];
|
|
@@ -221,6 +224,7 @@ export function runDiscoveryProcess({
|
|
|
221
224
|
|
|
222
225
|
child.on('close', (code, signal) => {
|
|
223
226
|
clearTimeout(timer);
|
|
227
|
+
if (cleanupOnClose) terminateProcessTree(child);
|
|
224
228
|
if (settled) return;
|
|
225
229
|
if (terminalError) {
|
|
226
230
|
reject(terminalError);
|
|
@@ -2,6 +2,7 @@ import * as vitest from './vitest.mjs';
|
|
|
2
2
|
import * as jest from './jest.mjs';
|
|
3
3
|
import * as playwright from './playwright.mjs';
|
|
4
4
|
import * as pythonRunner from './python.mjs';
|
|
5
|
+
import * as nodeTest from './node-test.mjs';
|
|
5
6
|
import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs } from './discovery.mjs';
|
|
6
7
|
|
|
7
8
|
export const RUNNERS = {
|
|
@@ -14,9 +15,10 @@ export const RUNNERS = {
|
|
|
14
15
|
// evidence means.
|
|
15
16
|
pytest: pythonRunner.pinned('pytest'),
|
|
16
17
|
unittest: pythonRunner.pinned('unittest'),
|
|
18
|
+
'node-test': nodeTest,
|
|
17
19
|
};
|
|
18
20
|
|
|
19
|
-
export const RUNNER_NAMES = ['vitest', 'jest', 'playwright', 'python', 'pytest', 'unittest'];
|
|
21
|
+
export const RUNNER_NAMES = ['vitest', 'jest', 'playwright', 'python', 'pytest', 'unittest', 'node-test'];
|
|
20
22
|
|
|
21
23
|
/**
|
|
22
24
|
* Runners that own files rather than projects: a defender under Playwright's
|
|
@@ -38,8 +38,8 @@ export async function discoverTests({ projectDir, version, timeoutMs, maxOutputB
|
|
|
38
38
|
}
|
|
39
39
|
// --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
|
|
40
40
|
/** The command line, as data. Exported so it is falsifiable without spawning anything. */
|
|
41
|
-
export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
|
|
42
|
-
[...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial ? ['--runInBand'] : []), '--runTestsByPath', ...files];
|
|
41
|
+
export const argvFor = (projectDir, files, outFile, { serial = false, workers = 1 } = {}) =>
|
|
42
|
+
[...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial || workers === 1 ? ['--runInBand'] : [`--maxWorkers=${workers}`]), '--runTestsByPath', ...files];
|
|
43
43
|
export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
|
|
44
44
|
|
|
45
45
|
/**
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/** Own only children and cleanups registered by this TestGuard process. */
|
|
2
|
+
const children = new Map();
|
|
3
|
+
const cleanups = new Map();
|
|
4
|
+
let installed = false;
|
|
5
|
+
let cancelling = false;
|
|
6
|
+
|
|
7
|
+
function cleanup() {
|
|
8
|
+
// Restore mutations before removing isolation, even across successive probes.
|
|
9
|
+
for (const [fn] of [...cleanups].reverse().sort((a, b) => b[1] - a[1])) {
|
|
10
|
+
try { fn(); } catch (error) { process.stderr.write(`cleanup failed: ${error.message}\n`); }
|
|
11
|
+
}
|
|
12
|
+
cleanups.clear();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function install() {
|
|
16
|
+
if (installed) return;
|
|
17
|
+
installed = true;
|
|
18
|
+
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
|
|
19
|
+
process.on(signal, async () => {
|
|
20
|
+
if (cancelling) return;
|
|
21
|
+
cancelling = true;
|
|
22
|
+
const waits = [...children].map(([child, stop]) => {
|
|
23
|
+
const closed = new Promise((resolve) => child.once('close', resolve));
|
|
24
|
+
stop();
|
|
25
|
+
return closed;
|
|
26
|
+
});
|
|
27
|
+
let timer;
|
|
28
|
+
await Promise.race([
|
|
29
|
+
Promise.allSettled(waits),
|
|
30
|
+
new Promise((resolve) => { timer = setTimeout(resolve, 1000); }),
|
|
31
|
+
]);
|
|
32
|
+
clearTimeout(timer);
|
|
33
|
+
cleanup();
|
|
34
|
+
process.exit(130);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
process.on('exit', () => {
|
|
38
|
+
for (const stop of children.values()) stop();
|
|
39
|
+
cleanup();
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function registerChild(child, stop) {
|
|
44
|
+
install();
|
|
45
|
+
children.set(child, stop);
|
|
46
|
+
child.once('close', () => children.delete(child));
|
|
47
|
+
if (cancelling) stop();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function registerCleanup(fn, { priority = 0 } = {}) {
|
|
51
|
+
install();
|
|
52
|
+
cleanups.set(fn, priority);
|
|
53
|
+
return () => cleanups.delete(fn);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function assertNotCancelled() {
|
|
57
|
+
if (cancelling) throw new Error('TestGuard command cancelled; no partial result may be published');
|
|
58
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { appendFileSync, statSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
// Node sets this only in its isolated test workers. Record the entry before
|
|
4
|
+
// evaluating project code; a declaration imported from a helper is not an entry.
|
|
5
|
+
if (process.env.NODE_TEST_CONTEXT?.startsWith('child')) {
|
|
6
|
+
const receipt = JSON.stringify({ file: process.argv[1], pid: process.pid, version: process.versions.node });
|
|
7
|
+
if (Buffer.byteLength(receipt) > 8192 || statSync(process.env.TESTGUARD_NODE_ENTRIES).size + Buffer.byteLength(receipt) > 16 * 1024 * 1024) {
|
|
8
|
+
throw new Error('node-test entry receipts exceeded their byte bound');
|
|
9
|
+
}
|
|
10
|
+
appendFileSync(process.env.TESTGUARD_NODE_ENTRIES, `${receipt},`);
|
|
11
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { isAbsolute, relative, resolve, sep } from 'node:path';
|
|
2
|
+
|
|
3
|
+
const MAX_RESULTS = 100_000;
|
|
4
|
+
const MAX_REPORT_BYTES = 16 * 1024 * 1024;
|
|
5
|
+
const boundedText = (value) => String(value ?? '').slice(0, 1000);
|
|
6
|
+
|
|
7
|
+
/** Keep only structured identity, never arbitrary error objects or assertion values. */
|
|
8
|
+
const errorIdentity = (error) => error == null ? undefined : {
|
|
9
|
+
failureType: boundedText(error.failureType),
|
|
10
|
+
code: boundedText(error.code),
|
|
11
|
+
message: boundedText(error.message ?? error),
|
|
12
|
+
fileExit: Object.hasOwn(error, 'exitCode') || Object.hasOwn(error, 'signal'),
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/** Pure report accumulator. File/suite summaries never count as assertion kills. */
|
|
16
|
+
export function createReport({ projectDir, version, collect = false }) {
|
|
17
|
+
const cases = [];
|
|
18
|
+
const parents = [];
|
|
19
|
+
const pending = new Map();
|
|
20
|
+
let bytes = 0;
|
|
21
|
+
let failed = false;
|
|
22
|
+
let diagnostic = '';
|
|
23
|
+
const add = (item) => {
|
|
24
|
+
bytes += Buffer.byteLength(JSON.stringify(item));
|
|
25
|
+
if (cases.length >= MAX_RESULTS || bytes > MAX_REPORT_BYTES) throw new Error('node-test results exceeded their report bound');
|
|
26
|
+
cases.push(item);
|
|
27
|
+
};
|
|
28
|
+
const accept = ({ type, data }) => {
|
|
29
|
+
if (['test:stderr', 'test:diagnostic'].includes(type)) {
|
|
30
|
+
diagnostic = `${diagnostic}\n${String(data?.message ?? '')}`.slice(-65536);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (!['test:start', 'test:pass', 'test:fail'].includes(type)) return;
|
|
34
|
+
if (!data || typeof data.name !== 'string' || !Number.isSafeInteger(data.nesting) || data.nesting < 0) throw new Error('malformed node-test event identity');
|
|
35
|
+
const file = typeof data.file === 'string' ? resolve(projectDir, data.file) : undefined;
|
|
36
|
+
const names = parents;
|
|
37
|
+
const key = JSON.stringify([file, data.nesting, data.name]);
|
|
38
|
+
if (type === 'test:start') {
|
|
39
|
+
if (pending.size >= MAX_RESULTS || data.nesting > 1000) throw new Error('node-test pending results exceeded their bound');
|
|
40
|
+
pending.set(key, (pending.get(key) ?? 0) + 1);
|
|
41
|
+
names.length = data.nesting;
|
|
42
|
+
names[data.nesting] = data.name;
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (!pending.has(key)) throw new Error('node-test finished a result without its start');
|
|
46
|
+
if (pending.get(key) === 1) pending.delete(key);
|
|
47
|
+
else pending.set(key, pending.get(key) - 1);
|
|
48
|
+
const error = errorIdentity(data.details?.error);
|
|
49
|
+
const skipped = ['skip', 'todo', 'expectFailure'].some((key) => Object.hasOwn(data, key) && data[key] !== false);
|
|
50
|
+
if (type === 'test:fail' && !skipped) failed = true;
|
|
51
|
+
const container = data.nesting === 0 && (isAbsolute(data.name) || file && resolve(projectDir, data.name) === file)
|
|
52
|
+
&& (data.line === undefined || data.line === 1) && (data.column === undefined || data.column === 1);
|
|
53
|
+
const suite = data.details?.type === 'suite' || error?.failureType === 'subtestsFailed';
|
|
54
|
+
const name = [...names.slice(0, data.nesting), data.name].join(' > ');
|
|
55
|
+
const rel = file ? relative(projectDir, file).split(sep).join('/') : '';
|
|
56
|
+
// Refuse ambiguous/escaped attribution rather than invent a killer id.
|
|
57
|
+
const outside = rel === '..' || rel.startsWith('../') || isAbsolute(rel);
|
|
58
|
+
if (!container && (!rel || outside || !name || name.length > 1000)) throw new Error(outside ? 'node-test result attribution is outside the selected project' : 'node-test result has unavailable or unsupported attribution');
|
|
59
|
+
add({ file: rel, name, container, suite, skipped, passed: type === 'test:pass', ...(error ? { error } : {}) });
|
|
60
|
+
};
|
|
61
|
+
const finish = (entries) => {
|
|
62
|
+
if (pending.size) throw new Error('node-test stopped before every started result finished');
|
|
63
|
+
return { schemaVersion: 1, engine: 'node-test', version, collect, complete: true, failed, entries, cases, diagnostic };
|
|
64
|
+
};
|
|
65
|
+
return { accept, finish };
|
|
66
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { appendFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { readBoundedJsonFile } from '../discovery.mjs';
|
|
3
|
+
import { createReport } from './report.mjs';
|
|
4
|
+
|
|
5
|
+
// The tool may itself be called from a Node test worker. This new controller
|
|
6
|
+
// must initialize as a parent, rather than Node's nested-run no-op mode.
|
|
7
|
+
delete process.env.NODE_TEST_CONTEXT;
|
|
8
|
+
const { run } = await import('node:test');
|
|
9
|
+
|
|
10
|
+
const collect = process.env.TESTGUARD_NODE_COLLECT === '1';
|
|
11
|
+
const workers = Number(process.env.TESTGUARD_NODE_WORKERS);
|
|
12
|
+
const files = process.argv.slice(2);
|
|
13
|
+
if (!Number.isSafeInteger(workers) || workers < 1) throw new Error('invalid node-test worker ceiling');
|
|
14
|
+
const report = createReport({ projectDir: process.cwd(), version: process.versions.node, collect });
|
|
15
|
+
// run() on the Node20 floor uses argv for default native collection. The
|
|
16
|
+
// launcher is not a project test entry. Explicit file runs keep their paths.
|
|
17
|
+
process.argv = [process.execPath];
|
|
18
|
+
// The floor predates run({testNamePatterns}) and --test-concurrency. Put the
|
|
19
|
+
// name filter only in CHILD argv, keeping native file containers collectible.
|
|
20
|
+
if (collect) process.execArgv.push('--test-name-pattern=(?!)');
|
|
21
|
+
const cancellation = new AbortController();
|
|
22
|
+
const stream = run({ signal: cancellation.signal, concurrency: workers, ...(files.length ? { files } : {}), ...(collect ? { testNamePatterns: ['(?!)'] } : {}) });
|
|
23
|
+
try {
|
|
24
|
+
for await (const event of stream) report.accept(event);
|
|
25
|
+
appendFileSync(process.env.TESTGUARD_NODE_ENTRIES, 'null]}');
|
|
26
|
+
const entries = readBoundedJsonFile(process.env.TESTGUARD_NODE_ENTRIES).entries;
|
|
27
|
+
if (!Array.isArray(entries) || entries.at(-1) !== null) throw new Error('incomplete node-test entry receipts');
|
|
28
|
+
entries.pop();
|
|
29
|
+
const doc = report.finish(entries);
|
|
30
|
+
const text = JSON.stringify(doc);
|
|
31
|
+
if (Buffer.byteLength(text) > 16 * 1024 * 1024) throw new Error('node-test report exceeded its byte bound');
|
|
32
|
+
if (collect) process.stdout.write(text);
|
|
33
|
+
else writeFileSync(process.env.TESTGUARD_NODE_REPORT, text, { mode: 0o600 });
|
|
34
|
+
if (doc.failed) process.exitCode = 1;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
cancellation.abort();
|
|
37
|
+
stream.destroy();
|
|
38
|
+
// The Node20 floor's test harness can swallow an uncaught controller error.
|
|
39
|
+
// Retain bounded diagnostics while leaving the incomplete report unwritten.
|
|
40
|
+
process.stderr.write(`node-test reporter failed: ${String(error.message ?? error).slice(0, 1000)}\n`);
|
|
41
|
+
process.exitCode = 1;
|
|
42
|
+
}
|