testguard-cli 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +196 -17
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +316 -0
- package/spec/README.md +27 -2
- package/spec/lib/authoring-input.mjs +40 -0
- package/spec/lib/origins.mjs +56 -0
- package/spec/lib/validate.mjs +129 -1
- package/spec/schemas/annotations.schema.json +65 -0
- package/spec/schemas/authoring-input.schema.json +67 -0
- package/spec/schemas/brief.schema.json +2 -0
- package/spec/schemas/claims.schema.json +8 -8
- package/spec/schemas/common.schema.json +73 -0
- package/spec/schemas/evidence.schema.json +41 -8
- package/spec/schemas/status.schema.json +31 -2
- package/src/brief/brief.mjs +11 -4
- package/src/claims/annotation-placement.mjs +357 -0
- package/src/claims/annotations.mjs +61 -2
- package/src/cli.mjs +40 -5
- package/src/commands/admit.mjs +21 -3
- package/src/commands/annotations.mjs +67 -0
- package/src/commands/claims.mjs +46 -6
- package/src/commands/probe.mjs +50 -7
- package/src/commands/replay.mjs +7 -0
- package/src/commands/scaffold.mjs +49 -2
- package/src/commands/sweep.mjs +7 -0
- package/src/evidence/writer.mjs +12 -4
- package/src/gate/changed.mjs +5 -1
- package/src/git.mjs +4 -4
- package/src/init/init.mjs +6 -0
- package/src/init/templates/SKILL.md +78 -0
- package/src/probe/attribution.mjs +19 -0
- package/src/probe/contention.mjs +20 -2
- package/src/probe/cost.mjs +22 -9
- package/src/probe/inject.mjs +2 -6
- package/src/probe/origin-policy.mjs +69 -0
- package/src/probe/policy-freshness.mjs +74 -0
- package/src/probe/probe.mjs +74 -45
- package/src/probe/runners/discovery.mjs +2 -0
- package/src/probe/runners/jest.mjs +2 -2
- package/src/probe/runners/lifecycle.mjs +58 -0
- package/src/probe/runners/playwright.mjs +6 -2
- package/src/probe/runners/python.mjs +10 -4
- package/src/probe/runners/shared.mjs +12 -6
- package/src/probe/runners/vitest.mjs +3 -3
- package/src/probe/universe.mjs +47 -0
- package/src/probe/worktree.mjs +3 -1
- package/src/render.mjs +17 -1
- package/src/replay/replay.mjs +8 -4
- package/src/scaffold/admission.mjs +117 -0
- package/src/scaffold/append.mjs +41 -0
- package/src/scaffold/apply-append.mjs +71 -0
- package/src/scaffold/fix-input.mjs +119 -0
- package/src/scaffold/fix-paths.mjs +90 -0
- package/src/scaffold/intent.mjs +2 -0
- package/src/scaffold/open-append.mjs +44 -0
- package/src/scaffold/plan-append.mjs +37 -0
- package/src/scaffold/plan-intent-input.mjs +38 -0
- package/src/scaffold/prepare-append.mjs +46 -0
- package/src/scaffold/request.mjs +30 -0
- package/src/scaffold/scaffold.mjs +23 -10
- package/src/status/status.mjs +42 -10
- package/src/sweep/pool.mjs +33 -0
- package/src/sweep/sweep.mjs +9 -4
package/src/commands/admit.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
1
2
|
import { existsSync } from 'node:fs';
|
|
2
3
|
import { dirname, join, relative, resolve } from 'node:path';
|
|
3
4
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
@@ -5,6 +6,7 @@ import { probe } from '../probe/probe.mjs';
|
|
|
5
6
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
7
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
8
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
10
|
import { PreconditionError } from '../probe/worktree.mjs';
|
|
9
11
|
import { hintFor } from '../brief/brief.mjs';
|
|
10
12
|
import { formatVerdict, PROVISIONAL_WARNING } from '../render.mjs';
|
|
@@ -44,6 +46,14 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
44
46
|
}
|
|
45
47
|
const confirmRuns = Number(values.confirm);
|
|
46
48
|
const budgetMs = Number(values.budget);
|
|
49
|
+
const commandBudgetMs = parseCommandBudget(values['command-budget']);
|
|
50
|
+
if (commandBudgetMs === null) { io.err('--command-budget must be at least 1000 milliseconds'); return 3; }
|
|
51
|
+
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
52
|
+
const workers = Number(values.workers ?? 1);
|
|
53
|
+
if (!Number.isSafeInteger(workers) || workers < 1) {
|
|
54
|
+
io.err('--workers must be a positive safe integer');
|
|
55
|
+
return 3;
|
|
56
|
+
}
|
|
47
57
|
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
48
58
|
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
49
59
|
return 3;
|
|
@@ -62,11 +72,15 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
62
72
|
|
|
63
73
|
// The named test must be a defender of the claim, declared or discovered;
|
|
64
74
|
// otherwise `killed` below would be someone else's test doing the work.
|
|
65
|
-
const
|
|
66
|
-
|
|
75
|
+
const selections = faults.map((f) => {
|
|
76
|
+
const { requested } = defenderSelection(claim, f);
|
|
77
|
+
return { requested, resolved: requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file) };
|
|
78
|
+
});
|
|
79
|
+
const declared = selections.every((s) => s.requested.length) ? selections[0].requested : null;
|
|
80
|
+
const isDefender = selections.some((s) => s.resolved.includes(testRel));
|
|
67
81
|
if (!isDefender) {
|
|
68
82
|
io.err(declared
|
|
69
|
-
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...
|
|
83
|
+
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...declared, testRel].map((d) => JSON.stringify(d)).join(', ')}] on the selected fault or its inherited claim in testguard.claims.json, then run admit again.`
|
|
70
84
|
: `${testRel} does not import ${[...new Set(faults.map((f) => f.file))].join(' or ')}, so it cannot be discovered as a defender of ${claimId}. Import the module under test, or declare "defendedBy": ["${testRel}"] on the claim.`);
|
|
71
85
|
return 3;
|
|
72
86
|
}
|
|
@@ -79,6 +93,9 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
79
93
|
claims: { ...all, claims: [{ ...claim, faults }] },
|
|
80
94
|
confirmRuns,
|
|
81
95
|
budgetMs,
|
|
96
|
+
commandBudget,
|
|
97
|
+
workers,
|
|
98
|
+
serial: values.serial || workers === 1,
|
|
82
99
|
mode: 'worktree',
|
|
83
100
|
includeDirty: true,
|
|
84
101
|
only: [claimId],
|
|
@@ -91,6 +108,7 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
91
108
|
});
|
|
92
109
|
if (process.stderr.isTTY && !values.quiet && !values.json) process.stderr.write('\r\x1b[K');
|
|
93
110
|
const outPath = values.out ? resolve(values.out) : join(projectDir, '.testguard', 'evidence-partial.json');
|
|
111
|
+
commandBudget?.assertOpen();
|
|
94
112
|
writeSpecDoc('evidence', outPath, evidence);
|
|
95
113
|
|
|
96
114
|
const decision = decideAdmission(evidence.records);
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { planAnnotationPlacement, applyAnnotationPlacement, loadAnnotationClaims } from '../claims/annotation-placement.mjs';
|
|
2
|
+
import { validate } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { SpecDocError } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
/** Authoring is explicitly selected, never an inspection side effect. */
|
|
6
|
+
export async function annotationsCommand({ projectDir, path, values, suppliedOptions = Object.keys(values), version }, io) {
|
|
7
|
+
const allowed = new Set(['claims', 'claim', 'annotate', 'apply', 'json']);
|
|
8
|
+
if (!values.annotate || suppliedOptions.some((key) => !allowed.has(key))) {
|
|
9
|
+
io.err('--apply requires --annotate; annotation authoring cannot combine inspection/output switches');
|
|
10
|
+
return 3;
|
|
11
|
+
}
|
|
12
|
+
const claimIds = (values.claim ?? []).flatMap((value) => value.split(','));
|
|
13
|
+
let claims;
|
|
14
|
+
try {
|
|
15
|
+
claims = loadAnnotationClaims(path);
|
|
16
|
+
if (!validate('claims', claims).ok) throw new Error('claims must conform before annotation placement');
|
|
17
|
+
}
|
|
18
|
+
catch { io.err('annotation input refused: claims could not be safely loaded or validated'); return 2; }
|
|
19
|
+
if (claimIds.some((id) => !id || !claims.claims.some((claim) => claim.id === id))) {
|
|
20
|
+
io.err('--claim must name existing non-empty claim IDs');
|
|
21
|
+
return 3;
|
|
22
|
+
}
|
|
23
|
+
const plan = planAnnotationPlacement({ projectDir, claims, claimIds });
|
|
24
|
+
const doc = {
|
|
25
|
+
schemaVersion: 1, tool: { name: 'testguard', version }, mode: values.apply ? 'apply' : 'preview',
|
|
26
|
+
state: plan.applicable ? 'preview' : 'refused', selected: plan.selectedClaimIds,
|
|
27
|
+
targets: plan.files.map(({ file, claimIds, changed }) => ({ file, claimIds, action: changed ? 'add' : 'unchanged' })),
|
|
28
|
+
refused: plan.refused,
|
|
29
|
+
};
|
|
30
|
+
if (values.apply) {
|
|
31
|
+
doc.state = 'refused';
|
|
32
|
+
doc.outcome = { changed: [], touched: [], unchanged: plan.files.filter((f) => !f.changed).map((f) => f.file), failed: [] };
|
|
33
|
+
if (plan.applicable) {
|
|
34
|
+
try {
|
|
35
|
+
const report = applyAnnotationPlacement({ projectDir, claimsPath: path, plan });
|
|
36
|
+
doc.state = report.state;
|
|
37
|
+
doc.outcome = {
|
|
38
|
+
changed: report.changed, touched: report.touched, unchanged: report.unchanged,
|
|
39
|
+
failed: report.failed.map(publicFailure), recoveryDir: report.recoveryDir,
|
|
40
|
+
lockRelease: { released: report.lockRelease.released, ...(report.lockRelease.reason ? { reason: 'owned lock release could not be verified' } : {}) },
|
|
41
|
+
};
|
|
42
|
+
} catch (error) {
|
|
43
|
+
doc.outcome.failed.push(publicFailure({ file: null, stage: 'preparation' }));
|
|
44
|
+
if (error.recoveryDir) doc.outcome.recoveryDir = error.recoveryDir;
|
|
45
|
+
if (error.lockRelease) doc.outcome.lockRelease = { released: error.lockRelease.released, ...(error.lockRelease.reason ? { reason: 'owned lock release could not be verified' } : {}) };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const checked = validate('annotations', doc);
|
|
50
|
+
if (!checked.ok) throw new SpecDocError(`annotation result does not conform: ${checked.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
51
|
+
if (values.json) io.out(JSON.stringify(doc, null, 2));
|
|
52
|
+
else {
|
|
53
|
+
io.out(`Annotation ${doc.mode}: ${doc.state} — file-level authoring, not verification or intent authentication.`);
|
|
54
|
+
for (const t of doc.targets) io.out(` ${t.action} ${t.file}: ${t.claimIds.join(', ')}`);
|
|
55
|
+
for (const r of doc.refused) io.out(` REFUSED ${r.file}: ${r.reason}`);
|
|
56
|
+
for (const f of doc.outcome?.failed ?? []) io.out(` FAILED ${f.file ?? '(operation)'} [${f.stage}]: ${f.reason}`);
|
|
57
|
+
if (doc.outcome?.recoveryDir) io.out(`Recovery originals retained: ${doc.outcome.recoveryDir}`);
|
|
58
|
+
if (doc.mode === 'preview' && doc.state === 'preview') io.out('No files changed. Review this selection, then rerun with --apply; the plan is recomputed against current files.');
|
|
59
|
+
}
|
|
60
|
+
return doc.state === 'preview' || doc.state === 'applied' ? 0 : 2;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Raw syscall messages may contain absolute source paths. Keep them in the
|
|
64
|
+
// private recovery journal, never the public authoring document.
|
|
65
|
+
function publicFailure({ file, stage }) {
|
|
66
|
+
return { file, stage, reason: `Annotation ${stage} failed; inspect private recovery details when available.` };
|
|
67
|
+
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
|
-
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
3
|
+
import { scanAnnotations, reconcile, annotationAdvisory } from '../claims/annotations.mjs';
|
|
4
4
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
5
5
|
import { discoverDefendersDetailed } from '../probe/discover.mjs';
|
|
6
6
|
import { classifyDefenders } from '../probe/mocks.mjs';
|
|
@@ -10,12 +10,36 @@ import { costReport, renderCost } from '../probe/cost.mjs';
|
|
|
10
10
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
11
11
|
import { existsSync } from 'node:fs';
|
|
12
12
|
import { checkAnchors, renderAnchorChecks } from '../claims/anchors.mjs';
|
|
13
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
14
|
+
import { annotationsCommand } from './annotations.mjs';
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
/** A selection warning is not a verdict: only a fresh probe can defend it. */
|
|
17
|
+
export function defenderNarrowing(projectDir, claims, records) {
|
|
18
|
+
const prior = new Map(records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]));
|
|
19
|
+
const warnings = [];
|
|
20
|
+
for (const c of claims.claims) {
|
|
21
|
+
const inherited = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [];
|
|
22
|
+
for (const f of c.faults) {
|
|
23
|
+
const record = prior.get(`${c.id}/${f.id}`);
|
|
24
|
+
if (!Object.hasOwn(f, 'defendedBy') || record?.verdict !== 'killed') continue;
|
|
25
|
+
const { requested } = defenderSelection(c, f);
|
|
26
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
27
|
+
if (record.defenders.selectionSource === 'fault' && JSON.stringify(record.defenders.requested) === JSON.stringify(requested) && JSON.stringify([...record.defenders.resolved].sort()) === JSON.stringify([...resolved].sort())) continue;
|
|
28
|
+
if (resolved.length < inherited.length && resolved.every((d) => inherited.includes(d))) {
|
|
29
|
+
warnings.push({ claimId: c.id, faultId: f.id, inherited, resolved });
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return warnings;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export async function claimsCommand({ projectDir, values, suppliedOptions, version }, io) {
|
|
15
37
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
38
|
+
if (values.annotate || values.apply) return annotationsCommand({ projectDir, path, values, suppliedOptions, version }, io);
|
|
16
39
|
const claims = loadClaims(path);
|
|
17
40
|
const annotations = scanAnnotations(projectDir);
|
|
18
41
|
const drift = reconcile(claims, annotations);
|
|
42
|
+
const advisory = annotationAdvisory(claims, annotations, { customClaims: Boolean(values.claims) });
|
|
19
43
|
const anchorChecks = values['check-anchors']
|
|
20
44
|
? await checkAnchors(projectDir, claims, { python: values.python ? resolve(values.python) : undefined })
|
|
21
45
|
: undefined;
|
|
@@ -28,24 +52,39 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
28
52
|
|
|
29
53
|
// Cost is read back out of evidence the probe already wrote; it never runs a test.
|
|
30
54
|
let cost;
|
|
55
|
+
const ev = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
56
|
+
const hasOverrides = claims.claims.some((c) => c.faults.some((f) => Object.hasOwn(f, 'defendedBy')));
|
|
57
|
+
const prior = (values.cost || hasOverrides) && existsSync(ev) ? readSpecDoc('evidence', ev) : undefined;
|
|
58
|
+
const narrowedDefenders = hasOverrides && prior ? defenderNarrowing(projectDir, claims, prior.records) : [];
|
|
31
59
|
if (values.cost) {
|
|
32
|
-
|
|
33
|
-
cost = existsSync(ev) ? costReport(readSpecDoc('evidence', ev).records) : undefined;
|
|
60
|
+
cost = prior ? costReport(prior.records, { run: prior.run }) : undefined;
|
|
34
61
|
if (!cost && !values.json) io.err(`no evidence at ${ev} — run \`testguard probe\` first; cost is derived from the run durations it records`);
|
|
35
62
|
}
|
|
36
63
|
|
|
37
64
|
if (values.json) {
|
|
38
|
-
io.out(JSON.stringify({ path, claims, annotations, drift, ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
65
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, annotationAdvisory: advisory, ...(hasOverrides ? { narrowedDefenders } : {}), ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
39
66
|
} else {
|
|
40
67
|
const annotated = new Set(drift.annotated);
|
|
41
68
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
42
69
|
io.out('');
|
|
70
|
+
for (const note of advisory.notes) io.out(note);
|
|
43
71
|
const signalLines = [];
|
|
44
72
|
for (const c of claims.claims) {
|
|
45
73
|
const declared = c.defendedBy?.length > 0;
|
|
46
74
|
const targets = [...new Set(c.faults.map((f) => f.file))];
|
|
47
75
|
let cover;
|
|
48
|
-
if (
|
|
76
|
+
if (c.faults.some((f) => Object.hasOwn(f, 'defendedBy'))) {
|
|
77
|
+
const selected = new Set();
|
|
78
|
+
for (const f of c.faults) {
|
|
79
|
+
const selection = defenderSelection(c, f);
|
|
80
|
+
const defenders = selection.requested.length ? resolveDefenders(projectDir, selection.requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
81
|
+
defenders.forEach((d) => selected.add(d));
|
|
82
|
+
const m = classifyDefenders(projectDir, f.file, defenders);
|
|
83
|
+
for (const s of m.signals) signalLines.push({ claim: c.id, target: f.file, ...s });
|
|
84
|
+
io.out(` ${c.id}/${f.id}: ${selection.selectionSource}${selection.requested.length ? '' : ' (discovery)'} — ${defenders.join(', ') || 'NO DEFENDER'}`);
|
|
85
|
+
}
|
|
86
|
+
cover = `${selected.size} selected defender${selected.size === 1 ? '' : 's'}`;
|
|
87
|
+
} else if (declared) {
|
|
49
88
|
const defenders = resolveDefenders(projectDir, c.defendedBy);
|
|
50
89
|
const mocking = new Set();
|
|
51
90
|
for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
@@ -70,6 +109,7 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
70
109
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
71
110
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
|
72
111
|
for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
|
|
112
|
+
for (const w of narrowedDefenders) io.out(`NARROWED-DEFENDERS ${w.claimId}/${w.faultId} selects ${w.resolved.length} of ${w.inherited.length} inherited defenders after a prior kill — re-probe; the old kill does not prove the narrower selection.`);
|
|
73
113
|
if (removed) {
|
|
74
114
|
io.out('');
|
|
75
115
|
io.out(renderRemoved(removed));
|
package/src/commands/probe.mjs
CHANGED
|
@@ -4,13 +4,16 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
|
4
4
|
import { probe } from '../probe/probe.mjs';
|
|
5
5
|
import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { gate } from '../baseline/baseline.mjs';
|
|
7
|
-
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
7
|
+
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING, renderOriginPolicy } from '../render.mjs';
|
|
8
8
|
import { computeStatus } from '../status/status.mjs';
|
|
9
9
|
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
10
10
|
import { costReport, renderCost } from '../probe/cost.mjs';
|
|
11
11
|
import { progressMode, stageReporter, clearStageLine, recordEvent, isProgressMode, progressStream } from '../probe/progress.mjs';
|
|
12
12
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
13
13
|
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
14
|
+
import { originPolicyRequest, evaluateOriginPolicy } from '../probe/origin-policy.mjs';
|
|
15
|
+
import { readNativeTestUniverse } from '../probe/universe.mjs';
|
|
16
|
+
import { originPolicyInputsFresh } from '../probe/policy-freshness.mjs';
|
|
14
17
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
15
18
|
|
|
16
19
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -35,8 +38,16 @@ export function partialScope(only, records) {
|
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
export async function probeCommand({ projectDir, values, version }, io) {
|
|
41
|
+
let eligibleKinds;
|
|
42
|
+
try { eligibleKinds = originPolicyRequest(values); }
|
|
43
|
+
catch (e) { io.err(e.message); return 3; }
|
|
38
44
|
const confirmRuns = Number(values.confirm);
|
|
39
45
|
const budgetMs = Number(values.budget);
|
|
46
|
+
const workers = Number(values.workers ?? 1);
|
|
47
|
+
if (!Number.isSafeInteger(workers) || workers < 1) {
|
|
48
|
+
io.err('--workers must be a positive safe integer');
|
|
49
|
+
return 3;
|
|
50
|
+
}
|
|
40
51
|
const commandBudgetMs = parseCommandBudget(values['command-budget']);
|
|
41
52
|
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
42
53
|
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
@@ -94,12 +105,13 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
94
105
|
previous,
|
|
95
106
|
confirmRuns,
|
|
96
107
|
budgetMs,
|
|
108
|
+
workers,
|
|
109
|
+
serial: values.serial || workers === 1,
|
|
97
110
|
commandBudget,
|
|
98
111
|
mode: values['in-place'] ? 'in-place' : 'worktree',
|
|
99
112
|
ref: values.ref ?? 'HEAD',
|
|
100
113
|
refExplicit: values.ref !== undefined,
|
|
101
114
|
ignoreDirty: values['ignore-dirty'],
|
|
102
|
-
serial: values.serial,
|
|
103
115
|
onWarn: (m) => io.err(`warning: ${m}`),
|
|
104
116
|
runnerCommand: values['runner-cmd'],
|
|
105
117
|
runnerName: values.runner,
|
|
@@ -123,17 +135,47 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
123
135
|
},
|
|
124
136
|
});
|
|
125
137
|
commandBudget?.assertOpen();
|
|
138
|
+
if (eligibleKinds) {
|
|
139
|
+
let universe;
|
|
140
|
+
// Custom commands carry a static universe, which cannot establish this
|
|
141
|
+
// native policy boundary. Discovery failures are unavailability, not kills.
|
|
142
|
+
if (!values['runner-cmd']) {
|
|
143
|
+
try {
|
|
144
|
+
universe = await readNativeTestUniverse({
|
|
145
|
+
projectDir, sourceDir: projectDir, runnerName: values.runner,
|
|
146
|
+
python: values.python ? resolve(values.python) : undefined,
|
|
147
|
+
budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
148
|
+
budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
149
|
+
assertOpen: () => commandBudget?.assertOpen(),
|
|
150
|
+
});
|
|
151
|
+
} catch { /* An absent binding can never pass policy. */ }
|
|
152
|
+
}
|
|
153
|
+
commandBudget?.assertOpen();
|
|
154
|
+
const currentClaims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
|
|
155
|
+
const fresh = originPolicyInputsFresh({ projectDir, claims: currentClaims, evidence, universe });
|
|
156
|
+
evidence.originPolicy = evaluateOriginPolicy({ claims: currentClaims, evidence, eligibleKinds, fresh });
|
|
157
|
+
}
|
|
158
|
+
commandBudget?.assertOpen();
|
|
126
159
|
writeSpecDoc('evidence', outPath, evidence);
|
|
127
160
|
|
|
128
161
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
162
|
+
const policy = evidence.originPolicy;
|
|
163
|
+
const exitCode = policy?.state === 'unavailable' ? 2 : policy?.state === 'failed' || g.new.length > 0 ? 1 : 0;
|
|
129
164
|
const scope = partialScope(only, evidence.records);
|
|
130
165
|
if (values.json) {
|
|
131
|
-
const status = withChangedRef(resolveChangedRef({ explicit: values.changed, projectDir }), (changedRef) => computeStatus({
|
|
132
|
-
|
|
166
|
+
const status = withChangedRef(resolveChangedRef({ explicit: values.changed, projectDir }), (changedRef) => computeStatus({
|
|
167
|
+
projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: outPath,
|
|
168
|
+
paths: {
|
|
169
|
+
claims: values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir),
|
|
170
|
+
evidence: outPath, provisional: provisionalEvidencePath(projectDir),
|
|
171
|
+
baseline: values.baseline ? resolve(values.baseline) : baselinePath(projectDir),
|
|
172
|
+
},
|
|
173
|
+
}), io.err);
|
|
174
|
+
const doc = { ...status, ...(policy ? { originPolicy: policy } : {}), run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode, ...(policy ? { originPolicy: policy } : {}), ...(scope ? { scope } : {}) }, ...(values.cost ? { cost: costReport(evidence.records, { run: evidence.run }) } : {}) };
|
|
133
175
|
const result = validate('status', doc);
|
|
134
176
|
if (!result.ok) throw new Error(`probe JSON document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
135
177
|
io.out(JSON.stringify(doc, null, 2));
|
|
136
|
-
return
|
|
178
|
+
return exitCode;
|
|
137
179
|
}
|
|
138
180
|
if (!values.quiet && baseline) {
|
|
139
181
|
io.out('');
|
|
@@ -146,12 +188,13 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
146
188
|
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
147
189
|
}
|
|
148
190
|
io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
|
|
191
|
+
if (policy) io.out(renderOriginPolicy(policy, { basis: 'current' }));
|
|
149
192
|
if (values.cost) {
|
|
150
193
|
io.out('');
|
|
151
|
-
io.out(renderCost(costReport(evidence.records)));
|
|
194
|
+
io.out(renderCost(costReport(evidence.records, { run: evidence.run })));
|
|
152
195
|
io.out('');
|
|
153
196
|
}
|
|
154
197
|
io.out(`evidence: ${outPath}${scope ? ` (partial: ${scope.requestedCount} claim${scope.requestedCount === 1 ? '' : 's'} requested, ${scope.probedCount} probed: --claim ${scope.requestedClaims.join(',')}; not the canonical evidence file)` : provisional ? ' (provisional; not the canonical evidence file)' : ''}`);
|
|
155
198
|
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
156
|
-
return
|
|
199
|
+
return exitCode;
|
|
157
200
|
}
|
package/src/commands/replay.mjs
CHANGED
|
@@ -14,6 +14,11 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
14
14
|
}
|
|
15
15
|
const confirmRuns = Number(values.confirm);
|
|
16
16
|
const budgetMs = Number(values.budget);
|
|
17
|
+
const workers = Number(values.workers ?? 1);
|
|
18
|
+
if (!Number.isSafeInteger(workers) || workers < 1) {
|
|
19
|
+
io.err('--workers must be a positive safe integer');
|
|
20
|
+
return 3;
|
|
21
|
+
}
|
|
17
22
|
const commandBudgetMs = parseCommandBudget(values['command-budget']);
|
|
18
23
|
const limit = values.max ? Number(values.max) : undefined;
|
|
19
24
|
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
@@ -31,6 +36,8 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
31
36
|
range,
|
|
32
37
|
confirmRuns,
|
|
33
38
|
budgetMs,
|
|
39
|
+
workers,
|
|
40
|
+
serial: values.serial || workers === 1,
|
|
34
41
|
commandBudget,
|
|
35
42
|
runnerCommand: values['runner-cmd'],
|
|
36
43
|
runnerName: values.runner,
|
|
@@ -1,11 +1,58 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve, relative, basename, extname } from 'node:path';
|
|
3
3
|
import { scaffoldFile } from '../scaffold/scaffold.mjs';
|
|
4
|
+
import { intentHandoff } from '../scaffold/intent.mjs';
|
|
4
5
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
6
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
7
|
import { PreconditionError } from '../probe/worktree.mjs';
|
|
8
|
+
import { draftAppendRequest, intentInputRequest } from '../scaffold/request.mjs';
|
|
9
|
+
import { planIntentInput } from '../scaffold/plan-intent-input.mjs';
|
|
10
|
+
import { planDraftAppend } from '../scaffold/plan-append.mjs';
|
|
11
|
+
import { applyDraftAppend } from '../scaffold/apply-append.mjs';
|
|
7
12
|
|
|
8
|
-
export async function scaffoldCommand({ projectDir, file, values, version }, io) {
|
|
13
|
+
export async function scaffoldCommand({ projectDir, file, files = file ? [file] : [], values, suppliedOptions = [], version }, io) {
|
|
14
|
+
let input;
|
|
15
|
+
try { input = intentInputRequest({ command: 'scaffold', values, suppliedOptions, files }); }
|
|
16
|
+
catch (error) { io.err(error.message); return 3; }
|
|
17
|
+
if (input) {
|
|
18
|
+
try {
|
|
19
|
+
const plan = planIntentInput({ projectDir, command: 'scaffold', values, suppliedOptions, files, toolVersion: version });
|
|
20
|
+
if (values.json) io.out(plan.output.trimEnd());
|
|
21
|
+
else {
|
|
22
|
+
io.out(`authoring input inspection (${input.kind}); verification not performed`);
|
|
23
|
+
if (input.kind === 'fix') {
|
|
24
|
+
const counts = plan.doc.input.counts;
|
|
25
|
+
io.out(`${counts.total} changed paths: ${counts.supported} supported, ${counts.deleted} deleted, ${counts.unsupported} unsupported, ${counts.excluded} excluded`);
|
|
26
|
+
}
|
|
27
|
+
io.out('Next: supply independent intent, review proposed faults, then probe. Input inspection is not verification.');
|
|
28
|
+
}
|
|
29
|
+
return 0;
|
|
30
|
+
} catch (error) { io.err(`intent input refused: ${error.message}`); return 2; }
|
|
31
|
+
}
|
|
32
|
+
let append;
|
|
33
|
+
try { append = draftAppendRequest({ command: 'scaffold', values, suppliedOptions, files }); }
|
|
34
|
+
catch (error) { io.err(error.message); return 3; }
|
|
35
|
+
if (append) {
|
|
36
|
+
try {
|
|
37
|
+
const request = { ...append, projectDir, toolVersion: version };
|
|
38
|
+
const plan = planDraftAppend(request);
|
|
39
|
+
if (values.json) {
|
|
40
|
+
writeSpecDoc('claims', plan.draft.path, plan.doc, { publish: output => io.out(output.trimEnd()) });
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
43
|
+
const report = applyDraftAppend({ request, plan });
|
|
44
|
+
if (!report.ok) {
|
|
45
|
+
io.err(`draft append ${report.state}: ${report.failed.map(f => `${f.stage}: ${f.reason}`).join('; ')}; write attempted: ${report.writeAttempted}; bytes written: ${report.bytesWritten}`);
|
|
46
|
+
if (report.recoveryDir) io.err(`private recovery: ${report.recoveryDir}`);
|
|
47
|
+
return 2;
|
|
48
|
+
}
|
|
49
|
+
io.out(`unproven draft ${report.state}: ${plan.draft.path} (${plan.stats.appended} appended faults)`);
|
|
50
|
+
if (report.recoveryDir) io.out(`private recovery: ${report.recoveryDir}`);
|
|
51
|
+
io.out('Next: review the proposed faults against supplied intent, then probe the draft. Authoring is not verification.');
|
|
52
|
+
return 0;
|
|
53
|
+
} catch (error) { io.err(`draft append refused: ${error.message}`); return 2; }
|
|
54
|
+
}
|
|
55
|
+
if (files.length > 1) { io.err('ordinary scaffold accepts one source file; use --into for multiple sources'); return 3; }
|
|
9
56
|
if (!file) {
|
|
10
57
|
io.err('usage: testguard scaffold <source-file> [--claim <ID>] [--out <path>] [--json]');
|
|
11
58
|
return 3;
|
|
@@ -38,6 +85,6 @@ export async function scaffoldCommand({ projectDir, file, values, version }, io)
|
|
|
38
85
|
io.out(`${stats.proposals} proposed fault${stats.proposals === 1 ? '' : 's'} in ${stats.claims} draft claim${stats.claims === 1 ? '' : 's'} for ${rel}${shapes ? ` — ${shapes}` : ''}`);
|
|
39
86
|
io.out(stats.defendedBy.length ? `defendedBy prefilled from imports: ${stats.defendedBy.join(', ')}` : 'no test file imports this module; probing the draft as-is will report NOCOVER');
|
|
40
87
|
io.out(`draft: ${outPath}`);
|
|
41
|
-
io.out(
|
|
88
|
+
io.out(`Next: ${intentHandoff} Replace each TODO statement, drop proposals that do not express the intended claim, then review before moving the claims into testguard.claims.json.`);
|
|
42
89
|
return 0;
|
|
43
90
|
}
|
package/src/commands/sweep.mjs
CHANGED
|
@@ -32,6 +32,11 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
32
32
|
const mode = values['save-paths'] ? 'save-paths' : 'changed';
|
|
33
33
|
const confirmRuns = Number(values.confirm);
|
|
34
34
|
const budgetMs = Number(values.budget);
|
|
35
|
+
const workers = Number(values.workers ?? 1);
|
|
36
|
+
if (!Number.isSafeInteger(workers) || workers < 1) {
|
|
37
|
+
io.err('--workers must be a positive safe integer');
|
|
38
|
+
return 3;
|
|
39
|
+
}
|
|
35
40
|
const commandBudgetMs = parseCommandBudget(values['command-budget']);
|
|
36
41
|
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
37
42
|
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
@@ -77,6 +82,8 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
77
82
|
cap,
|
|
78
83
|
confirmRuns,
|
|
79
84
|
budgetMs,
|
|
85
|
+
workers,
|
|
86
|
+
serial: values.serial || workers === 1,
|
|
80
87
|
commandBudget,
|
|
81
88
|
runnerCommand: values['runner-cmd'],
|
|
82
89
|
runnerName: values.runner,
|
package/src/evidence/writer.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { assertNotCancelled } from '../probe/runners/lifecycle.mjs';
|
|
1
2
|
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
|
|
2
3
|
import { dirname } from 'node:path';
|
|
3
4
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
@@ -13,18 +14,25 @@ function describe(kind, path, result) {
|
|
|
13
14
|
* Write a spec document — but only if it conforms. The tool is held to its
|
|
14
15
|
* own spec at the moment of output, not by a separate test someone might skip.
|
|
15
16
|
*/
|
|
16
|
-
export function writeSpecDoc(kind, path, doc) {
|
|
17
|
+
export function writeSpecDoc(kind, path, doc, { publish } = {}) {
|
|
18
|
+
assertNotCancelled();
|
|
17
19
|
const result = validate(kind, doc);
|
|
18
20
|
if (!result.ok) throw new SpecDocError('refusing to write: ' + describe(kind, path, result));
|
|
21
|
+
const output = JSON.stringify(doc, null, 2) + '\n';
|
|
22
|
+
if (publish !== undefined) {
|
|
23
|
+
if (typeof publish !== 'function') throw new TypeError('publish must be a function');
|
|
24
|
+
return publish(output);
|
|
25
|
+
}
|
|
19
26
|
mkdirSync(dirname(path), { recursive: true });
|
|
20
|
-
writeFileSync(path,
|
|
27
|
+
writeFileSync(path, output);
|
|
21
28
|
}
|
|
22
29
|
|
|
23
30
|
/** Read a spec document, refusing one that does not conform. */
|
|
24
|
-
export function readSpecDoc(kind, path) {
|
|
31
|
+
export function readSpecDoc(kind, path, { source } = {}) {
|
|
25
32
|
let doc;
|
|
26
33
|
try {
|
|
27
|
-
|
|
34
|
+
if (source !== undefined && typeof source !== 'string') throw new Error('source must be a string');
|
|
35
|
+
doc = JSON.parse(source === undefined ? readFileSync(path, 'utf8') : source);
|
|
28
36
|
} catch (e) {
|
|
29
37
|
throw new SpecDocError(`cannot read ${kind} document ${path}: ${e.message}`);
|
|
30
38
|
}
|
package/src/gate/changed.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { loadClaims, defaultClaimsPath, ClaimsError } from '../claims/load.mjs';
|
|
|
5
5
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
9
|
import { IS_PY_TEST } from '../probe/pyimports.mjs';
|
|
9
10
|
import { globToRegExp } from '../util/glob.mjs';
|
|
10
11
|
|
|
@@ -228,7 +229,10 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
228
229
|
const defenderFiles = new Map(); // test file → claimIds
|
|
229
230
|
for (const c of claims.claims) {
|
|
230
231
|
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
231
|
-
const defenders =
|
|
232
|
+
const defenders = [...new Set(c.faults.flatMap((f) => {
|
|
233
|
+
const { requested } = defenderSelection(c, f);
|
|
234
|
+
return requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file);
|
|
235
|
+
}))];
|
|
232
236
|
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
233
237
|
}
|
|
234
238
|
|
package/src/git.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { randomBytes } from 'node:crypto';
|
|
|
7
7
|
export class GitError extends Error {}
|
|
8
8
|
|
|
9
9
|
export function gitRaw(args, cwd, env) {
|
|
10
|
-
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: env ? { ...process.env, ...env } : process.env });
|
|
10
|
+
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, maxBuffer: 16 * 1024 * 1024, env: env ? { ...process.env, ...env } : process.env });
|
|
11
11
|
if (r.status !== 0) throw new GitError(`git ${args.join(' ')}: ${(r.stderr || r.stdout).trim()}`);
|
|
12
12
|
return r.stdout;
|
|
13
13
|
}
|
|
@@ -27,7 +27,7 @@ export function headSha(dir, ref = 'HEAD') {
|
|
|
27
27
|
|
|
28
28
|
/** Is `ancestor` reachable from `ref`? False when either does not resolve. */
|
|
29
29
|
export function isAncestor(dir, ancestor, ref = 'HEAD') {
|
|
30
|
-
return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir }).status === 0;
|
|
30
|
+
return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir, timeout: 30_000 }).status === 0;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
|
|
@@ -59,7 +59,7 @@ export function snapshotWorkingTree(repo) {
|
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
export function removeWorktree(repo, dest) {
|
|
62
|
-
spawnSync('git', ['worktree', 'remove', '--force', dest], { cwd: repo });
|
|
62
|
+
spawnSync('git', ['worktree', 'remove', '--force', dest], { cwd: repo, timeout: 30_000 });
|
|
63
63
|
rmSync(dest, { recursive: true, force: true });
|
|
64
|
-
spawnSync('git', ['worktree', 'prune'], { cwd: repo });
|
|
64
|
+
spawnSync('git', ['worktree', 'prune'], { cwd: repo, timeout: 30_000 });
|
|
65
65
|
}
|
package/src/init/init.mjs
CHANGED
|
@@ -56,6 +56,12 @@ Before writing or changing tests, run \`testguard status --json [dir]\` and foll
|
|
|
56
56
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
57
57
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
58
58
|
|
|
59
|
+
Once per AI session, identify the exact CLI used (project-local first, then repository-root, then PATH) and read its \`--version\`.
|
|
60
|
+
If network policy permits, check \`npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000\` once; this is optional AI advice, not a CLI or hook network call.
|
|
61
|
+
Compare stable versions numerically by SemVer; do not downgrade or replace a prerelease/development checkout. If a newer stable version exists, suggest it and ask before updating, respecting pins and the original package manager.
|
|
62
|
+
Never install automatically or alter lockfiles/CI pins without approval. On an unavailable check, continue without claiming up-to-date; keep advice separate from evidence, verdicts and exit codes. Do not repeat a declined suggestion this session.
|
|
63
|
+
Existing copied skills and sections are preserved by ordinary init; review and merge template changes manually, or use \`init --force\` only with approval to overwrite the skill.
|
|
64
|
+
|
|
59
65
|
${lines.join('\n')}
|
|
60
66
|
${AGENTS_END}`;
|
|
61
67
|
}
|