testguard-cli 0.18.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +104 -0
- package/README.md +84 -1097
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +90 -24
- package/spec/README.md +4 -2
- package/spec/lib/origins.mjs +19 -0
- package/spec/lib/validate.mjs +22 -1
- package/spec/schemas/claims.schema.json +1 -15
- package/spec/schemas/common.schema.json +16 -0
- package/spec/schemas/evidence.schema.json +3 -16
- package/spec/schemas/ignore.schema.json +2 -2
- package/spec/schemas/status.schema.json +23 -0
- package/src/brief/brief.mjs +18 -0
- package/src/cli.mjs +34 -10
- package/src/commands/admit.mjs +8 -1
- package/src/commands/baseline.mjs +16 -2
- package/src/commands/brief.mjs +2 -2
- package/src/commands/claims.mjs +1 -1
- package/src/commands/concerns.mjs +2 -10
- package/src/commands/gate.mjs +2 -4
- package/src/commands/init.mjs +12 -2
- package/src/commands/probe.mjs +13 -5
- package/src/commands/replay.mjs +9 -2
- package/src/commands/runner-options.mjs +28 -0
- package/src/commands/sweep.mjs +8 -1
- package/src/gate/changed.mjs +11 -0
- package/src/git.mjs +12 -2
- package/src/init/init.mjs +169 -74
- package/src/probe/attribution.mjs +5 -0
- package/src/probe/contention.mjs +14 -3
- package/src/probe/cost.mjs +7 -1
- package/src/probe/policy-freshness.mjs +5 -1
- package/src/probe/probe.mjs +31 -5
- package/src/probe/runners/python.mjs +19 -5
- package/src/probe/universe.mjs +14 -2
- package/src/replay/replay.mjs +9 -8
- package/src/supply/concerns.mjs +14 -8
- package/src/sweep/sweep.mjs +2 -0
package/src/commands/brief.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown } from '../brief/brief.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown, installLabel } from '../brief/brief.mjs';
|
|
5
5
|
import { computeStatus } from '../status/status.mjs';
|
|
6
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
7
7
|
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
@@ -39,7 +39,7 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
39
39
|
return 3;
|
|
40
40
|
}
|
|
41
41
|
// Where this binary came from, so a stale install is visible in the session-start context.
|
|
42
|
-
const install =
|
|
42
|
+
const install = installLabel(process.argv[1]);
|
|
43
43
|
const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes, install });
|
|
44
44
|
if (!values.text && !values.markdown) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
45
45
|
if (values.markdown) io.out(renderBriefMarkdown({ ...brief, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length, install }).trimEnd());
|
package/src/commands/claims.mjs
CHANGED
|
@@ -41,7 +41,7 @@ export async function claimsCommand({ projectDir, values, suppliedOptions, versi
|
|
|
41
41
|
const drift = reconcile(claims, annotations);
|
|
42
42
|
const advisory = annotationAdvisory(claims, annotations, { customClaims: Boolean(values.claims) });
|
|
43
43
|
const anchorChecks = values['check-anchors']
|
|
44
|
-
? await checkAnchors(projectDir, claims, { python: values.python
|
|
44
|
+
? await checkAnchors(projectDir, claims, { python: values.python || undefined }) // a bare name is a PATH lookup; see explicitInterpreter
|
|
45
45
|
: undefined;
|
|
46
46
|
|
|
47
47
|
// A claim that disappeared is invisible to every other command; compare
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { loadConcerns, renderConcerns } from '../supply/concerns.mjs';
|
|
3
|
-
import { validate } from '../../spec/lib/validate.mjs';
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
5
|
* `testguard concerns [dir]`: the one-sentence scopes a sweep can be aimed by.
|
|
@@ -13,16 +12,9 @@ import { validate } from '../../spec/lib/validate.mjs';
|
|
|
13
12
|
*/
|
|
14
13
|
export async function concernsCommand({ projectDir, values, version }, io) {
|
|
15
14
|
const path = values.concerns ? resolve(values.concerns) : undefined;
|
|
15
|
+
// A file that does not parse or does not conform throws a SpecDocError
|
|
16
|
+
// naming it, which the CLI reports as exit 2: wrong with the FILE.
|
|
16
17
|
const loaded = loadConcerns(projectDir, { path });
|
|
17
|
-
|
|
18
|
-
if (loaded.source) {
|
|
19
|
-
const result = validate('concerns', { schemaVersion: 1, concerns: loaded.raw });
|
|
20
|
-
if (!result.ok) {
|
|
21
|
-
io.err(`${loaded.source} does not conform to the spec:`);
|
|
22
|
-
for (const e of result.errors) io.err(` ${e.path}: ${e.message}`);
|
|
23
|
-
return 2;
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
18
|
if (values.json) {
|
|
27
19
|
io.out(JSON.stringify({ schemaVersion: 1, tool: { name: 'testguard', version }, source: loaded.source, shadowed: loaded.shadowed, concerns: loaded.concerns }, null, 2));
|
|
28
20
|
} else {
|
package/src/commands/gate.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { join, resolve } from 'node:path';
|
|
2
|
-
import { computeChangedGate, resolveChangedRef, renderGate,
|
|
2
|
+
import { computeChangedGate, resolveChangedRef, renderGate, explainExclusions } from '../gate/changed.mjs';
|
|
3
3
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
4
|
|
|
5
5
|
export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.json');
|
|
@@ -11,9 +11,7 @@ export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.jso
|
|
|
11
11
|
*/
|
|
12
12
|
export async function gateCommand({ projectDir, values, version }, io) {
|
|
13
13
|
if (values.explain) {
|
|
14
|
-
|
|
15
|
-
for (const g of DEFAULT_EXCLUDES) io.out(` ${g}`);
|
|
16
|
-
io.out('Non-source files (anything but .js .mjs .cjs .ts .mts .cts .jsx .tsx) are excluded before these patterns apply.');
|
|
14
|
+
for (const line of explainExclusions()) io.out(line);
|
|
17
15
|
return 0;
|
|
18
16
|
}
|
|
19
17
|
const resolved = resolveChangedRef({ explicit: values.changed, projectDir });
|
package/src/commands/init.mjs
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
|
-
import { initProject } from '../init/init.mjs';
|
|
1
|
+
import { initProject, InitUsageError } from '../init/init.mjs';
|
|
2
2
|
|
|
3
3
|
export async function initCommand({ projectDir, values }, io) {
|
|
4
|
-
|
|
4
|
+
let r;
|
|
5
|
+
try {
|
|
6
|
+
r = initProject({ projectDir, force: values.force, here: values.here, ciEvidence: values['ci-evidence'], mcp: values.mcp });
|
|
7
|
+
} catch (e) {
|
|
8
|
+
// A bad option is the caller's to fix, and init has written nothing yet:
|
|
9
|
+
// every option is checked before the first write.
|
|
10
|
+
if (!(e instanceof InitUsageError)) throw e;
|
|
11
|
+
io.err(e.message);
|
|
12
|
+
io.err("Run 'testguard init --help' for its options. Nothing was written.");
|
|
13
|
+
return 3;
|
|
14
|
+
}
|
|
5
15
|
const { done, skipped, warnings, agentRoot, dir, mcpConfig } = r;
|
|
6
16
|
if (values.json) {
|
|
7
17
|
io.out(JSON.stringify({ done, skipped, warnings, agentRoot, dir, ...(mcpConfig ? { mcpConfig } : {}) }, null, 2));
|
package/src/commands/probe.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
|
14
14
|
import { originPolicyRequest, evaluateOriginPolicy } from '../probe/origin-policy.mjs';
|
|
15
15
|
import { readNativeTestUniverse } from '../probe/universe.mjs';
|
|
16
16
|
import { originPolicyInputsFresh } from '../probe/policy-freshness.mjs';
|
|
17
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
17
18
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
18
19
|
|
|
19
20
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -57,6 +58,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
57
58
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
58
59
|
return 3;
|
|
59
60
|
}
|
|
61
|
+
const runner = runnerOptions(values);
|
|
62
|
+
if (runner.error) {
|
|
63
|
+
io.err(runner.error);
|
|
64
|
+
return 3;
|
|
65
|
+
}
|
|
60
66
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
61
67
|
if (values.progress && !isProgressMode(values.progress)) {
|
|
62
68
|
io.err(`--progress must be one of auto, tty, plain, ndjson, none (got ${values.progress})`);
|
|
@@ -113,10 +119,10 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
113
119
|
refExplicit: values.ref !== undefined,
|
|
114
120
|
ignoreDirty: values['ignore-dirty'],
|
|
115
121
|
onWarn: (m) => io.err(`warning: ${m}`),
|
|
116
|
-
runnerCommand:
|
|
122
|
+
runnerCommand: runner.runnerCommand,
|
|
117
123
|
runnerName: values.runner,
|
|
118
124
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
119
|
-
python:
|
|
125
|
+
python: runner.python,
|
|
120
126
|
only,
|
|
121
127
|
escalate: !values['no-escalate'],
|
|
122
128
|
toolVersion: version,
|
|
@@ -139,11 +145,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
139
145
|
let universe;
|
|
140
146
|
// Custom commands carry a static universe, which cannot establish this
|
|
141
147
|
// native policy boundary. Discovery failures are unavailability, not kills.
|
|
142
|
-
if (!
|
|
148
|
+
if (!runner.runnerCommand) {
|
|
143
149
|
try {
|
|
144
150
|
universe = await readNativeTestUniverse({
|
|
145
151
|
projectDir, sourceDir: projectDir, runnerName: values.runner,
|
|
146
|
-
python:
|
|
152
|
+
python: runner.python,
|
|
147
153
|
budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
148
154
|
budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
149
155
|
assertOpen: () => commandBudget?.assertOpen(),
|
|
@@ -187,7 +193,9 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
187
193
|
const killed = evidence.records.filter((r) => r.verdict === 'killed').length;
|
|
188
194
|
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
189
195
|
}
|
|
190
|
-
io.out(renderSummary(evidence.records, evidence.run)
|
|
196
|
+
io.out(renderSummary(evidence.records, evidence.run));
|
|
197
|
+
// Its own line: the summary ends with the origins line, which is not a sentence to append to.
|
|
198
|
+
io.out(baseline ? `${g.new.length} new since baseline, ${g.baselined.length} baselined.` : 'No baseline.');
|
|
191
199
|
if (policy) io.out(renderOriginPolicy(policy, { basis: 'current' }));
|
|
192
200
|
if (values.cost) {
|
|
193
201
|
io.out('');
|
package/src/commands/replay.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { dirname, join, resolve } from 'node:path';
|
|
|
2
2
|
import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
|
|
3
3
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
4
|
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
5
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
5
6
|
|
|
6
7
|
export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
|
|
7
8
|
export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
|
|
@@ -29,6 +30,11 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
29
30
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
30
31
|
return 3;
|
|
31
32
|
}
|
|
33
|
+
const runner = runnerOptions(values);
|
|
34
|
+
if (runner.error) {
|
|
35
|
+
io.err(runner.error);
|
|
36
|
+
return 3;
|
|
37
|
+
}
|
|
32
38
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
33
39
|
|
|
34
40
|
const doc = await replay({
|
|
@@ -39,9 +45,10 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
39
45
|
workers,
|
|
40
46
|
serial: values.serial || workers === 1,
|
|
41
47
|
commandBudget,
|
|
42
|
-
runnerCommand:
|
|
48
|
+
runnerCommand: runner.runnerCommand,
|
|
43
49
|
runnerName: values.runner,
|
|
44
50
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
51
|
+
python: runner.python,
|
|
45
52
|
limit,
|
|
46
53
|
toolVersion: version,
|
|
47
54
|
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
|
|
@@ -55,7 +62,7 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
55
62
|
// Beside the replay document, whatever --out says: the two are one result,
|
|
56
63
|
// and splitting them across directories loses the pairing — and leaves a
|
|
57
64
|
// file behind in a repository the run is only meant to read.
|
|
58
|
-
const calPath = values
|
|
65
|
+
const calPath = values['calibration-out'] ? resolve(values['calibration-out']) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
|
|
59
66
|
// Replay and calibration are one result. Cross the deadline boundary only
|
|
60
67
|
// after both complete documents exist in memory; serialization is then the
|
|
61
68
|
// documented non-interruptible final step, so expiry can never leave half
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { parseCommandTemplate } from '../probe/runners/shared.mjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The runner options every command that runs tests shares — probe, admit,
|
|
5
|
+
* sweep, replay — checked once, before any work. A malformed `--runner-cmd` is
|
|
6
|
+
* a usage error (exit 3) with the reason, never a RangeError surfacing as
|
|
7
|
+
* "this is a bug in testguard"; and an empty value was asked for, so it is
|
|
8
|
+
* refused rather than read as "no command".
|
|
9
|
+
*
|
|
10
|
+
* `python` is passed through as typed. Whether it is a command on PATH or a
|
|
11
|
+
* path is decided in one place (`explicitInterpreter`), the same way for
|
|
12
|
+
* `--python` and `TESTGUARD_PYTHON`.
|
|
13
|
+
*/
|
|
14
|
+
export function runnerOptions(values) {
|
|
15
|
+
const runnerCommand = values['runner-cmd'];
|
|
16
|
+
if (runnerCommand !== undefined) {
|
|
17
|
+
try {
|
|
18
|
+
parseCommandTemplate(runnerCommand);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return { error: error.message };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const python = values.python;
|
|
24
|
+
if (python !== undefined && python.trim() === '') {
|
|
25
|
+
return { error: '--python must name an interpreter: a path such as .venv/bin/python, or a command on PATH such as python3' };
|
|
26
|
+
}
|
|
27
|
+
return { runnerCommand, python };
|
|
28
|
+
}
|
package/src/commands/sweep.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import { sweep, renderSweep, ConcernError } from '../sweep/sweep.mjs';
|
|
|
3
3
|
import { resolveChangedRef } from '../gate/changed.mjs';
|
|
4
4
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
5
5
|
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
6
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
6
7
|
|
|
7
8
|
export const sweepPath = (projectDir) => join(projectDir, '.testguard', 'sweep.json');
|
|
8
9
|
/**
|
|
@@ -46,6 +47,11 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
46
47
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
47
48
|
return 3;
|
|
48
49
|
}
|
|
50
|
+
const runner = runnerOptions(values);
|
|
51
|
+
if (runner.error) {
|
|
52
|
+
io.err(runner.error);
|
|
53
|
+
return 3;
|
|
54
|
+
}
|
|
49
55
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
50
56
|
// A concern that aims at the write surface or at a glob needs no diff, the
|
|
51
57
|
// same as --save-paths. Only a concern that defers to the gate does.
|
|
@@ -85,9 +91,10 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
85
91
|
workers,
|
|
86
92
|
serial: values.serial || workers === 1,
|
|
87
93
|
commandBudget,
|
|
88
|
-
runnerCommand:
|
|
94
|
+
runnerCommand: runner.runnerCommand,
|
|
89
95
|
runnerName: values.runner,
|
|
90
96
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
97
|
+
python: runner.python,
|
|
91
98
|
toolVersion: version,
|
|
92
99
|
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
93
100
|
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
package/src/gate/changed.mjs
CHANGED
|
@@ -45,6 +45,17 @@ export const DEFAULT_EXCLUDES = Object.freeze([
|
|
|
45
45
|
]);
|
|
46
46
|
const DEFAULT_EXCLUDE_RES = DEFAULT_EXCLUDES.map(globToRegExp);
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* What `gate --explain` prints, built from the two lists the gate itself
|
|
50
|
+
* applies. A hand-written copy of the extension list omitted `.py` for as long
|
|
51
|
+
* as the gate had treated Python as source.
|
|
52
|
+
*/
|
|
53
|
+
export const explainExclusions = () => [
|
|
54
|
+
'Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):',
|
|
55
|
+
...DEFAULT_EXCLUDES.map((g) => ` ${g}`),
|
|
56
|
+
`Non-source files (anything but ${[...SOURCE_EXT].join(' ')}) are excluded before these patterns apply.`,
|
|
57
|
+
];
|
|
58
|
+
|
|
48
59
|
/** Whether a source-shaped path is outside TestGuard's claimable module surface. */
|
|
49
60
|
export const isDefaultExcluded = (file) => DEFAULT_EXCLUDE_RES.some((re) => re.test(file));
|
|
50
61
|
|
package/src/git.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import { rmSync } from 'node:fs';
|
|
3
|
-
import { tmpdir } from 'node:os';
|
|
3
|
+
import { devNull, tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
6
|
|
|
@@ -33,7 +33,17 @@ export function isAncestor(dir, ancestor, ref = 'HEAD') {
|
|
|
33
33
|
/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
|
|
34
34
|
export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Leading options for a git command that checks files out into TestGuard's own
|
|
38
|
+
* scratch tree. That tree is the instrument, not a checkout the user made, so
|
|
39
|
+
* their post-checkout hook (husky, lefthook, Git LFS) must not run on it: a
|
|
40
|
+
* failing one fails the whole command after the files are already written, and
|
|
41
|
+
* Git LFS's fails whenever git-lfs is not on PATH. Filters, LFS smudge
|
|
42
|
+
* included, are config, not hooks, and still apply.
|
|
43
|
+
*/
|
|
44
|
+
export const NO_HOOKS = Object.freeze(['-c', `core.hooksPath=${devNull}`]);
|
|
45
|
+
|
|
46
|
+
export const addWorktree = (repo, dest, ref = 'HEAD') => git([...NO_HOOKS, 'worktree', 'add', '--detach', dest, ref], repo);
|
|
37
47
|
|
|
38
48
|
/**
|
|
39
49
|
* A dangling commit holding the working tree exactly as it is — tracked
|