testguard-cli 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +84 -1097
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +72 -24
- package/spec/README.md +4 -2
- package/spec/lib/validate.mjs +22 -1
- package/spec/schemas/evidence.schema.json +2 -2
- package/spec/schemas/ignore.schema.json +2 -2
- package/spec/schemas/status.schema.json +23 -0
- package/src/brief/brief.mjs +18 -0
- package/src/cli.mjs +34 -10
- package/src/commands/admit.mjs +8 -1
- package/src/commands/baseline.mjs +16 -2
- package/src/commands/brief.mjs +2 -2
- package/src/commands/claims.mjs +1 -1
- package/src/commands/concerns.mjs +2 -10
- package/src/commands/gate.mjs +2 -4
- package/src/commands/init.mjs +12 -2
- package/src/commands/probe.mjs +13 -5
- package/src/commands/replay.mjs +9 -2
- package/src/commands/runner-options.mjs +28 -0
- package/src/commands/sweep.mjs +8 -1
- package/src/gate/changed.mjs +11 -0
- package/src/git.mjs +12 -2
- package/src/init/init.mjs +169 -74
- package/src/probe/attribution.mjs +5 -0
- package/src/probe/contention.mjs +14 -3
- package/src/probe/cost.mjs +7 -1
- package/src/probe/policy-freshness.mjs +5 -1
- package/src/probe/probe.mjs +31 -5
- package/src/probe/runners/python.mjs +19 -5
- package/src/probe/universe.mjs +14 -2
- package/src/replay/replay.mjs +9 -8
- package/src/supply/concerns.mjs +14 -8
- package/src/sweep/sweep.mjs +2 -0
package/src/commands/probe.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
|
14
14
|
import { originPolicyRequest, evaluateOriginPolicy } from '../probe/origin-policy.mjs';
|
|
15
15
|
import { readNativeTestUniverse } from '../probe/universe.mjs';
|
|
16
16
|
import { originPolicyInputsFresh } from '../probe/policy-freshness.mjs';
|
|
17
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
17
18
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
18
19
|
|
|
19
20
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -57,6 +58,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
57
58
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
58
59
|
return 3;
|
|
59
60
|
}
|
|
61
|
+
const runner = runnerOptions(values);
|
|
62
|
+
if (runner.error) {
|
|
63
|
+
io.err(runner.error);
|
|
64
|
+
return 3;
|
|
65
|
+
}
|
|
60
66
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
61
67
|
if (values.progress && !isProgressMode(values.progress)) {
|
|
62
68
|
io.err(`--progress must be one of auto, tty, plain, ndjson, none (got ${values.progress})`);
|
|
@@ -113,10 +119,10 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
113
119
|
refExplicit: values.ref !== undefined,
|
|
114
120
|
ignoreDirty: values['ignore-dirty'],
|
|
115
121
|
onWarn: (m) => io.err(`warning: ${m}`),
|
|
116
|
-
runnerCommand:
|
|
122
|
+
runnerCommand: runner.runnerCommand,
|
|
117
123
|
runnerName: values.runner,
|
|
118
124
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
119
|
-
python:
|
|
125
|
+
python: runner.python,
|
|
120
126
|
only,
|
|
121
127
|
escalate: !values['no-escalate'],
|
|
122
128
|
toolVersion: version,
|
|
@@ -139,11 +145,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
139
145
|
let universe;
|
|
140
146
|
// Custom commands carry a static universe, which cannot establish this
|
|
141
147
|
// native policy boundary. Discovery failures are unavailability, not kills.
|
|
142
|
-
if (!
|
|
148
|
+
if (!runner.runnerCommand) {
|
|
143
149
|
try {
|
|
144
150
|
universe = await readNativeTestUniverse({
|
|
145
151
|
projectDir, sourceDir: projectDir, runnerName: values.runner,
|
|
146
|
-
python:
|
|
152
|
+
python: runner.python,
|
|
147
153
|
budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
148
154
|
budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
149
155
|
assertOpen: () => commandBudget?.assertOpen(),
|
|
@@ -187,7 +193,9 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
187
193
|
const killed = evidence.records.filter((r) => r.verdict === 'killed').length;
|
|
188
194
|
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
189
195
|
}
|
|
190
|
-
io.out(renderSummary(evidence.records, evidence.run)
|
|
196
|
+
io.out(renderSummary(evidence.records, evidence.run));
|
|
197
|
+
// Its own line: the summary ends with the origins line, which is not a sentence to append to.
|
|
198
|
+
io.out(baseline ? `${g.new.length} new since baseline, ${g.baselined.length} baselined.` : 'No baseline.');
|
|
191
199
|
if (policy) io.out(renderOriginPolicy(policy, { basis: 'current' }));
|
|
192
200
|
if (values.cost) {
|
|
193
201
|
io.out('');
|
package/src/commands/replay.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { dirname, join, resolve } from 'node:path';
|
|
|
2
2
|
import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
|
|
3
3
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
4
|
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
5
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
5
6
|
|
|
6
7
|
export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
|
|
7
8
|
export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
|
|
@@ -29,6 +30,11 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
29
30
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
30
31
|
return 3;
|
|
31
32
|
}
|
|
33
|
+
const runner = runnerOptions(values);
|
|
34
|
+
if (runner.error) {
|
|
35
|
+
io.err(runner.error);
|
|
36
|
+
return 3;
|
|
37
|
+
}
|
|
32
38
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
33
39
|
|
|
34
40
|
const doc = await replay({
|
|
@@ -39,9 +45,10 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
39
45
|
workers,
|
|
40
46
|
serial: values.serial || workers === 1,
|
|
41
47
|
commandBudget,
|
|
42
|
-
runnerCommand:
|
|
48
|
+
runnerCommand: runner.runnerCommand,
|
|
43
49
|
runnerName: values.runner,
|
|
44
50
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
51
|
+
python: runner.python,
|
|
45
52
|
limit,
|
|
46
53
|
toolVersion: version,
|
|
47
54
|
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
|
|
@@ -55,7 +62,7 @@ export async function replayCommand({ projectDir, values, version }, io) {
|
|
|
55
62
|
// Beside the replay document, whatever --out says: the two are one result,
|
|
56
63
|
// and splitting them across directories loses the pairing — and leaves a
|
|
57
64
|
// file behind in a repository the run is only meant to read.
|
|
58
|
-
const calPath = values
|
|
65
|
+
const calPath = values['calibration-out'] ? resolve(values['calibration-out']) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
|
|
59
66
|
// Replay and calibration are one result. Cross the deadline boundary only
|
|
60
67
|
// after both complete documents exist in memory; serialization is then the
|
|
61
68
|
// documented non-interruptible final step, so expiry can never leave half
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { parseCommandTemplate } from '../probe/runners/shared.mjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The runner options every command that runs tests shares — probe, admit,
|
|
5
|
+
* sweep, replay — checked once, before any work. A malformed `--runner-cmd` is
|
|
6
|
+
* a usage error (exit 3) with the reason, never a RangeError surfacing as
|
|
7
|
+
* "this is a bug in testguard"; and an empty value was asked for, so it is
|
|
8
|
+
* refused rather than read as "no command".
|
|
9
|
+
*
|
|
10
|
+
* `python` is passed through as typed. Whether it is a command on PATH or a
|
|
11
|
+
* path is decided in one place (`explicitInterpreter`), the same way for
|
|
12
|
+
* `--python` and `TESTGUARD_PYTHON`.
|
|
13
|
+
*/
|
|
14
|
+
export function runnerOptions(values) {
|
|
15
|
+
const runnerCommand = values['runner-cmd'];
|
|
16
|
+
if (runnerCommand !== undefined) {
|
|
17
|
+
try {
|
|
18
|
+
parseCommandTemplate(runnerCommand);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return { error: error.message };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const python = values.python;
|
|
24
|
+
if (python !== undefined && python.trim() === '') {
|
|
25
|
+
return { error: '--python must name an interpreter: a path such as .venv/bin/python, or a command on PATH such as python3' };
|
|
26
|
+
}
|
|
27
|
+
return { runnerCommand, python };
|
|
28
|
+
}
|
package/src/commands/sweep.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import { sweep, renderSweep, ConcernError } from '../sweep/sweep.mjs';
|
|
|
3
3
|
import { resolveChangedRef } from '../gate/changed.mjs';
|
|
4
4
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
5
5
|
import { createCommandBudget, parseCommandBudget } from '../command-budget.mjs';
|
|
6
|
+
import { runnerOptions } from './runner-options.mjs';
|
|
6
7
|
|
|
7
8
|
export const sweepPath = (projectDir) => join(projectDir, '.testguard', 'sweep.json');
|
|
8
9
|
/**
|
|
@@ -46,6 +47,11 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
46
47
|
io.err('--command-budget must be at least 1000 milliseconds');
|
|
47
48
|
return 3;
|
|
48
49
|
}
|
|
50
|
+
const runner = runnerOptions(values);
|
|
51
|
+
if (runner.error) {
|
|
52
|
+
io.err(runner.error);
|
|
53
|
+
return 3;
|
|
54
|
+
}
|
|
49
55
|
const commandBudget = createCommandBudget(commandBudgetMs);
|
|
50
56
|
// A concern that aims at the write surface or at a glob needs no diff, the
|
|
51
57
|
// same as --save-paths. Only a concern that defers to the gate does.
|
|
@@ -85,9 +91,10 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
85
91
|
workers,
|
|
86
92
|
serial: values.serial || workers === 1,
|
|
87
93
|
commandBudget,
|
|
88
|
-
runnerCommand:
|
|
94
|
+
runnerCommand: runner.runnerCommand,
|
|
89
95
|
runnerName: values.runner,
|
|
90
96
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
97
|
+
python: runner.python,
|
|
91
98
|
toolVersion: version,
|
|
92
99
|
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
93
100
|
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
package/src/gate/changed.mjs
CHANGED
|
@@ -45,6 +45,17 @@ export const DEFAULT_EXCLUDES = Object.freeze([
|
|
|
45
45
|
]);
|
|
46
46
|
const DEFAULT_EXCLUDE_RES = DEFAULT_EXCLUDES.map(globToRegExp);
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* What `gate --explain` prints, built from the two lists the gate itself
|
|
50
|
+
* applies. A hand-written copy of the extension list omitted `.py` for as long
|
|
51
|
+
* as the gate had treated Python as source.
|
|
52
|
+
*/
|
|
53
|
+
export const explainExclusions = () => [
|
|
54
|
+
'Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):',
|
|
55
|
+
...DEFAULT_EXCLUDES.map((g) => ` ${g}`),
|
|
56
|
+
`Non-source files (anything but ${[...SOURCE_EXT].join(' ')}) are excluded before these patterns apply.`,
|
|
57
|
+
];
|
|
58
|
+
|
|
48
59
|
/** Whether a source-shaped path is outside TestGuard's claimable module surface. */
|
|
49
60
|
export const isDefaultExcluded = (file) => DEFAULT_EXCLUDE_RES.some((re) => re.test(file));
|
|
50
61
|
|
package/src/git.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import { rmSync } from 'node:fs';
|
|
3
|
-
import { tmpdir } from 'node:os';
|
|
3
|
+
import { devNull, tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
6
|
|
|
@@ -33,7 +33,17 @@ export function isAncestor(dir, ancestor, ref = 'HEAD') {
|
|
|
33
33
|
/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
|
|
34
34
|
export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Leading options for a git command that checks files out into TestGuard's own
|
|
38
|
+
* scratch tree. That tree is the instrument, not a checkout the user made, so
|
|
39
|
+
* their post-checkout hook (husky, lefthook, Git LFS) must not run on it: a
|
|
40
|
+
* failing one fails the whole command after the files are already written, and
|
|
41
|
+
* Git LFS's fails whenever git-lfs is not on PATH. Filters, LFS smudge
|
|
42
|
+
* included, are config, not hooks, and still apply.
|
|
43
|
+
*/
|
|
44
|
+
export const NO_HOOKS = Object.freeze(['-c', `core.hooksPath=${devNull}`]);
|
|
45
|
+
|
|
46
|
+
export const addWorktree = (repo, dest, ref = 'HEAD') => git([...NO_HOOKS, 'worktree', 'add', '--detach', dest, ref], repo);
|
|
37
47
|
|
|
38
48
|
/**
|
|
39
49
|
* A dangling commit holding the working tree exactly as it is — tracked
|
package/src/init/init.mjs
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'node:fs';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync, chmodSync } from 'node:fs';
|
|
2
2
|
import { join, dirname, relative, resolve, sep } from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { spawnSync } from 'node:child_process';
|
|
5
|
+
import { PreconditionError } from '../probe/worktree.mjs';
|
|
5
6
|
|
|
6
7
|
const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
|
|
7
8
|
const AGENTS_BEGIN = '<!-- testguard:begin -->';
|
|
8
9
|
const AGENTS_END = '<!-- testguard:end -->';
|
|
9
|
-
|
|
10
|
+
/** Where fetch-ci-evidence.sh downloads CI's artifact, relative to the project. */
|
|
11
|
+
export const CI_DOWNLOAD_DIR = '.testguard/ci';
|
|
12
|
+
const GITIGNORE_HEADER = '# TestGuard: regenerated per run (baseline.json IS committed)';
|
|
13
|
+
// Every regenerated file a command writes under .testguard/. test/init.test.mjs
|
|
14
|
+
// derives that set from the code's own path joins and fails when one is missing
|
|
15
|
+
// here, so a new artifact cannot ship without its ignore line.
|
|
16
|
+
export const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json', '.testguard/replay.json', '.testguard/calibration.json', '.testguard/ci-self-evidence.json', '.testguard/ci/', '.testguard/sweep.json', '.testguard/sweep-evidence.json'];
|
|
10
17
|
|
|
11
18
|
/**
|
|
12
|
-
* The `.testguard/` files that
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
19
|
+
* The `.testguard/` files that are NOT ignored: the frozen contract
|
|
20
|
+
* (baseline.json), the CI-fetch helper `init --ci-evidence` writes for the user
|
|
21
|
+
* to run by hand, and status.json — a name reserved for a status snapshot a
|
|
22
|
+
* project may choose to keep (`status --json > .testguard/status.json`). No
|
|
23
|
+
* command writes status.json; it is listed so that both ignore forms leave it
|
|
24
|
+
* trackable alike: init's per-file lines never name it, and the `.testguard/*`
|
|
25
|
+
* form baseline advises negates it. Everything else in the directory is
|
|
26
|
+
* regenerated. Named here so the two places that reason about the directory —
|
|
27
|
+
* the lines init writes and the advice baseline prints — cannot disagree.
|
|
17
28
|
*/
|
|
18
29
|
export const COMMITTED_OUTPUTS = ['.testguard/baseline.json', '.testguard/status.json', '.testguard/fetch-ci-evidence.sh'];
|
|
19
30
|
|
|
31
|
+
/** A bad option to init: the CLI reports it as a usage error (exit 3). */
|
|
32
|
+
export class InitUsageError extends Error {}
|
|
33
|
+
|
|
20
34
|
/**
|
|
21
35
|
* The session-start hook command. It runs from the git root (where the agent
|
|
22
36
|
* session lives) and must never fetch from the network: a local install of
|
|
@@ -78,6 +92,63 @@ export function isGitIgnored(root, file) {
|
|
|
78
92
|
return spawnSync('git', ['check-ignore', '-q', '--', file], { cwd: root }).status === 0;
|
|
79
93
|
}
|
|
80
94
|
|
|
95
|
+
/**
|
|
96
|
+
* The directory a TestGuard session-start hook briefs, or null when the
|
|
97
|
+
* command is not a TestGuard hook.
|
|
98
|
+
*
|
|
99
|
+
* Recognised by what every release's hook has run — `testguard brief --text`,
|
|
100
|
+
* through `npx -y testguard-cli`, `npx --no-install testguard`, a local
|
|
101
|
+
* `node_modules/.bin/testguard` or a `testguard` on PATH — not by the exact
|
|
102
|
+
* text of the current hookCommand(). Matching only the current text (and two
|
|
103
|
+
* named npx forms) meant any change to hookCommand() made the next `init`
|
|
104
|
+
* append a second hook beside the first, and every session briefed twice.
|
|
105
|
+
* The project is the token after `--text` when it is an argument rather than
|
|
106
|
+
* a redirect or a shell operator.
|
|
107
|
+
*/
|
|
108
|
+
export function briefHookDir(command) {
|
|
109
|
+
if (typeof command !== 'string') return null;
|
|
110
|
+
const m = /(?:^|[\s/])testguard(?:-cli)?(?:@[^\s"|;&]+)?\s+brief\s+--text\b(?:\s+([^\s"|;&]+))?/.exec(command);
|
|
111
|
+
if (!m) return null;
|
|
112
|
+
const token = m[1];
|
|
113
|
+
return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const isPlainObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Read `.claude/settings.json` for the hook, or refuse. A file init cannot
|
|
120
|
+
* safely merge into is a precondition failure naming the file — never a crash,
|
|
121
|
+
* and never after another file has been written.
|
|
122
|
+
*/
|
|
123
|
+
function readSettings(settingsPath) {
|
|
124
|
+
if (!existsSync(settingsPath)) return {};
|
|
125
|
+
let settings;
|
|
126
|
+
try {
|
|
127
|
+
settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
128
|
+
} catch (e) {
|
|
129
|
+
throw new PreconditionError(`${settingsPath} is not valid JSON (${e.message}); fix it before init can add the hook. Nothing was written.`);
|
|
130
|
+
}
|
|
131
|
+
const refuse = (what) => { throw new PreconditionError(`${settingsPath}: ${what}; fix it before init can add the hook. Nothing was written.`); };
|
|
132
|
+
if (!isPlainObject(settings)) refuse('the settings must be a JSON object');
|
|
133
|
+
if (settings.hooks !== undefined && !isPlainObject(settings.hooks)) refuse('"hooks" must be an object');
|
|
134
|
+
if (settings.hooks?.SessionStart !== undefined && !Array.isArray(settings.hooks.SessionStart)) refuse('"hooks.SessionStart" must be an array');
|
|
135
|
+
return settings;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* The `.gitignore` text with init's lines in it, or null when nothing is
|
|
140
|
+
* missing. Lines a previous init wrote stay where they are; lines a newer
|
|
141
|
+
* release added go directly after the last of them, so re-running init after
|
|
142
|
+
* an upgrade extends the one block instead of appending a second header.
|
|
143
|
+
*/
|
|
144
|
+
function gitignoreWithLines(gi, missing) {
|
|
145
|
+
const lines = gi.split('\n');
|
|
146
|
+
let last = -1;
|
|
147
|
+
lines.forEach((l, i) => { if (GITIGNORE_LINES.includes(l.trim())) last = i; });
|
|
148
|
+
if (last === -1) return (gi ? gi.replace(/\s*$/, '\n') : '') + GITIGNORE_HEADER + '\n' + missing.join('\n') + '\n';
|
|
149
|
+
return [...lines.slice(0, last + 1), ...missing, ...lines.slice(last + 1)].join('\n');
|
|
150
|
+
}
|
|
151
|
+
|
|
81
152
|
/**
|
|
82
153
|
* Install the operating layer.
|
|
83
154
|
*
|
|
@@ -93,35 +164,59 @@ export function isGitIgnored(root, file) {
|
|
|
93
164
|
* hook line and an AGENTS.md bullet, never duplicates. Every written file is
|
|
94
165
|
* checked against .gitignore; an ignored file is reported as ignored, never
|
|
95
166
|
* as something to commit.
|
|
167
|
+
*
|
|
168
|
+
* All or nothing: every input is read and every refusal raised before the
|
|
169
|
+
* first write, so a bad option or an unmergeable settings.json leaves the
|
|
170
|
+
* project exactly as it was.
|
|
96
171
|
*/
|
|
97
|
-
const
|
|
98
|
-
github
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
BRANCH
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
172
|
+
const CI_EVIDENCE_HELPER = (platform) => {
|
|
173
|
+
const github = platform === 'github';
|
|
174
|
+
const tool = github ? 'gh' : 'glab';
|
|
175
|
+
const download = github
|
|
176
|
+
? 'gh run download --name "testguard-evidence-$BRANCH" --dir "$OUT"'
|
|
177
|
+
: 'glab ci artifact "$BRANCH" testguard:probe --path "$OUT/"';
|
|
178
|
+
const missing = github ? 'no testguard-evidence-$BRANCH artifact to download' : 'no testguard:probe artifact to download for $BRANCH';
|
|
179
|
+
// Where the evidence sits in the downloaded artifact. The action's
|
|
180
|
+
// `evidence` output names the one file its run wrote, uploaded at the
|
|
181
|
+
// artifact's root; ci-self-evidence.json is what an earlier workflow example
|
|
182
|
+
// uploaded. The GitLab template keeps the project's path from the repository
|
|
183
|
+
// root, so a project in a subdirectory finds its own evidence under that
|
|
184
|
+
// prefix — and never another project's from the same artifact.
|
|
185
|
+
const candidates = github
|
|
186
|
+
? ['evidence.json', 'evidence-provisional.json', 'ci-self-evidence.json']
|
|
187
|
+
: ['${PREFIX}.testguard/evidence.json', '${PREFIX}.testguard/evidence-provisional.json'];
|
|
188
|
+
return `#!/bin/sh
|
|
111
189
|
# Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
|
|
112
190
|
# want it: the session-start hook never touches the network, and this script
|
|
113
191
|
# exits 0 with a message whenever anything is missing.
|
|
114
192
|
BRANCH="\${1:-main}"
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
193
|
+
# Run from the project this helper belongs to, whatever the caller's cwd.
|
|
194
|
+
case "$0" in */*) cd "\${0%/*}/.." || exit 0 ;; *) cd .. || exit 0 ;; esac
|
|
195
|
+
OUT=${CI_DOWNLOAD_DIR}
|
|
196
|
+
command -v ${tool} >/dev/null 2>&1 || { echo "${tool} is not installed; see https://github.com/raccioly/testguard/blob/main/docs/guides/ci/${github ? 'github-actions' : 'gitlab'}.md#read-cis-evidence-locally" >&2; exit 0; }
|
|
197
|
+
# The CLI the session-start hook would run: this project's install, then the
|
|
198
|
+
# repository root's, then a testguard on PATH. Never a package-runner fetch.
|
|
199
|
+
TG=
|
|
200
|
+
ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
|
|
201
|
+
${github ? '' : '# This project\'s path from the repository root, with a trailing slash; empty at the root.\nPREFIX=$(git rev-parse --show-prefix 2>/dev/null)\n'}for c in ./node_modules/.bin/testguard "\${ROOT:+$ROOT/node_modules/.bin/testguard}"; do
|
|
202
|
+
if [ -n "$c" ] && [ -x "$c" ]; then TG=$c; break; fi
|
|
203
|
+
done
|
|
204
|
+
[ -n "$TG" ] || TG=$(command -v testguard 2>/dev/null)
|
|
205
|
+
[ -n "$TG" ] || { echo "no testguard CLI found (./node_modules/.bin, the repository root's node_modules/.bin, or PATH); install it, or run npx testguard-cli brief . --text --evidence <file>" >&2; exit 0; }
|
|
206
|
+
# A fresh download directory, so an earlier download is never briefed as this one.
|
|
207
|
+
rm -rf "$OUT" && mkdir -p "$OUT" || exit 0
|
|
208
|
+
${download} >/dev/null 2>&1 || { echo "${missing}" >&2; exit 0; }
|
|
209
|
+
for f in ${candidates.map((c) => `"$OUT/${c}"`).join(' ')}; do
|
|
210
|
+
[ -f "$f" ] && exec "$TG" brief . --text --evidence "$f"
|
|
211
|
+
done
|
|
212
|
+
echo "the downloaded artifact holds no evidence for this project (looked for ${candidates.join(', ')})" >&2
|
|
213
|
+
exit 0
|
|
214
|
+
`;
|
|
122
215
|
};
|
|
216
|
+
const CI_PLATFORMS = ['github', 'gitlab'];
|
|
123
217
|
|
|
124
218
|
export function initProject({ projectDir, force = false, here = false, ciEvidence, mcp = false }) {
|
|
219
|
+
if (ciEvidence !== undefined && !CI_PLATFORMS.includes(ciEvidence)) throw new InitUsageError(`--ci-evidence must be github or gitlab, not ${JSON.stringify(ciEvidence)}`);
|
|
125
220
|
projectDir = realpathSync(resolve(projectDir));
|
|
126
221
|
const root = here ? null : gitRoot(projectDir);
|
|
127
222
|
const agentRoot = root ?? projectDir;
|
|
@@ -132,65 +227,56 @@ export function initProject({ projectDir, force = false, here = false, ciEvidenc
|
|
|
132
227
|
let mcpConfig;
|
|
133
228
|
const skipped = [];
|
|
134
229
|
const warnings = [];
|
|
230
|
+
const writes = [];
|
|
135
231
|
const rel = (p) => relative(agentRoot, p).split(sep).join('/');
|
|
136
232
|
const note = (p) => { if (isGitIgnored(ignoreRoot, p)) warnings.push(`${rel(p)} is ignored by .gitignore — it was written, but git will not track it and agents in a fresh clone will never see it; unignore it or move it`); };
|
|
233
|
+
const write = (path, content, { mode, checkIgnored = true } = {}) => writes.push({ path, content, mode, checkIgnored });
|
|
234
|
+
|
|
235
|
+
// ---- preflight: read everything, refuse here, write nothing yet ----
|
|
236
|
+
const settingsPath = join(agentRoot, '.claude', 'settings.json');
|
|
237
|
+
const settings = readSettings(settingsPath);
|
|
137
238
|
|
|
138
239
|
const skillDir = join(agentRoot, '.claude', 'skills', 'testguard');
|
|
139
240
|
const skillPath = join(skillDir, 'SKILL.md');
|
|
140
241
|
if (!existsSync(skillPath) || force) {
|
|
141
242
|
const replaced = force && existsSync(skillPath);
|
|
142
|
-
|
|
143
|
-
writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
|
|
243
|
+
write(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
|
|
144
244
|
done.push(`${rel(skillPath)}${replaced ? ' (replaced)' : ''}`);
|
|
145
|
-
note(skillPath);
|
|
146
245
|
} else {
|
|
147
246
|
skipped.push(`${rel(skillPath)} exists (use --force to replace)`);
|
|
148
247
|
}
|
|
149
248
|
|
|
150
|
-
const settingsPath = join(agentRoot, '.claude', 'settings.json');
|
|
151
|
-
let settings = {};
|
|
152
|
-
if (existsSync(settingsPath)) {
|
|
153
|
-
try {
|
|
154
|
-
settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
155
|
-
} catch (e) {
|
|
156
|
-
throw new Error(`${rel(settingsPath)} is not valid JSON (${e.message}); fix it before init can add the hook`);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
249
|
settings.hooks ??= {};
|
|
160
250
|
settings.hooks.SessionStart ??= [];
|
|
251
|
+
const groups = settings.hooks.SessionStart;
|
|
161
252
|
const cmd = hookCommand(dir);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
// that token is an argument rather than a redirect or a shell operator.
|
|
167
|
-
// Testing for any non-space there mistook `brief --text 2>/dev/null` for a
|
|
168
|
-
// hook belonging to a directory, so a legacy root hook was never replaced.
|
|
169
|
-
const hookDir = (text) => {
|
|
170
|
-
const m = /brief --text(?:\s+([^\s"|;&]+))?/.exec(text);
|
|
171
|
-
const token = m?.[1];
|
|
172
|
-
return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
|
|
173
|
-
};
|
|
174
|
-
const legacy = settings.hooks.SessionStart.findIndex((e) => {
|
|
175
|
-
const text = JSON.stringify(e);
|
|
176
|
-
return /npx\s+(-y|--no-install)[^"]*brief --text/.test(text) && hookDir(text) === dir;
|
|
177
|
-
});
|
|
253
|
+
// Every TestGuard hook already briefing this project, whichever release wrote it.
|
|
254
|
+
const ours = [];
|
|
255
|
+
groups.forEach((g, gi) => { if (Array.isArray(g?.hooks)) g.hooks.forEach((h, hi) => { if (briefHookDir(h?.command) === dir) ours.push({ gi, hi, command: h.command }); }); });
|
|
256
|
+
const hasCmd = ours.length === 1 && ours[0].command === cmd;
|
|
178
257
|
if (hasCmd) {
|
|
179
258
|
skipped.push(`SessionStart hook for ${dir === '.' ? 'this project' : dir} already present in ${rel(settingsPath)}`);
|
|
180
259
|
} else {
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
|
|
260
|
+
if (ours.length) {
|
|
261
|
+
// Replace the first in place, keeping its position and any other keys
|
|
262
|
+
// (a timeout, say); drop the rest, which are duplicates an older init left.
|
|
263
|
+
const [first, ...extra] = ours;
|
|
264
|
+
groups[first.gi].hooks[first.hi] = { ...groups[first.gi].hooks[first.hi], command: cmd };
|
|
265
|
+
for (const { gi, hi } of extra.reverse()) groups[gi].hooks.splice(hi, 1);
|
|
266
|
+
settings.hooks.SessionStart = groups.filter((g, gi) => !extra.some((x) => x.gi === gi) || g.hooks.length > 0);
|
|
267
|
+
const npx = ours.some((o) => /\bnpx\b/.test(o.command));
|
|
268
|
+
done.push(npx
|
|
269
|
+
? `${rel(settingsPath)}: replaced the network-reaching npx hook with a local-then-PATH one`
|
|
270
|
+
: `${rel(settingsPath)}: replaced an earlier TestGuard hook for ${dir === '.' ? 'this project' : dir} in place`);
|
|
271
|
+
} else {
|
|
272
|
+
groups.push({ hooks: [{ type: 'command', command: cmd }] });
|
|
184
273
|
}
|
|
185
|
-
|
|
186
|
-
mkdirSync(dirname(settingsPath), { recursive: true });
|
|
187
|
-
writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
274
|
+
write(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
188
275
|
// Describes hookCommand() above, which emits no npx of any kind: a local
|
|
189
276
|
// install, then a testguard on PATH, then nothing. The claim
|
|
190
277
|
// TG-README-HOOK-MATCHES-THE-CODE holds this sentence to that, here and in
|
|
191
278
|
// every other surface that describes the hook.
|
|
192
279
|
done.push(`${rel(settingsPath)}: SessionStart hook → brief --text${dir === '.' ? '' : ` ${dir}`} (local install first, then a testguard on PATH, never a fetch)`);
|
|
193
|
-
note(settingsPath);
|
|
194
280
|
}
|
|
195
281
|
|
|
196
282
|
const agentsPath = join(agentRoot, 'AGENTS.md');
|
|
@@ -204,41 +290,50 @@ export function initProject({ projectDir, force = false, here = false, ciEvidenc
|
|
|
204
290
|
if (entries.includes(dir)) {
|
|
205
291
|
skipped.push(`AGENTS.md already has the TestGuard section for ${dir === '.' ? 'this project' : dir}`);
|
|
206
292
|
} else {
|
|
207
|
-
|
|
293
|
+
write(agentsPath, agents.slice(0, begin) + agentsBlock([...entries, dir]) + agents.slice(end + AGENTS_END.length));
|
|
208
294
|
done.push(legacyBlock ? `AGENTS.md: TestGuard section rewritten to list ${dir}` : `AGENTS.md: TestGuard section now lists ${dir}`);
|
|
209
|
-
note(agentsPath);
|
|
210
295
|
}
|
|
211
296
|
} else {
|
|
212
|
-
|
|
297
|
+
write(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + agentsBlock([dir]) + '\n');
|
|
213
298
|
done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
|
|
214
|
-
note(agentsPath);
|
|
215
299
|
}
|
|
216
300
|
|
|
217
301
|
const giPath = join(projectDir, '.gitignore');
|
|
218
302
|
const gi = existsSync(giPath) ? readFileSync(giPath, 'utf8') : '';
|
|
219
303
|
const entries = gi.split('\n').map((x) => x.trim());
|
|
220
304
|
const dirIgnored = entries.some((x) => /^\/?\.testguard\/?$/.test(x));
|
|
221
|
-
|
|
305
|
+
// `.testguard/*` with negations is the form `baseline` advises: it already
|
|
306
|
+
// covers every output, present and future, so per-file lines would be noise.
|
|
307
|
+
const starForm = entries.some((x) => /^\/?\.testguard\/\*$/.test(x));
|
|
308
|
+
const missing = dirIgnored || starForm ? [] : GITIGNORE_LINES.filter((l) => !entries.includes(l));
|
|
222
309
|
if (dirIgnored) {
|
|
223
310
|
skipped.push(`${rel(giPath)} ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline`);
|
|
311
|
+
} else if (starForm) {
|
|
312
|
+
skipped.push(`${rel(giPath)} ignores .testguard/* — every regenerated output is covered; keep ${COMMITTED_OUTPUTS.map((p) => `!${p}`).join(', ')} beside it`);
|
|
224
313
|
} else if (missing.length) {
|
|
225
|
-
|
|
314
|
+
write(giPath, gitignoreWithLines(gi, missing), { checkIgnored: false });
|
|
226
315
|
done.push(`${rel(giPath)}: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
|
|
227
316
|
} else {
|
|
228
317
|
skipped.push(`${rel(giPath)} already ignores the regenerated files`);
|
|
229
318
|
}
|
|
230
|
-
if (ciEvidence) {
|
|
319
|
+
if (ciEvidence !== undefined) {
|
|
231
320
|
const helperPath = join(projectDir, '.testguard', 'fetch-ci-evidence.sh');
|
|
232
|
-
const body = CI_EVIDENCE_HELPERS[ciEvidence];
|
|
233
|
-
if (!body) throw new Error(`--ci-evidence must be github or gitlab, not ${ciEvidence}`);
|
|
234
321
|
if (!existsSync(helperPath) || force) {
|
|
235
|
-
|
|
236
|
-
writeFileSync(helperPath, body, { mode: 0o755 });
|
|
322
|
+
write(helperPath, CI_EVIDENCE_HELPER(ciEvidence), { mode: 0o755, checkIgnored: false });
|
|
237
323
|
done.push(`.testguard/fetch-ci-evidence.sh (${ciEvidence}) — run it on demand; the session-start hook stays offline`);
|
|
238
324
|
} else {
|
|
239
325
|
skipped.push('.testguard/fetch-ci-evidence.sh exists (use --force to replace)');
|
|
240
326
|
}
|
|
241
327
|
}
|
|
328
|
+
|
|
329
|
+
// ---- every check passed: write ----
|
|
330
|
+
for (const w of writes) {
|
|
331
|
+
mkdirSync(dirname(w.path), { recursive: true });
|
|
332
|
+
writeFileSync(w.path, w.content, w.mode === undefined ? undefined : { mode: w.mode });
|
|
333
|
+
if (w.mode !== undefined) chmodSync(w.path, w.mode); // an existing file keeps its old mode otherwise
|
|
334
|
+
if (w.checkIgnored) note(w.path);
|
|
335
|
+
}
|
|
336
|
+
|
|
242
337
|
if (mcp) {
|
|
243
338
|
// Printed, never written: a harness's own config is the person's file
|
|
244
339
|
// (and often global), so init shows the snippet and lets them place it.
|
|
@@ -131,6 +131,11 @@ export function isReusable(prior, current) {
|
|
|
131
131
|
// A record that exists because the probe threw is a statement about the run,
|
|
132
132
|
// not about the code. Reusing it would make one disturbed run permanent.
|
|
133
133
|
if (prior.detail?.reason === 'probe-error') return false;
|
|
134
|
+
// Defenders that never loaded say something about the environment — a
|
|
135
|
+
// broken --runner-cmd, a missing dependency, an interpreter without the test
|
|
136
|
+
// requirements — and none of that is hashed below. Fixing it changes no
|
|
137
|
+
// input, so the failure would keep being reported as the current answer.
|
|
138
|
+
if (prior.detail?.reason === 'defenders-failed-to-load') return false;
|
|
134
139
|
const origin = prior.defenders.selectionSource ?? (prior.defenders.requested.length ? 'claim' : 'discovery');
|
|
135
140
|
if (current.selectionSource && origin !== current.selectionSource) return false;
|
|
136
141
|
const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
|
package/src/probe/contention.mjs
CHANGED
|
@@ -90,8 +90,19 @@ export function detectContention({ self = process.pid, alive = defaultAlive, ...
|
|
|
90
90
|
return { detected: runners.length > 0, runners };
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
/**
|
|
94
|
-
|
|
93
|
+
/**
|
|
94
|
+
* What would actually change this run. `--serial` is the same as `--workers 1`,
|
|
95
|
+
* which is the default, so advising it to a run that is already serial offers
|
|
96
|
+
* a flag that changes nothing; a custom command's concurrency is its own.
|
|
97
|
+
*/
|
|
98
|
+
function contentionAdvice({ serial = true, workers = 1, custom = false }) {
|
|
99
|
+
if (custom) return 'Wait for it to finish; a --runner-cmd manages its own concurrency, so make that command run one test file at a time if it does not already.';
|
|
100
|
+
if (serial || workers <= 1) return 'Wait for it to finish: this probe already runs one test file at a time (--workers 1), so --serial would change nothing.';
|
|
101
|
+
return `Wait for it to finish, or lower --workers ${workers} (or pass --serial) to run one test file at a time.`;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** One line for the operator: what is running, what it costs, and what would help. */
|
|
105
|
+
export const contentionWarning = (c, policy = {}) =>
|
|
95
106
|
`${c.runners.length} test runner${c.runners.length === 1 ? ' is' : 's are'} already running (${c.runners.map((r) => `pid ${r.pid}`).join(', ')}); ` +
|
|
96
107
|
'defenders run N times each here, and a suite that times out under parallel load yields TIMEOUT or FLAKY-DEFENDER — verdicts about the load, not the claim. ' +
|
|
97
|
-
|
|
108
|
+
contentionAdvice(policy);
|
package/src/probe/cost.mjs
CHANGED
|
@@ -109,7 +109,13 @@ export function costReport(records, { run } = {}) {
|
|
|
109
109
|
};
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
|
|
112
|
+
// Measured durations are performance.now() differences, so they are fractional:
|
|
113
|
+
// round before choosing the unit, or 659.1493330000001ms prints verbatim and
|
|
114
|
+
// 999.6 prints as "1000ms".
|
|
115
|
+
const secs = (ms) => {
|
|
116
|
+
const whole = Math.round(ms);
|
|
117
|
+
return whole >= 10_000 ? `${Math.round(ms / 1000)}s` : whole >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${whole}ms`;
|
|
118
|
+
};
|
|
113
119
|
/** Exported so a caller can format a duration the same way the report does. */
|
|
114
120
|
export const formatMs = secs;
|
|
115
121
|
|
|
@@ -36,7 +36,11 @@ export function originPolicyInputsFresh({ projectDir, claims, evidence, universe
|
|
|
36
36
|
if (methodOf(evidence.run) !== 'fault-injection' || evidence.run.provisional || evidence.run.confirmRuns < 3) return false;
|
|
37
37
|
const testUniverseHash = hashNativeTestUniverse(universe.manifests);
|
|
38
38
|
if (universe.testUniverseHash !== testUniverseHash) return false;
|
|
39
|
-
|
|
39
|
+
// Where the runner came from is part of its identity: a global vitest of
|
|
40
|
+
// the same version is a different binary from the project's, the same
|
|
41
|
+
// rule verdict reuse applies (sameRunner in probe.mjs).
|
|
42
|
+
const matchesRunner = (recorded, current) => recorded?.name === current?.name && recorded?.version === current?.version
|
|
43
|
+
&& (recorded?.source ?? null) === (current?.source ?? null);
|
|
40
44
|
if (!universe.primaryRunner || !matchesRunner(evidence.run.runner, universe.primaryRunner)) return false;
|
|
41
45
|
if ((evidence.run.runners ?? []).some((recorded) => !universe.runners?.some((current) => matchesRunner(recorded, current)))) return false;
|
|
42
46
|
const allTests = Object.freeze([...new Set([...universe.manifests.values()].flatMap((m) => m.files))].sort());
|