testguard-cli 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +62 -7
- package/package.json +2 -1
- package/spec/GATE-SEMANTICS.md +51 -0
- package/spec/README.md +3 -2
- package/spec/lib/validate.mjs +19 -1
- package/spec/schemas/brief.schema.json +11 -1
- package/spec/schemas/common.schema.json +27 -0
- package/spec/schemas/gate.schema.json +52 -0
- package/spec/schemas/status.schema.json +26 -4
- package/src/brief/brief.mjs +26 -2
- package/src/cli.mjs +23 -3
- package/src/commands/brief.mjs +18 -10
- package/src/commands/claims.mjs +1 -1
- package/src/commands/gate.mjs +47 -0
- package/src/commands/probe.mjs +3 -1
- package/src/commands/status.mjs +2 -1
- package/src/gate/changed.mjs +222 -0
- package/src/init/init.mjs +1 -1
- package/src/init/templates/SKILL.md +19 -1
- package/src/probe/discover.mjs +1 -1
- package/src/probe/probe.mjs +17 -13
- package/src/probe/runners/index.mjs +20 -0
- package/src/probe/runners/jest.mjs +9 -0
- package/src/probe/{runner-vitest.mjs → runners/shared.mjs} +29 -21
- package/src/probe/runners/vitest.mjs +7 -0
- package/src/status/status.mjs +43 -3
package/src/commands/brief.mjs
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBrief } from '../brief/brief.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
|
|
5
5
|
import { computeStatus } from '../status/status.mjs';
|
|
6
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
7
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
7
8
|
|
|
8
9
|
export async function briefCommand({ projectDir, values, version }, io) {
|
|
9
10
|
const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
11
|
+
let status;
|
|
12
|
+
try {
|
|
13
|
+
// A detected base that does not resolve is silent here: the brief is a
|
|
14
|
+
// session-start hook's output and must never add noise; `status` and
|
|
15
|
+
// `gate` are where that warning is printed.
|
|
16
|
+
status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }));
|
|
17
|
+
} catch {
|
|
18
|
+
status = undefined; // a brief must never fail because status could not be computed
|
|
19
|
+
}
|
|
10
20
|
if (!existsSync(evPath)) {
|
|
11
|
-
// A missing brief must never break an agent's session start
|
|
12
|
-
|
|
21
|
+
// A missing brief must never break an agent's session start — but unclaimed
|
|
22
|
+
// changes are still said, because the claim comes before the probe.
|
|
23
|
+
if (values.text) {
|
|
24
|
+
if (status?.changes?.uncovered?.length) io.out(buildUnclaimedBrief({ tool: { name: 'testguard', version }, next: status.next, changes: status.changes }).text.trimEnd());
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
13
27
|
io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
|
|
14
28
|
return 2;
|
|
15
29
|
}
|
|
@@ -21,13 +35,7 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
21
35
|
io.err('--max must be an integer from 1 to 50');
|
|
22
36
|
return 3;
|
|
23
37
|
}
|
|
24
|
-
|
|
25
|
-
try {
|
|
26
|
-
next = computeStatus({ projectDir, toolVersion: version }).next;
|
|
27
|
-
} catch {
|
|
28
|
-
next = undefined; // a brief must never fail because status could not be computed
|
|
29
|
-
}
|
|
30
|
-
const brief = buildBrief(evidence, baseline, { max, next });
|
|
38
|
+
const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes });
|
|
31
39
|
if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
32
40
|
io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
33
41
|
return 0;
|
package/src/commands/claims.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
3
|
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
4
|
-
import { resolveDefenders } from '../probe/
|
|
4
|
+
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
5
5
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
6
6
|
|
|
7
7
|
export async function claimsCommand({ projectDir, values }, io) {
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { join, resolve } from 'node:path';
|
|
2
|
+
import { computeChangedGate, resolveChangedRef, renderGate, DEFAULT_EXCLUDES } from '../gate/changed.mjs';
|
|
3
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.json');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `testguard gate --changed <ref>`: fail when a changed source file carries
|
|
9
|
+
* no claim and no excusing ignore entry. Exit 0 covered · 1 uncovered (or
|
|
10
|
+
* --strict with nothing evaluated) · 2 cannot evaluate · 3 no reference.
|
|
11
|
+
*/
|
|
12
|
+
export async function gateCommand({ projectDir, values, version }, io) {
|
|
13
|
+
if (values.explain) {
|
|
14
|
+
io.out('Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):');
|
|
15
|
+
for (const g of DEFAULT_EXCLUDES) io.out(` ${g}`);
|
|
16
|
+
io.out('Non-source files (anything but .js .mjs .cjs .ts .mts .cts .jsx .tsx) are excluded before these patterns apply.');
|
|
17
|
+
return 0;
|
|
18
|
+
}
|
|
19
|
+
const resolved = resolveChangedRef({ explicit: values.changed });
|
|
20
|
+
if (!resolved) {
|
|
21
|
+
io.err('gate needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. In GitHub Actions and GitLab CI the base branch is detected automatically.');
|
|
22
|
+
return 3;
|
|
23
|
+
}
|
|
24
|
+
const ref = resolved.ref;
|
|
25
|
+
// The gate itself never degrades: a detected reference that does not resolve is exit 2 here, because the gate's whole job is the measurement.
|
|
26
|
+
if (!resolved.required && !values.json && !values.quiet) io.err(`--changed not given; using ${ref} from ${resolved.from}`);
|
|
27
|
+
const doc = computeChangedGate({
|
|
28
|
+
projectDir,
|
|
29
|
+
ref,
|
|
30
|
+
includeDirty: values['include-dirty'],
|
|
31
|
+
exclude: values.exclude ?? [],
|
|
32
|
+
strict: values.strict,
|
|
33
|
+
toolVersion: version,
|
|
34
|
+
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
35
|
+
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
|
36
|
+
});
|
|
37
|
+
const outPath = values.out ? resolve(values.out) : gatePath(projectDir);
|
|
38
|
+
writeSpecDoc('gate', outPath, doc);
|
|
39
|
+
if (values.json) {
|
|
40
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
41
|
+
} else {
|
|
42
|
+
io.out(renderGate(doc));
|
|
43
|
+
if (doc.uncovered.length) io.out(`Next: state the claim for each UNCLAIMED file (scaffold proposes the faults), or add a testguard.ignore.json path entry with a reason that a reviewer will accept.`);
|
|
44
|
+
if (!values.quiet) io.out(`gate: ${outPath}`);
|
|
45
|
+
}
|
|
46
|
+
return doc.exitCode;
|
|
47
|
+
}
|
package/src/commands/probe.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
|
|
|
6
6
|
import { gate } from '../baseline/baseline.mjs';
|
|
7
7
|
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
8
8
|
import { computeStatus } from '../status/status.mjs';
|
|
9
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
9
10
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
10
11
|
|
|
11
12
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -42,6 +43,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
42
43
|
mode: values['in-place'] ? 'in-place' : 'worktree',
|
|
43
44
|
ref: values.ref,
|
|
44
45
|
runnerCommand: values['runner-cmd'],
|
|
46
|
+
runnerName: values.runner,
|
|
45
47
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
46
48
|
only,
|
|
47
49
|
escalate: !values['no-escalate'],
|
|
@@ -57,7 +59,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
57
59
|
|
|
58
60
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
59
61
|
if (values.json) {
|
|
60
|
-
const status = computeStatus({ projectDir, toolVersion: version });
|
|
62
|
+
const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
|
|
61
63
|
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
|
|
62
64
|
return g.new.length > 0 ? 1 : 0;
|
|
63
65
|
}
|
package/src/commands/status.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { computeStatus, renderStatus } from '../status/status.mjs';
|
|
2
2
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
3
4
|
|
|
4
5
|
export async function statusCommand({ projectDir, values, version }, io) {
|
|
5
|
-
const doc = computeStatus({ projectDir, toolVersion: version });
|
|
6
|
+
const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
|
|
6
7
|
const result = validate('status', doc);
|
|
7
8
|
if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
8
9
|
io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { existsSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { extname, join, relative, resolve, dirname } from 'node:path';
|
|
3
|
+
import { git, repoRoot, headSha, GitError } from '../git.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { globToRegExp } from '../util/glob.mjs';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Claim coverage of a change.
|
|
12
|
+
*
|
|
13
|
+
* `probe` answers "do the tests defend the claims that exist?". This answers
|
|
14
|
+
* the question every field report so far turned on: "does the code I just
|
|
15
|
+
* changed have any claim at all?". A repository with ten old claims and one
|
|
16
|
+
* new, unclaimed feature exits 0 forever under `probe`; under `gate` the
|
|
17
|
+
* delta is measured, file by file, and an uncovered file is a finding.
|
|
18
|
+
*
|
|
19
|
+
* The unit is the file, on purpose: a fault anchors to a file, so the file is
|
|
20
|
+
* the smallest unit the rest of the contract already understands.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
|
|
24
|
+
const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
25
|
+
|
|
26
|
+
/** Files that are source by extension but never carry claims. Printed by --explain. */
|
|
27
|
+
export const DEFAULT_EXCLUDES = Object.freeze([
|
|
28
|
+
'**/*.d.ts',
|
|
29
|
+
'**/*.config.*',
|
|
30
|
+
'**/*.stories.*',
|
|
31
|
+
'**/*.generated.*',
|
|
32
|
+
'**/fixtures/**',
|
|
33
|
+
'**/__fixtures__/**',
|
|
34
|
+
'**/__mocks__/**',
|
|
35
|
+
'**/__snapshots__/**',
|
|
36
|
+
'.testguard/**',
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The reference the change is measured against. Explicit flag first, then the
|
|
43
|
+
* environments CI sets, then nothing: a wrong default would make the gate pass
|
|
44
|
+
* trivially (an upstream that already contains the change has an empty diff),
|
|
45
|
+
* so no ref is an error the caller reports, never a silent guess.
|
|
46
|
+
*/
|
|
47
|
+
export function detectChangedRef(env = process.env) {
|
|
48
|
+
if (env.TESTGUARD_CHANGED_REF) return { ref: env.TESTGUARD_CHANGED_REF, from: 'TESTGUARD_CHANGED_REF' };
|
|
49
|
+
// GitHub Actions, pull_request events.
|
|
50
|
+
if (env.GITHUB_BASE_REF) return { ref: `origin/${env.GITHUB_BASE_REF}`, from: 'GITHUB_BASE_REF' };
|
|
51
|
+
// GitLab merge request pipelines. The diff base sha is exact and needs no
|
|
52
|
+
// remote-tracking ref; the target branch name is the fallback (it needs
|
|
53
|
+
// `git fetch origin <branch>` or GIT_DEPTH: 0 on the job).
|
|
54
|
+
const sha = env.CI_MERGE_REQUEST_DIFF_BASE_SHA;
|
|
55
|
+
if (sha && /^[0-9a-f]{7,40}$/i.test(sha) && !/^0+$/.test(sha)) return { ref: sha, from: 'CI_MERGE_REQUEST_DIFF_BASE_SHA' };
|
|
56
|
+
if (env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME) return { ref: `origin/${env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME}`, from: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME' };
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Resolve the reference a command should measure against, and whether it may
|
|
62
|
+
* be dropped on failure. An explicit `--changed` is the user's word: if it
|
|
63
|
+
* does not resolve, that is an error. A reference detected from CI variables
|
|
64
|
+
* is a convenience: if it does not resolve (shallow clone, temp directory,
|
|
65
|
+
* no remote) the command says so on stderr and continues without a change
|
|
66
|
+
* measurement, because `status` and `brief` must keep working everywhere.
|
|
67
|
+
*/
|
|
68
|
+
export function resolveChangedRef({ explicit, env = process.env } = {}) {
|
|
69
|
+
if (explicit) return { ref: explicit, from: '--changed', required: true };
|
|
70
|
+
const d = detectChangedRef(env);
|
|
71
|
+
return d ? { ...d, required: false } : null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Run `compute(ref)` for a resolved reference. A required reference propagates
|
|
76
|
+
* its error; a detected one that fails yields `undefined` after `warn(message)`.
|
|
77
|
+
*/
|
|
78
|
+
export function withChangedRef(resolved, compute, warn = () => {}) {
|
|
79
|
+
if (!resolved) return compute(undefined);
|
|
80
|
+
try {
|
|
81
|
+
return compute(resolved.ref);
|
|
82
|
+
} catch (e) {
|
|
83
|
+
if (resolved.required || !(e instanceof GitError)) throw e;
|
|
84
|
+
warn(`change coverage skipped: ${resolved.ref} (from ${resolved.from}) does not resolve here — ${e.message.split('. ')[0]}. Pass --changed <ref> to make this an error.`);
|
|
85
|
+
return compute(undefined);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Files changed since the merge-base of `ref` and HEAD, repo-root-relative,
|
|
91
|
+
* deletions dropped. With `includeDirty` the working tree (tracked edits,
|
|
92
|
+
* staged or not, plus untracked non-ignored files) is compared instead of HEAD.
|
|
93
|
+
*/
|
|
94
|
+
export function changedFiles({ root, ref, includeDirty = false }) {
|
|
95
|
+
let base;
|
|
96
|
+
try {
|
|
97
|
+
base = git(['merge-base', ref, 'HEAD'], root);
|
|
98
|
+
} catch (e) {
|
|
99
|
+
throw new GitError(`cannot resolve --changed ${ref}: ${e.message.split('\n').pop()}. In CI the base must be present locally: GitHub Actions — actions/checkout with fetch-depth: 0; GitLab — GIT_DEPTH: 0 or \`git fetch origin <target branch>\` before the job (merge request pipelines also provide CI_MERGE_REQUEST_DIFF_BASE_SHA, which needs no fetch).`);
|
|
100
|
+
}
|
|
101
|
+
const head = headSha(root);
|
|
102
|
+
const out = new Set(git(['diff', '--name-only', '--diff-filter=d', '-z', base, ...(includeDirty ? [] : ['HEAD'])], root).split('\0').filter(Boolean));
|
|
103
|
+
if (includeDirty) for (const f of git(['ls-files', '--others', '--exclude-standard', '-z'], root).split('\0').filter(Boolean)) out.add(f);
|
|
104
|
+
return { base, head, files: [...out].sort() };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const isExpired = (entry, now) => Boolean(entry.expires) && new Date(entry.expires).getTime() <= now.getTime();
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The claim to attach to, if any: one whose fault files live in the same
|
|
111
|
+
* directory as the changed file. A claim that merely shares `src/` is not
|
|
112
|
+
* "nearest", it is noise, and a wrong `--claim` hint is worse than none.
|
|
113
|
+
*/
|
|
114
|
+
function nearestClaim(file, faultFiles) {
|
|
115
|
+
const dir = dirname(file);
|
|
116
|
+
for (const [ff, ids] of faultFiles) if (dirname(ff) === dir) return ids[0];
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Compute the gate for one change. Pure apart from reading the repository,
|
|
122
|
+
* the claims file and the ignore file; the caller decides what to print and
|
|
123
|
+
* what to write.
|
|
124
|
+
*/
|
|
125
|
+
export function computeChangedGate({ projectDir, ref, includeDirty = false, exclude = [], strict = false, now = new Date(), toolVersion = '0.0.0', generatedAt = now.toISOString(), claimsPath, ignorePath }) {
|
|
126
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
127
|
+
const root = repoRoot(projectDir);
|
|
128
|
+
const relProject = relative(root, projectDir).split('\\').join('/');
|
|
129
|
+
const { base, head, files } = changedFiles({ root, ref, includeDirty });
|
|
130
|
+
|
|
131
|
+
// Repo-root paths → project paths; a change outside the project directory is not this project's change.
|
|
132
|
+
const changed = [];
|
|
133
|
+
for (const f of files) {
|
|
134
|
+
if (!relProject) { changed.push(f); continue; }
|
|
135
|
+
if (f.startsWith(relProject + '/')) changed.push(f.slice(relProject.length + 1));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const cPath = claimsPath ?? defaultClaimsPath(projectDir);
|
|
139
|
+
const claims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
|
|
140
|
+
const iPath = ignorePath ?? defaultIgnorePath(projectDir);
|
|
141
|
+
const ignore = existsSync(iPath) ? readSpecDoc('ignore', iPath) : { entries: [] };
|
|
142
|
+
|
|
143
|
+
// Coverage index: which files carry a fault, which test files defend a claim.
|
|
144
|
+
const faultFiles = new Map(); // file → claimIds
|
|
145
|
+
const defenderFiles = new Map(); // test file → claimIds
|
|
146
|
+
for (const c of claims.claims) {
|
|
147
|
+
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
148
|
+
const defenders = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
|
|
149
|
+
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const pathEntries = ignore.entries.filter((e) => e.kind === 'path').map((e) => ({ entry: e, re: globToRegExp(e.pattern), expired: isExpired(e, now), files: [] }));
|
|
153
|
+
const userExcludes = exclude.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
154
|
+
const defaultExcludes = DEFAULT_EXCLUDES.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
155
|
+
|
|
156
|
+
const excluded = [];
|
|
157
|
+
const covered = [];
|
|
158
|
+
const uncovered = [];
|
|
159
|
+
for (const file of changed) {
|
|
160
|
+
if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
|
|
161
|
+
const ux = userExcludes.find((x) => x.re.test(file));
|
|
162
|
+
if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
|
|
163
|
+
const dx = defaultExcludes.find((x) => x.re.test(file));
|
|
164
|
+
if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
|
|
165
|
+
|
|
166
|
+
const isTest = TEST_RE.test(file);
|
|
167
|
+
if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
|
|
168
|
+
if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
|
|
169
|
+
const active = pathEntries.find((p) => !p.expired && p.re.test(file));
|
|
170
|
+
if (active) { active.files.push(file); covered.push({ file, by: 'ignore', pattern: active.entry.pattern }); continue; }
|
|
171
|
+
for (const p of pathEntries) if (p.expired && p.re.test(file)) p.files.push(file);
|
|
172
|
+
|
|
173
|
+
// Uncovered. Point at a same-directory claim so the author knows where to attach.
|
|
174
|
+
const nearest = nearestClaim(file, faultFiles);
|
|
175
|
+
const item = {
|
|
176
|
+
file,
|
|
177
|
+
kind: isTest ? 'test' : 'source',
|
|
178
|
+
...(nearest ? { nearestClaimId: nearest } : {}),
|
|
179
|
+
suggestion: isTest
|
|
180
|
+
? `add ${file} to the defendedBy of the claim it defends${nearest ? ` (nearest: ${nearest})` : ''}, or state a new claim for what it asserts`
|
|
181
|
+
: `testguard scaffold ${file}${nearest ? ` --claim ${nearest}` : ''}`,
|
|
182
|
+
};
|
|
183
|
+
uncovered.push(item);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const evaluated = covered.length + uncovered.length;
|
|
187
|
+
const reliedOn = pathEntries.filter((p) => !p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, ...(p.entry.expires ? { expires: p.entry.expires } : {}), files: p.files }));
|
|
188
|
+
const expired = pathEntries.filter((p) => p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, expires: p.entry.expires, files: p.files }));
|
|
189
|
+
const exitCode = uncovered.length > 0 ? 1 : strict && changed.length > 0 && evaluated === 0 ? 1 : 0;
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
schemaVersion: 1,
|
|
193
|
+
tool: { name: 'testguard', version: toolVersion },
|
|
194
|
+
generatedAt,
|
|
195
|
+
ref,
|
|
196
|
+
base,
|
|
197
|
+
head,
|
|
198
|
+
includeDirty,
|
|
199
|
+
strict,
|
|
200
|
+
changed: changed.length,
|
|
201
|
+
evaluated,
|
|
202
|
+
excluded,
|
|
203
|
+
covered,
|
|
204
|
+
uncovered,
|
|
205
|
+
reliedOn,
|
|
206
|
+
expired,
|
|
207
|
+
exitCode,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function renderGate(doc) {
|
|
212
|
+
const lines = [];
|
|
213
|
+
const where = doc.includeDirty ? 'working tree' : `HEAD ${doc.head.slice(0, 7)}`;
|
|
214
|
+
lines.push(`gate: ${doc.changed} file${doc.changed === 1 ? '' : 's'} changed since ${doc.ref} (merge-base ${doc.base.slice(0, 7)}) in the ${where}; ${doc.evaluated} evaluated, ${doc.excluded.length} excluded, ${doc.covered.length} covered, ${doc.uncovered.length} uncovered`);
|
|
215
|
+
for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
|
|
216
|
+
for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
217
|
+
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
|
|
218
|
+
if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
|
|
219
|
+
if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
|
|
220
|
+
if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
|
|
221
|
+
return lines.join('\n');
|
|
222
|
+
}
|
package/src/init/init.mjs
CHANGED
|
@@ -14,7 +14,7 @@ Before writing or changing tests, run \`testguard status --json\` and follow \`n
|
|
|
14
14
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
15
15
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
16
16
|
${AGENTS_END}`;
|
|
17
|
-
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/scaffold-*.json'];
|
|
17
|
+
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Install the agent operating layer into a consumer project: the skill, the
|
|
@@ -21,13 +21,31 @@ which files exist. `state` is one of:
|
|
|
21
21
|
| state | meaning | you do |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
|
+
| `unclaimed-changes` | files you changed carry no claim (only when a reference is known: `--changed <ref>`, or CI's base branch) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
24
25
|
| `unprobed` | claims never probed | `testguard probe` |
|
|
25
26
|
| `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
|
|
26
27
|
| `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
|
|
27
28
|
| `unproven` | a finding is not covered by the baseline | `next.target` names it; see the verdict table |
|
|
28
29
|
| `clean` | everything killed or baselined | `testguard baseline` if `next` says so; otherwise nothing |
|
|
29
30
|
|
|
30
|
-
Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
31
|
+
Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
32
|
+
|
|
33
|
+
## Every change needs a claim
|
|
34
|
+
|
|
35
|
+
`probe` verifies only the claims that exist; it is silent about unclaimed
|
|
36
|
+
code by construction. So, for every source file you create or change:
|
|
37
|
+
|
|
38
|
+
1. `testguard gate --changed HEAD --include-dirty` (the pre-commit shape; in a
|
|
39
|
+
PR, `--changed origin/<base>`). One unclaimed file exits `1`.
|
|
40
|
+
2. For each `UNCLAIMED` file: `testguard scaffold <file>` proposes faults;
|
|
41
|
+
state what the code guarantees; add the claim, its faults and its
|
|
42
|
+
`defendedBy` to `testguard.claims.json`. A new test file must be named in
|
|
43
|
+
some claim's `defendedBy`, or it is unclaimed too.
|
|
44
|
+
3. Only when a file genuinely carries nothing to claim (generated code, a
|
|
45
|
+
thin wrapper whose logic is claimed elsewhere): a `path` entry in
|
|
46
|
+
`testguard.ignore.json` with a reason and, where possible, an `expires`.
|
|
47
|
+
The gate prints every entry it relied on; a reviewer reads them.
|
|
48
|
+
4. Then `testguard probe --claim <ID> --include-dirty` for the new claim.
|
|
31
49
|
|
|
32
50
|
## Verdict → action
|
|
33
51
|
|
package/src/probe/discover.mjs
CHANGED
package/src/probe/probe.mjs
CHANGED
|
@@ -6,7 +6,8 @@ import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../g
|
|
|
6
6
|
import { discoverDefenders } from './discover.mjs';
|
|
7
7
|
import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
|
|
8
8
|
import { applyFault, locate } from './inject.mjs';
|
|
9
|
-
import
|
|
9
|
+
import { resolveDefenders, parseCommandTemplate } from './runners/shared.mjs';
|
|
10
|
+
import { selectRunner, RUNNERS } from './runners/index.mjs';
|
|
10
11
|
import { classify, shouldStopEarly } from './classify.mjs';
|
|
11
12
|
import { blastRadius, rank } from './rank.mjs';
|
|
12
13
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
@@ -14,9 +15,9 @@ import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
|
14
15
|
|
|
15
16
|
const isKill = (r) => r.outcome === 'fail' && r.assertionFailures > 0;
|
|
16
17
|
|
|
17
|
-
function readRunnerVersion(projectDir) {
|
|
18
|
+
function readRunnerVersion(projectDir, name = 'vitest') {
|
|
18
19
|
try {
|
|
19
|
-
return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve(
|
|
20
|
+
return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve(`${name}/package.json`), 'utf8')).version;
|
|
20
21
|
} catch {
|
|
21
22
|
return undefined;
|
|
22
23
|
}
|
|
@@ -34,6 +35,7 @@ export async function probe({
|
|
|
34
35
|
ref = 'HEAD',
|
|
35
36
|
budgetMs = 120_000,
|
|
36
37
|
runnerCommand,
|
|
38
|
+
runnerName = 'auto',
|
|
37
39
|
nodeModules,
|
|
38
40
|
only,
|
|
39
41
|
includeDirty = false,
|
|
@@ -75,7 +77,7 @@ export async function probe({
|
|
|
75
77
|
const watched = new Set(targets);
|
|
76
78
|
for (const claim of claims.claims) {
|
|
77
79
|
if (selected && !selected.has(claim.id)) continue;
|
|
78
|
-
const declared = claim.defendedBy?.length ?
|
|
80
|
+
const declared = claim.defendedBy?.length ? resolveDefenders(projectDir, claim.defendedBy) : claim.faults.flatMap((f) => discoverDefenders(projectDir, f.file));
|
|
79
81
|
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
80
82
|
for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
81
83
|
}
|
|
@@ -89,27 +91,29 @@ export async function probe({
|
|
|
89
91
|
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
90
92
|
const records = [];
|
|
91
93
|
let runnerVersion;
|
|
94
|
+
let runner = RUNNERS[runnerName === 'auto' ? 'vitest' : runnerName];
|
|
92
95
|
try {
|
|
93
|
-
const commandTemplate = runnerCommand ?
|
|
96
|
+
const commandTemplate = runnerCommand ? parseCommandTemplate(runnerCommand) : undefined;
|
|
94
97
|
if (!commandTemplate) {
|
|
95
98
|
// An unresolvable runner is a precondition failure, not a flaky defender.
|
|
96
|
-
const
|
|
97
|
-
if (
|
|
98
|
-
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${
|
|
99
|
+
const sel = await selectRunner({ projectDir: iso.projectDir, name: runnerName });
|
|
100
|
+
if (sel.error) {
|
|
101
|
+
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
|
|
99
102
|
(mode === 'worktree' ? 'No usable node_modules was linked: pass --node-modules <path>, or run with --in-place.' : 'Install dependencies first.'));
|
|
100
103
|
}
|
|
101
|
-
|
|
104
|
+
runner = sel.runner;
|
|
105
|
+
runnerVersion = sel.version;
|
|
102
106
|
}
|
|
103
|
-
const allTests =
|
|
107
|
+
const allTests = runner.tests(iso.projectDir);
|
|
104
108
|
const baselineCache = new Map();
|
|
105
109
|
const prior = previous && previous.run.confirmRuns === confirmRuns
|
|
106
110
|
? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
|
|
107
111
|
: new Map();
|
|
108
|
-
const runDefenders = (files) =>
|
|
112
|
+
const runDefenders = (files) => runner.run({ projectDir: iso.projectDir, files, budgetMs, commandTemplate });
|
|
109
113
|
|
|
110
114
|
for (const claim of claims.claims) {
|
|
111
115
|
if (selected && !selected.has(claim.id)) continue;
|
|
112
|
-
const declared = claim.defendedBy?.length ?
|
|
116
|
+
const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
113
117
|
for (const fault of claim.faults) {
|
|
114
118
|
const defenders = declared ?? discoverDefenders(iso.projectDir, fault.file);
|
|
115
119
|
const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
|
|
@@ -130,7 +134,7 @@ export async function probe({
|
|
|
130
134
|
startedAt,
|
|
131
135
|
finishedAt: new Date().toISOString(),
|
|
132
136
|
repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
|
|
133
|
-
runner: { name:
|
|
137
|
+
runner: { name: runner.name, ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
|
|
134
138
|
confirmRuns,
|
|
135
139
|
...(confirmRuns < 3 ? { provisional: true } : {}),
|
|
136
140
|
mode,
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import * as vitest from './vitest.mjs';
|
|
2
|
+
import * as jest from './jest.mjs';
|
|
3
|
+
|
|
4
|
+
export const RUNNERS = { vitest, jest };
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Pick the runner: an explicit name, or the first of vitest, jest that is
|
|
8
|
+
* resolvable from the project. Returns { runner, version } or { error }.
|
|
9
|
+
*/
|
|
10
|
+
export async function selectRunner({ projectDir, name = 'auto' }) {
|
|
11
|
+
const candidates = name === 'auto' ? [vitest, jest] : [RUNNERS[name]];
|
|
12
|
+
if (!candidates[0]) return { error: `unknown runner "${name}"; use vitest, jest or auto` };
|
|
13
|
+
const messages = [];
|
|
14
|
+
for (const r of candidates) {
|
|
15
|
+
const c = await r.check({ projectDir });
|
|
16
|
+
if (c.ok) return { runner: r, version: c.version };
|
|
17
|
+
messages.push(`${r.name}: ${c.message}`);
|
|
18
|
+
}
|
|
19
|
+
return { error: messages.join('; ') };
|
|
20
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
|
|
2
|
+
|
|
3
|
+
export const name = 'jest';
|
|
4
|
+
// jest's default testMatch also collects anything under __tests__/.
|
|
5
|
+
export const testGlobs = [...TEST_GLOBS, '**/__tests__/**/*.js', '**/__tests__/**/*.mjs', '**/__tests__/**/*.cjs', '**/__tests__/**/*.ts', '**/__tests__/**/*.tsx', '**/__tests__/**/*.jsx'];
|
|
6
|
+
export const check = (opts) => checkBinary({ ...opts, bin: 'jest' });
|
|
7
|
+
export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
|
|
8
|
+
// --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
|
|
9
|
+
export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'jest', '--ci', '--json', `--outputFile=${outFile}`, '--runTestsByPath', ...files] });
|