testguard-cli 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +54 -4
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +51 -0
- package/spec/README.md +3 -2
- package/spec/lib/validate.mjs +19 -1
- package/spec/schemas/brief.schema.json +11 -1
- package/spec/schemas/common.schema.json +27 -0
- package/spec/schemas/gate.schema.json +52 -0
- package/spec/schemas/status.schema.json +26 -4
- package/src/brief/brief.mjs +26 -2
- package/src/cli.mjs +17 -3
- package/src/commands/brief.mjs +18 -10
- package/src/commands/gate.mjs +47 -0
- package/src/commands/probe.mjs +2 -1
- package/src/commands/status.mjs +2 -1
- package/src/gate/changed.mjs +222 -0
- package/src/init/init.mjs +1 -1
- package/src/init/templates/SKILL.md +19 -1
- package/src/status/status.mjs +42 -2
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.5.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
The change gate: unclaimed code is now a finding.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **`testguard gate --changed <ref>`** — the change gate. Every escaped
|
|
17
|
+
defect in the field reports was a *claim gap*: the feature shipped green
|
|
18
|
+
with zero claims, and `probe` is silent about unclaimed code by
|
|
19
|
+
construction. `gate` measures the files changed since
|
|
20
|
+
`merge-base(ref, HEAD)` (or in the working tree with `--include-dirty`)
|
|
21
|
+
and exits `1` when any changed source file carries no fault, does not
|
|
22
|
+
resolve as a defender (test files), and is not excused by an unexpired
|
|
23
|
+
`path` entry in `testguard.ignore.json`. Every reliance on an ignore entry
|
|
24
|
+
is printed with its reason; expired entries excuse nothing. Non-source
|
|
25
|
+
files and documented never-claimed patterns are excluded and listed
|
|
26
|
+
(`--explain`, `--exclude <glob>`); `--strict` fails a change that
|
|
27
|
+
evaluated nothing. The base branch is detected in GitHub Actions and
|
|
28
|
+
GitLab CI (`TESTGUARD_CHANGED_REF` overrides). A new spec kind,
|
|
29
|
+
`gate.schema.json`, with conformance examples.
|
|
30
|
+
- **`status --changed <ref>`** — a new state `unclaimed-changes` and action
|
|
31
|
+
`claim` that precede every evidence state; `next.file` names the first
|
|
32
|
+
unclaimed file. `brief` renders unclaimed files before the findings, and
|
|
33
|
+
briefs them even when there is no evidence yet.
|
|
34
|
+
- `testguard.ignore.json` at the repository root: TestGuard's own excused
|
|
35
|
+
paths (dispatch, thin command wrappers, rendering), with reasons.
|
|
36
|
+
- Nine new self-claims: the gate, status and brief invariants, explicit
|
|
37
|
+
`--changed` being required, `init` idempotency, and — because the gate
|
|
38
|
+
flagged it on its own pull request — the release version-sync check,
|
|
39
|
+
which had no test before. The gate runs on this repository's own pull
|
|
40
|
+
requests.
|
|
41
|
+
- GitHub Action `command: gate` with a `changed-ref` input; pre-commit hook
|
|
42
|
+
`testguard-gate`; a GitLab CI template under `packaging/gitlab/` with
|
|
43
|
+
`testguard:gate` (merge request pipelines, measured against
|
|
44
|
+
`CI_MERGE_REQUEST_DIFF_BASE_SHA`) and `testguard:probe`.
|
|
45
|
+
|
|
10
46
|
## [0.4.0] - 2026-09-17
|
|
11
47
|
|
|
12
48
|
### Added
|
package/README.md
CHANGED
|
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.5.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
58
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -68,6 +68,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
68
68
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
69
69
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
70
70
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
71
|
+
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim at all
|
|
71
72
|
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
72
73
|
```
|
|
73
74
|
|
|
@@ -146,6 +147,54 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
146
147
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
147
148
|
so the hook can never break a session.
|
|
148
149
|
|
|
150
|
+
### Every change needs a claim
|
|
151
|
+
|
|
152
|
+
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
153
|
+
reports so far was a **claim gap**: the feature shipped green with zero
|
|
154
|
+
claims, and a verifier with no claim about a feature is silent about it by
|
|
155
|
+
construction. `gate` closes that hole on the delta:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
npx testguard-cli gate --changed origin/main # in a PR: the files changed since the base branch
|
|
159
|
+
npx testguard-cli gate --changed HEAD --include-dirty # before a commit: the working tree, staged or not
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Every changed source file must carry a fault, resolve as a defender of a
|
|
163
|
+
claim (test files), or be excused by an unexpired `path` entry in
|
|
164
|
+
`testguard.ignore.json` — with a reason a reviewer will accept. One
|
|
165
|
+
unclaimed file exits `1`; there is no percentage. Every reliance on an ignore
|
|
166
|
+
entry is printed, so a reviewer sees *why* the gate passed; an expired entry
|
|
167
|
+
excuses nothing. Non-source files and documented never-claimed patterns
|
|
168
|
+
(`*.d.ts`, `*.config.*`, fixtures, mocks; `--explain` lists them) are
|
|
169
|
+
excluded and said so; `--strict` fails a change that evaluated nothing.
|
|
170
|
+
|
|
171
|
+
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
172
|
+
**before** any evidence state and makes the claim the next action; the brief
|
|
173
|
+
lists the unclaimed files first. The claim is written before more code.
|
|
174
|
+
|
|
175
|
+
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
176
|
+
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
177
|
+
`CI_MERGE_REQUEST_TARGET_BRANCH_NAME`); `TESTGUARD_CHANGED_REF` overrides both.
|
|
178
|
+
The base must exist locally: GitHub — `actions/checkout` with `fetch-depth: 0`;
|
|
179
|
+
GitLab — the diff base sha needs nothing extra on a merge request pipeline,
|
|
180
|
+
the branch name needs `GIT_DEPTH: 0` or a `git fetch origin <target>`. A
|
|
181
|
+
detected base that does not resolve is a warning for `status` and `brief`
|
|
182
|
+
(they keep working) and an error for `gate` (its whole job is the measurement).
|
|
183
|
+
|
|
184
|
+
```yaml
|
|
185
|
+
# GitHub Actions
|
|
186
|
+
- uses: actions/checkout@v4
|
|
187
|
+
with: { fetch-depth: 0 }
|
|
188
|
+
- uses: raccioly/testguard@v0.5.0
|
|
189
|
+
with: { command: gate }
|
|
190
|
+
|
|
191
|
+
# GitLab CI — or include: remote: the template in packaging/gitlab/
|
|
192
|
+
testguard:gate:
|
|
193
|
+
image: node:22
|
|
194
|
+
rules: [{ if: $CI_PIPELINE_SOURCE == "merge_request_event" }]
|
|
195
|
+
script: [npx -y testguard-cli gate .]
|
|
196
|
+
```
|
|
197
|
+
|
|
149
198
|
### Built for agents to run
|
|
150
199
|
|
|
151
200
|
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
@@ -220,9 +269,10 @@ through every verdict.
|
|
|
220
269
|
|
|
221
270
|
## Status
|
|
222
271
|
|
|
223
|
-
**v0.4.**
|
|
224
|
-
a mechanical scaffold,
|
|
225
|
-
|
|
272
|
+
**v0.4.** Eight commands, vitest and jest runners, hand-authored faults plus
|
|
273
|
+
a mechanical scaffold, an agent operating layer (`status`, `init`) and a
|
|
274
|
+
change gate (`gate`). The contract
|
|
275
|
+
spine — eight JSON Schemas shared with the other Guard tools — is under
|
|
226
276
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
227
277
|
|
|
228
278
|
Not yet: test generation (the two-gate acceptance loop), runners beyond
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -85,6 +85,57 @@ the suppress-up-to-count semantics are identical.
|
|
|
85
85
|
- An annotation is **strictly additive**. It never changes, suppresses, or
|
|
86
86
|
drops a finding. Ranking may read annotations; verdicts never do.
|
|
87
87
|
|
|
88
|
+
## Claim coverage of a change
|
|
89
|
+
|
|
90
|
+
`probe` asks whether the tests defend the claims that exist. It says nothing
|
|
91
|
+
about code that has no claim, by construction — and every escaped defect the
|
|
92
|
+
field reports share was a *claim gap*, not a defender gap: the feature shipped
|
|
93
|
+
green with zero claims. A repository with ten old claims and one new,
|
|
94
|
+
unclaimed module exits 0 under `probe` forever.
|
|
95
|
+
|
|
96
|
+
The change gate closes that hole. Given a reference, a tool measures the
|
|
97
|
+
**delta** — the files changed since `merge-base(ref, HEAD)`, or since that
|
|
98
|
+
merge-base in the working tree — file by file:
|
|
99
|
+
|
|
100
|
+
- A changed file is **excluded** when it is not source (by extension) or
|
|
101
|
+
matches a documented never-claimed pattern (`*.d.ts`, `*.config.*`,
|
|
102
|
+
fixtures, mocks, snapshots, the tool's own directory). Exclusions are
|
|
103
|
+
listed, never silent.
|
|
104
|
+
- A changed source file is **covered** when at least one fault anchors to it,
|
|
105
|
+
when it is a test file that resolves as a defender of some claim, or when an
|
|
106
|
+
**unexpired** `path` ignore entry excuses it.
|
|
107
|
+
- Anything else is **uncovered**. The unit is the file, on purpose: a fault
|
|
108
|
+
anchors to a file, so the file is the smallest unit the rest of the
|
|
109
|
+
contract already understands.
|
|
110
|
+
|
|
111
|
+
Rules:
|
|
112
|
+
|
|
113
|
+
1. **One uncovered file gates.** Exit `1`. There is no threshold and no
|
|
114
|
+
percentage; a percentage is how the gap hid before.
|
|
115
|
+
2. **Every reliance on an ignore entry is reported.** The gate may pass
|
|
116
|
+
*because of* an excuse; a reviewer reads which entries carried it, with
|
|
117
|
+
their reasons and expiry. An expired entry excuses nothing and is reported
|
|
118
|
+
as expired.
|
|
119
|
+
3. **A test file needs a claim too.** A test that defends no claim is the
|
|
120
|
+
authorship trap the pattern exists for; it is uncovered until a claim
|
|
121
|
+
names it in `defendedBy` (or discovery resolves it as a defender).
|
|
122
|
+
4. **Never a silent pass on nothing.** A non-empty change whose files were
|
|
123
|
+
all excluded passes with an explicit "0 evaluated" line; a tool offers a
|
|
124
|
+
strict mode that fails it instead. An empty change is an honest `0`.
|
|
125
|
+
5. **The reference is never guessed.** An explicit flag, or a CI-provided base
|
|
126
|
+
branch, or an error. An upstream that already contains the change has an
|
|
127
|
+
empty diff and would pass trivially.
|
|
128
|
+
6. **Unclaimed changes precede every evidence state.** When the status
|
|
129
|
+
document knows a reference and finds uncovered files, its state is
|
|
130
|
+
`unclaimed-changes` and its next action is to write the claim, before any
|
|
131
|
+
`unproven` finding is surfaced. The brief renders them first. The claim is
|
|
132
|
+
written before more code.
|
|
133
|
+
|
|
134
|
+
Exit codes: `0` every changed source file is claimed or excused (or nothing
|
|
135
|
+
changed); `1` at least one uncovered file (or strict mode over an
|
|
136
|
+
all-excluded change); `2` the change cannot be evaluated (unresolvable
|
|
137
|
+
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
138
|
+
|
|
88
139
|
## Severity floor
|
|
89
140
|
|
|
90
141
|
`--severity <level>` gates only findings whose claim severity is at or above
|
package/spec/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Guard spec — the contract spine
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Eight formats that tools following the Guard pattern share. They share
|
|
4
4
|
*formats*, not code: [`docguard-cli`](https://www.npmjs.com/package/docguard-cli)
|
|
5
5
|
is Node ESM, [`websec-validator`](https://pypi.org/project/websec-validator/)
|
|
6
6
|
is Python, and porting one runtime into the other is not worth it. A tool
|
|
@@ -21,7 +21,8 @@ The pattern all such tools run:
|
|
|
21
21
|
| `ignore` | [`schemas/ignore.schema.json`](schemas/ignore.schema.json) | Reviewable scoping. Every entry has a reason. |
|
|
22
22
|
| `calibration` | [`schemas/calibration.schema.json`](schemas/calibration.schema.json) | P(finding is real) per bucket, with a Wilson interval and the sample size behind it. |
|
|
23
23
|
| `brief` | [`schemas/brief.schema.json`](schemas/brief.schema.json) | What to tell an agent before it writes code — ranked, capped, never a single score, and carrying the one next action. |
|
|
24
|
-
| `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed. |
|
|
24
|
+
| `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed, and changed files that carry no claim. |
|
|
25
|
+
| `gate` | [`schemas/gate.schema.json`](schemas/gate.schema.json) | Claim coverage of one change: which changed files carry a claim, which are excused (and by which ignore entry), which are unclaimed. The delta gate for code that has no claim yet. |
|
|
25
26
|
|
|
26
27
|
Shared definitions (verdicts, fault classes, provenance, annotations) live in
|
|
27
28
|
[`schemas/common.schema.json`](schemas/common.schema.json). Gate behaviour —
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -6,7 +6,7 @@ import { fingerprint } from './fingerprint.mjs';
|
|
|
6
6
|
|
|
7
7
|
const schemaDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'schemas');
|
|
8
8
|
|
|
9
|
-
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status']);
|
|
9
|
+
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status', 'gate']);
|
|
10
10
|
export const PASSING_VERDICTS = Object.freeze(new Set(['killed']));
|
|
11
11
|
|
|
12
12
|
const ajv = new Ajv2020({ strict: true, allErrors: true });
|
|
@@ -99,8 +99,26 @@ const semantic = {
|
|
|
99
99
|
status(doc) {
|
|
100
100
|
const errors = [];
|
|
101
101
|
if (['write-test', 'review-fault-change'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
102
|
+
if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
|
|
102
103
|
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
103
104
|
if (doc.state === 'clean' && (doc.counts.new ?? 0) > 0) errors.push({ path: '/state', message: 'clean with new findings' });
|
|
105
|
+
if (doc.state === 'unclaimed-changes' && !(doc.changes && doc.changes.uncovered.length > 0)) errors.push({ path: '/changes/uncovered', message: 'unclaimed-changes requires at least one uncovered changed file' });
|
|
106
|
+
if (doc.state === 'unclaimed-changes' && doc.next.action !== 'claim') errors.push({ path: '/next/action', message: 'unclaimed-changes requires next.action = claim' });
|
|
107
|
+
if (doc.changes && doc.changes.uncovered.length > 0 && !['no-claims', 'unclaimed-changes'].includes(doc.state)) errors.push({ path: '/state', message: 'uncovered changed files are hidden behind a later state; unclaimed-changes precedes every evidence state' });
|
|
108
|
+
return errors;
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
gate(doc) {
|
|
112
|
+
const errors = [];
|
|
113
|
+
if (doc.evaluated !== doc.covered.length + doc.uncovered.length) errors.push({ path: '/evaluated', message: `evaluated (${doc.evaluated}) must equal covered (${doc.covered.length}) + uncovered (${doc.uncovered.length})` });
|
|
114
|
+
if (doc.changed !== doc.evaluated + doc.excluded.length) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length})` });
|
|
115
|
+
if (doc.uncovered.length > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'an uncovered changed file must exit 1; the gate never passes over unclaimed code' });
|
|
116
|
+
if (doc.uncovered.length === 0 && doc.exitCode === 1 && !(doc.strict && doc.changed > 0 && doc.evaluated === 0)) errors.push({ path: '/exitCode', message: 'exit 1 without uncovered files is only valid under --strict when a non-empty change evaluated nothing' });
|
|
117
|
+
doc.covered.forEach((c, i) => {
|
|
118
|
+
if (c.by === 'ignore' && !c.pattern) errors.push({ path: `/covered/${i}/pattern`, message: 'a file covered by an ignore entry must name the pattern that excused it' });
|
|
119
|
+
if (c.by !== 'ignore' && !c.claimIds?.length) errors.push({ path: `/covered/${i}/claimIds`, message: `a file covered by a ${c.by} must name the claim(s)` });
|
|
120
|
+
});
|
|
121
|
+
for (const e of doc.expired) if (!e.expires) errors.push({ path: '/expired', message: `expired entry "${e.pattern}" has no expires instant` });
|
|
104
122
|
return errors;
|
|
105
123
|
},
|
|
106
124
|
|
|
@@ -39,12 +39,22 @@
|
|
|
39
39
|
"type": "object",
|
|
40
40
|
"required": ["action", "command", "why"],
|
|
41
41
|
"properties": {
|
|
42
|
-
"action": { "enum": ["scaffold", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
42
|
+
"action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
43
43
|
"command": { "type": "string", "minLength": 1, "maxLength": 1024 },
|
|
44
44
|
"why": { "type": "string", "minLength": 1, "maxLength": 2000 }
|
|
45
45
|
},
|
|
46
46
|
"additionalProperties": false
|
|
47
47
|
},
|
|
48
|
+
"unclaimed": {
|
|
49
|
+
"description": "Changed files that carry no claim, when the brief was built with a change reference. Rendered before the findings: an agent writes the claim before more code.",
|
|
50
|
+
"type": "object",
|
|
51
|
+
"required": ["ref", "files"],
|
|
52
|
+
"properties": {
|
|
53
|
+
"ref": { "type": "string", "minLength": 1, "maxLength": 256 },
|
|
54
|
+
"files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } }
|
|
55
|
+
},
|
|
56
|
+
"additionalProperties": false
|
|
57
|
+
},
|
|
48
58
|
"items": {
|
|
49
59
|
"type": "array",
|
|
50
60
|
"maxItems": 50,
|
|
@@ -77,6 +77,33 @@
|
|
|
77
77
|
},
|
|
78
78
|
"additionalProperties": false
|
|
79
79
|
},
|
|
80
|
+
"uncoveredFile": {
|
|
81
|
+
"description": "A changed source or test file that no claim covers and no unexpired ignore entry excuses. Shared by the gate and status documents.",
|
|
82
|
+
"type": "object",
|
|
83
|
+
"required": ["file", "kind", "suggestion"],
|
|
84
|
+
"properties": {
|
|
85
|
+
"file": { "$ref": "#/$defs/repoPath" },
|
|
86
|
+
"kind": { "enum": ["source", "test"] },
|
|
87
|
+
"nearestClaimId": {
|
|
88
|
+
"description": "The claim whose fault files share the longest directory prefix with this file — where to attach, not a verdict.",
|
|
89
|
+
"$ref": "#/$defs/identifier"
|
|
90
|
+
},
|
|
91
|
+
"suggestion": { "type": "string", "minLength": 1, "maxLength": 1024 }
|
|
92
|
+
},
|
|
93
|
+
"additionalProperties": false
|
|
94
|
+
},
|
|
95
|
+
"ignoreReliance": {
|
|
96
|
+
"description": "An ignore entry the gate relied on (or would have, had it not expired), with the changed files it matched.",
|
|
97
|
+
"type": "object",
|
|
98
|
+
"required": ["pattern", "reason", "files"],
|
|
99
|
+
"properties": {
|
|
100
|
+
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
|
|
101
|
+
"reason": { "type": "string", "minLength": 8, "maxLength": 2000 },
|
|
102
|
+
"expires": { "$ref": "#/$defs/isoDateTime" },
|
|
103
|
+
"files": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repoPath" } }
|
|
104
|
+
},
|
|
105
|
+
"additionalProperties": false
|
|
106
|
+
},
|
|
80
107
|
"annotation": {
|
|
81
108
|
"description": "Strictly additive. An annotation never changes, suppresses, or drops a finding; it only attaches context to it.",
|
|
82
109
|
"type": "object",
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "urn:guard-spec:v1:gate",
|
|
4
|
+
"title": "Guard spec v1 — change gate",
|
|
5
|
+
"description": "Claim coverage of one change. `probe` asks whether the tests defend the claims that exist; `gate` asks whether the files a change touched carry any claim at all. Measured per file against the merge-base of a reference and HEAD (or the working tree). A file is covered when a fault anchors to it, when it is a test file that defends a claim, or when an unexpired ignore entry excuses it — and every reliance on an ignore entry is listed, so a reviewer sees why the gate passed.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schemaVersion", "tool", "generatedAt", "ref", "base", "head", "includeDirty", "strict", "changed", "evaluated", "excluded", "covered", "uncovered", "reliedOn", "expired", "exitCode"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"$schema": { "type": "string" },
|
|
10
|
+
"schemaVersion": { "const": 1 },
|
|
11
|
+
"tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
|
|
12
|
+
"generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
|
|
13
|
+
"ref": { "description": "The reference the change was measured against, as given (a branch, a tag, a sha).", "type": "string", "minLength": 1, "maxLength": 256 },
|
|
14
|
+
"base": { "description": "merge-base(ref, HEAD): the commit the change is relative to.", "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
15
|
+
"head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
16
|
+
"includeDirty": { "description": "True when the working tree was compared instead of HEAD.", "type": "boolean" },
|
|
17
|
+
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
|
+
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
19
|
+
"evaluated": { "description": "covered + uncovered: the changed files that are source and not excluded.", "type": "integer", "minimum": 0 },
|
|
20
|
+
"excluded": {
|
|
21
|
+
"type": "array",
|
|
22
|
+
"items": {
|
|
23
|
+
"type": "object",
|
|
24
|
+
"required": ["file", "by"],
|
|
25
|
+
"properties": {
|
|
26
|
+
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
|
|
27
|
+
"by": { "description": "`non-source`, `default:<glob>` or `exclude:<glob>`.", "type": "string", "minLength": 1, "maxLength": 600 }
|
|
28
|
+
},
|
|
29
|
+
"additionalProperties": false
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"covered": {
|
|
33
|
+
"type": "array",
|
|
34
|
+
"items": {
|
|
35
|
+
"type": "object",
|
|
36
|
+
"required": ["file", "by"],
|
|
37
|
+
"properties": {
|
|
38
|
+
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
|
|
39
|
+
"by": { "enum": ["fault", "defender", "ignore"] },
|
|
40
|
+
"claimIds": { "type": "array", "minItems": 1, "items": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" } },
|
|
41
|
+
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 }
|
|
42
|
+
},
|
|
43
|
+
"additionalProperties": false
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"uncovered": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } },
|
|
47
|
+
"reliedOn": { "description": "Unexpired ignore entries that excused at least one changed file. The gate passed *because of* these; a reviewer reads them.", "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } },
|
|
48
|
+
"expired": { "description": "Expired ignore entries that would have matched a changed file. They excuse nothing; the files are reported as uncovered.", "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } },
|
|
49
|
+
"exitCode": { "enum": [0, 1] }
|
|
50
|
+
},
|
|
51
|
+
"additionalProperties": false
|
|
52
|
+
}
|
|
@@ -11,17 +11,21 @@
|
|
|
11
11
|
"tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
|
|
12
12
|
"generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
|
|
13
13
|
"state": {
|
|
14
|
-
"description": "no-claims: no claims file. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
15
|
-
"enum": ["no-claims", "unprobed", "evidence-stale", "provisional-only", "unproven", "clean"]
|
|
14
|
+
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
15
|
+
"enum": ["no-claims", "unclaimed-changes", "unprobed", "evidence-stale", "provisional-only", "unproven", "clean"]
|
|
16
16
|
},
|
|
17
17
|
"next": {
|
|
18
|
-
"description": "The one action that moves the project forward. `command` is a shell line when the action is a tool command, or an instruction when it is not (write-test).",
|
|
18
|
+
"description": "The one action that moves the project forward. `command` is a shell line when the action is a tool command, or an instruction when it is not (write-test, claim for a test file).",
|
|
19
19
|
"type": "object",
|
|
20
20
|
"required": ["action", "command", "why"],
|
|
21
21
|
"properties": {
|
|
22
|
-
"action": { "enum": ["scaffold", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
22
|
+
"action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
23
23
|
"command": { "type": "string", "minLength": 1, "maxLength": 1024 },
|
|
24
24
|
"why": { "type": "string", "minLength": 1, "maxLength": 2000 },
|
|
25
|
+
"file": {
|
|
26
|
+
"description": "For claim (and for scaffold when a change is known): the first uncovered changed file the action is about.",
|
|
27
|
+
"$ref": "urn:guard-spec:v1:common#/$defs/repoPath"
|
|
28
|
+
},
|
|
25
29
|
"target": {
|
|
26
30
|
"description": "For write-test / review-fault-change: the finding the action is about.",
|
|
27
31
|
"type": "object",
|
|
@@ -95,10 +99,28 @@
|
|
|
95
99
|
"additionalProperties": false
|
|
96
100
|
}
|
|
97
101
|
},
|
|
102
|
+
"changes": {
|
|
103
|
+
"description": "Claim coverage of the change since `ref`, present only when a reference was given (--changed) or detected from CI. Uncovered files come first in every rendering: the claim is written before more code. Same per-file shapes as the gate document.",
|
|
104
|
+
"type": "object",
|
|
105
|
+
"required": ["ref", "base", "changed", "evaluated", "excluded", "uncovered", "reliedOn", "expired"],
|
|
106
|
+
"properties": {
|
|
107
|
+
"ref": { "type": "string", "minLength": 1, "maxLength": 256 },
|
|
108
|
+
"base": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
109
|
+
"includeDirty": { "type": "boolean" },
|
|
110
|
+
"changed": { "type": "integer", "minimum": 0 },
|
|
111
|
+
"evaluated": { "type": "integer", "minimum": 0 },
|
|
112
|
+
"excluded": { "type": "integer", "minimum": 0 },
|
|
113
|
+
"uncovered": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } },
|
|
114
|
+
"reliedOn": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } },
|
|
115
|
+
"expired": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } }
|
|
116
|
+
},
|
|
117
|
+
"additionalProperties": false
|
|
118
|
+
},
|
|
98
119
|
"paths": {
|
|
99
120
|
"type": "object",
|
|
100
121
|
"properties": {
|
|
101
122
|
"claims": { "type": "string" },
|
|
123
|
+
"ignore": { "type": "string" },
|
|
102
124
|
"evidence": { "type": "string" },
|
|
103
125
|
"provisionalEvidence": { "type": "string" },
|
|
104
126
|
"baseline": { "type": "string" }
|
package/src/brief/brief.mjs
CHANGED
|
@@ -31,7 +31,7 @@ function orderItems(a, b) {
|
|
|
31
31
|
return (b.isNew - a.isNew) || (ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict)) || ((b.rank ?? 0) - (a.rank ?? 0));
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString(), next } = {}) {
|
|
34
|
+
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString(), next, changes } = {}) {
|
|
35
35
|
const g = gate(evidence.records, baseline);
|
|
36
36
|
const toItem = (r, isNew) => ({
|
|
37
37
|
fingerprint: r.fingerprint,
|
|
@@ -54,7 +54,8 @@ export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Dat
|
|
|
54
54
|
new: g.new.length + g.belowFloor.length,
|
|
55
55
|
baselined: g.baselined.length,
|
|
56
56
|
};
|
|
57
|
-
const
|
|
57
|
+
const unclaimed = changes?.uncovered?.length ? { ref: changes.ref, files: changes.uncovered } : undefined;
|
|
58
|
+
const doc = { schemaVersion: 1, tool: evidence.tool, generatedAt, head: evidence.run.repo.head, heading: HEADING, summary, ...(next ? { next: { action: next.action, command: next.command, why: next.why } } : {}), ...(unclaimed ? { unclaimed } : {}), items, text: '' };
|
|
58
59
|
doc.text = renderBriefText({ ...doc, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length });
|
|
59
60
|
return doc;
|
|
60
61
|
}
|
|
@@ -68,6 +69,13 @@ export function renderBriefText(brief, { hasBaseline, total }) {
|
|
|
68
69
|
`testguard ${brief.tool.version}${brief.head ? ` @ ${brief.head.slice(0, 12)}` : ''} — ${brief.summary.claims} claims, ${total} faults probed, ${unproven} unproven` +
|
|
69
70
|
(unproven === 0 ? '.' : hasBaseline ? ` (${brief.summary.new} new since baseline).` : ' (no baseline; everything is new).'),
|
|
70
71
|
];
|
|
72
|
+
// Unclaimed changes come before everything else: the claim is written
|
|
73
|
+
// before more code, and TestGuard is silent about unclaimed code otherwise.
|
|
74
|
+
if (brief.unclaimed) {
|
|
75
|
+
const n = brief.unclaimed.files.length;
|
|
76
|
+
lines.push('', `UNCLAIMED CHANGES since ${brief.unclaimed.ref}: ${n} changed file${n === 1 ? '' : 's'} carr${n === 1 ? 'ies' : 'y'} no claim. State the claim first; nothing below can see this code.`);
|
|
77
|
+
for (const u of brief.unclaimed.files) lines.push(` - ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
78
|
+
}
|
|
71
79
|
if (brief.next) lines.push('', `NEXT [${brief.next.action}]: ${brief.next.command}`, ` why: ${brief.next.why}`);
|
|
72
80
|
if (brief.items.length === 0) {
|
|
73
81
|
lines.push('', 'Every probed claim is defended. Keep it that way: new claims need a fault and a test that fails on it.');
|
|
@@ -87,3 +95,19 @@ export function renderBriefText(brief, { hasBaseline, total }) {
|
|
|
87
95
|
if (unproven > brief.items.length) lines.push('', `… and ${unproven - brief.items.length} more in the evidence file.`);
|
|
88
96
|
return lines.join('\n') + '\n';
|
|
89
97
|
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* A brief for a project that has unclaimed changes but no evidence yet. The
|
|
101
|
+
* session-start hook must still tell the agent to write the claim; an empty
|
|
102
|
+
* summary is honest, silence is not.
|
|
103
|
+
*/
|
|
104
|
+
export function buildUnclaimedBrief({ tool, next, changes, generatedAt = new Date().toISOString() }) {
|
|
105
|
+
const summary = { claims: 0, byVerdict: {}, new: 0, baselined: 0 };
|
|
106
|
+
const doc = { schemaVersion: 1, tool, generatedAt, heading: HEADING, summary, ...(next ? { next: { action: next.action, command: next.command, why: next.why } } : {}), unclaimed: { ref: changes.ref, files: changes.uncovered }, items: [], text: '' };
|
|
107
|
+
const lines = [HEADING, '', `testguard ${tool.version} — no evidence yet; ${changes.uncovered.length} unclaimed changed file${changes.uncovered.length === 1 ? '' : 's'} since ${changes.ref}.`];
|
|
108
|
+
lines.push('', `UNCLAIMED CHANGES since ${changes.ref}: state the claim first; nothing can be probed for this code until it has one.`);
|
|
109
|
+
for (const u of changes.uncovered) lines.push(` - ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
110
|
+
if (next) lines.push('', `NEXT [${next.action}]: ${next.command}`, ` why: ${next.why}`);
|
|
111
|
+
doc.text = lines.join('\n') + '\n';
|
|
112
|
+
return doc;
|
|
113
|
+
}
|
package/src/cli.mjs
CHANGED
|
@@ -13,6 +13,7 @@ import { briefCommand } from './commands/brief.mjs';
|
|
|
13
13
|
import { scaffoldCommand } from './commands/scaffold.mjs';
|
|
14
14
|
import { statusCommand } from './commands/status.mjs';
|
|
15
15
|
import { initCommand } from './commands/init.mjs';
|
|
16
|
+
import { gateCommand } from './commands/gate.mjs';
|
|
16
17
|
|
|
17
18
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
18
19
|
|
|
@@ -24,6 +25,7 @@ const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a
|
|
|
24
25
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
25
26
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
26
27
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
28
|
+
testguard gate [dir] fail when a changed source file carries no claim and no excusing ignore entry
|
|
27
29
|
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
28
30
|
|
|
29
31
|
probe
|
|
@@ -49,17 +51,24 @@ probe
|
|
|
49
51
|
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
50
52
|
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
51
53
|
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
52
|
-
|
|
54
|
+
gate --changed <ref> measure the change since merge-base(ref, HEAD); auto-detected in GitHub Actions / GitLab CI
|
|
55
|
+
--include-dirty compare the working tree (staged, unstaged and untracked) instead of HEAD — the pre-commit shape
|
|
56
|
+
--exclude <glob> (repeatable) more files that never carry claims; --explain lists the defaults
|
|
57
|
+
--strict a non-empty change that evaluates nothing is a failure, not a note
|
|
58
|
+
--ignore <path> ignore file (default: <dir>/testguard.ignore.json; kind=path entries excuse files, with a reason)
|
|
59
|
+
exit 0 every changed source file is claimed or excused · 1 unclaimed file · 2 cannot evaluate · 3 no reference
|
|
60
|
+
status --json (exit 0 clean · 1 unproven/stale/unclaimed · 2 nothing to probe yet)
|
|
61
|
+
--changed <ref> also compute claim coverage of the change; unclaimed-changes then precedes every evidence state
|
|
53
62
|
init --force (replace an existing skill file) --json
|
|
54
63
|
every command accepts --json; probe/baseline emit the status document plus their own result
|
|
55
64
|
claims --json
|
|
56
65
|
baseline --evidence <path> --out <path> --allow-provisional (freeze unconfirmed evidence; normally refused)
|
|
57
66
|
brief --evidence <path> --baseline <path> --max <n> --text (print only; safe for hooks)
|
|
58
67
|
|
|
59
|
-
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
|
|
68
|
+
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or unclaimed changes) · 2 precondition failed · 3 usage
|
|
60
69
|
`;
|
|
61
70
|
|
|
62
|
-
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand };
|
|
71
|
+
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand };
|
|
63
72
|
|
|
64
73
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
65
74
|
let parsed;
|
|
@@ -79,6 +88,11 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
79
88
|
ref: { type: 'string', default: 'HEAD' },
|
|
80
89
|
claim: { type: 'string' },
|
|
81
90
|
'include-dirty': { type: 'boolean', default: false },
|
|
91
|
+
changed: { type: 'string' },
|
|
92
|
+
exclude: { type: 'string', multiple: true },
|
|
93
|
+
strict: { type: 'boolean', default: false },
|
|
94
|
+
explain: { type: 'boolean', default: false },
|
|
95
|
+
ignore: { type: 'string' },
|
|
82
96
|
'allow-provisional': { type: 'boolean', default: false },
|
|
83
97
|
force: { type: 'boolean', default: false },
|
|
84
98
|
verbose: { type: 'boolean', default: false },
|
package/src/commands/brief.mjs
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBrief } from '../brief/brief.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
|
|
5
5
|
import { computeStatus } from '../status/status.mjs';
|
|
6
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
7
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
7
8
|
|
|
8
9
|
export async function briefCommand({ projectDir, values, version }, io) {
|
|
9
10
|
const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
11
|
+
let status;
|
|
12
|
+
try {
|
|
13
|
+
// A detected base that does not resolve is silent here: the brief is a
|
|
14
|
+
// session-start hook's output and must never add noise; `status` and
|
|
15
|
+
// `gate` are where that warning is printed.
|
|
16
|
+
status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }));
|
|
17
|
+
} catch {
|
|
18
|
+
status = undefined; // a brief must never fail because status could not be computed
|
|
19
|
+
}
|
|
10
20
|
if (!existsSync(evPath)) {
|
|
11
|
-
// A missing brief must never break an agent's session start
|
|
12
|
-
|
|
21
|
+
// A missing brief must never break an agent's session start — but unclaimed
|
|
22
|
+
// changes are still said, because the claim comes before the probe.
|
|
23
|
+
if (values.text) {
|
|
24
|
+
if (status?.changes?.uncovered?.length) io.out(buildUnclaimedBrief({ tool: { name: 'testguard', version }, next: status.next, changes: status.changes }).text.trimEnd());
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
13
27
|
io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
|
|
14
28
|
return 2;
|
|
15
29
|
}
|
|
@@ -21,13 +35,7 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
21
35
|
io.err('--max must be an integer from 1 to 50');
|
|
22
36
|
return 3;
|
|
23
37
|
}
|
|
24
|
-
|
|
25
|
-
try {
|
|
26
|
-
next = computeStatus({ projectDir, toolVersion: version }).next;
|
|
27
|
-
} catch {
|
|
28
|
-
next = undefined; // a brief must never fail because status could not be computed
|
|
29
|
-
}
|
|
30
|
-
const brief = buildBrief(evidence, baseline, { max, next });
|
|
38
|
+
const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes });
|
|
31
39
|
if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
32
40
|
io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
33
41
|
return 0;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { join, resolve } from 'node:path';
|
|
2
|
+
import { computeChangedGate, resolveChangedRef, renderGate, DEFAULT_EXCLUDES } from '../gate/changed.mjs';
|
|
3
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.json');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `testguard gate --changed <ref>`: fail when a changed source file carries
|
|
9
|
+
* no claim and no excusing ignore entry. Exit 0 covered · 1 uncovered (or
|
|
10
|
+
* --strict with nothing evaluated) · 2 cannot evaluate · 3 no reference.
|
|
11
|
+
*/
|
|
12
|
+
export async function gateCommand({ projectDir, values, version }, io) {
|
|
13
|
+
if (values.explain) {
|
|
14
|
+
io.out('Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):');
|
|
15
|
+
for (const g of DEFAULT_EXCLUDES) io.out(` ${g}`);
|
|
16
|
+
io.out('Non-source files (anything but .js .mjs .cjs .ts .mts .cts .jsx .tsx) are excluded before these patterns apply.');
|
|
17
|
+
return 0;
|
|
18
|
+
}
|
|
19
|
+
const resolved = resolveChangedRef({ explicit: values.changed });
|
|
20
|
+
if (!resolved) {
|
|
21
|
+
io.err('gate needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. In GitHub Actions and GitLab CI the base branch is detected automatically.');
|
|
22
|
+
return 3;
|
|
23
|
+
}
|
|
24
|
+
const ref = resolved.ref;
|
|
25
|
+
// The gate itself never degrades: a detected reference that does not resolve is exit 2 here, because the gate's whole job is the measurement.
|
|
26
|
+
if (!resolved.required && !values.json && !values.quiet) io.err(`--changed not given; using ${ref} from ${resolved.from}`);
|
|
27
|
+
const doc = computeChangedGate({
|
|
28
|
+
projectDir,
|
|
29
|
+
ref,
|
|
30
|
+
includeDirty: values['include-dirty'],
|
|
31
|
+
exclude: values.exclude ?? [],
|
|
32
|
+
strict: values.strict,
|
|
33
|
+
toolVersion: version,
|
|
34
|
+
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
35
|
+
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
|
36
|
+
});
|
|
37
|
+
const outPath = values.out ? resolve(values.out) : gatePath(projectDir);
|
|
38
|
+
writeSpecDoc('gate', outPath, doc);
|
|
39
|
+
if (values.json) {
|
|
40
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
41
|
+
} else {
|
|
42
|
+
io.out(renderGate(doc));
|
|
43
|
+
if (doc.uncovered.length) io.out(`Next: state the claim for each UNCLAIMED file (scaffold proposes the faults), or add a testguard.ignore.json path entry with a reason that a reviewer will accept.`);
|
|
44
|
+
if (!values.quiet) io.out(`gate: ${outPath}`);
|
|
45
|
+
}
|
|
46
|
+
return doc.exitCode;
|
|
47
|
+
}
|
package/src/commands/probe.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
|
|
|
6
6
|
import { gate } from '../baseline/baseline.mjs';
|
|
7
7
|
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
8
8
|
import { computeStatus } from '../status/status.mjs';
|
|
9
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
9
10
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
10
11
|
|
|
11
12
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -58,7 +59,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
58
59
|
|
|
59
60
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
60
61
|
if (values.json) {
|
|
61
|
-
const status = computeStatus({ projectDir, toolVersion: version });
|
|
62
|
+
const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
|
|
62
63
|
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
|
|
63
64
|
return g.new.length > 0 ? 1 : 0;
|
|
64
65
|
}
|
package/src/commands/status.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { computeStatus, renderStatus } from '../status/status.mjs';
|
|
2
2
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
3
4
|
|
|
4
5
|
export async function statusCommand({ projectDir, values, version }, io) {
|
|
5
|
-
const doc = computeStatus({ projectDir, toolVersion: version });
|
|
6
|
+
const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
|
|
6
7
|
const result = validate('status', doc);
|
|
7
8
|
if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
8
9
|
io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { existsSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { extname, join, relative, resolve, dirname } from 'node:path';
|
|
3
|
+
import { git, repoRoot, headSha, GitError } from '../git.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { globToRegExp } from '../util/glob.mjs';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Claim coverage of a change.
|
|
12
|
+
*
|
|
13
|
+
* `probe` answers "do the tests defend the claims that exist?". This answers
|
|
14
|
+
* the question every field report so far turned on: "does the code I just
|
|
15
|
+
* changed have any claim at all?". A repository with ten old claims and one
|
|
16
|
+
* new, unclaimed feature exits 0 forever under `probe`; under `gate` the
|
|
17
|
+
* delta is measured, file by file, and an uncovered file is a finding.
|
|
18
|
+
*
|
|
19
|
+
* The unit is the file, on purpose: a fault anchors to a file, so the file is
|
|
20
|
+
* the smallest unit the rest of the contract already understands.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
|
|
24
|
+
const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
25
|
+
|
|
26
|
+
/** Files that are source by extension but never carry claims. Printed by --explain. */
|
|
27
|
+
export const DEFAULT_EXCLUDES = Object.freeze([
|
|
28
|
+
'**/*.d.ts',
|
|
29
|
+
'**/*.config.*',
|
|
30
|
+
'**/*.stories.*',
|
|
31
|
+
'**/*.generated.*',
|
|
32
|
+
'**/fixtures/**',
|
|
33
|
+
'**/__fixtures__/**',
|
|
34
|
+
'**/__mocks__/**',
|
|
35
|
+
'**/__snapshots__/**',
|
|
36
|
+
'.testguard/**',
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The reference the change is measured against. Explicit flag first, then the
|
|
43
|
+
* environments CI sets, then nothing: a wrong default would make the gate pass
|
|
44
|
+
* trivially (an upstream that already contains the change has an empty diff),
|
|
45
|
+
* so no ref is an error the caller reports, never a silent guess.
|
|
46
|
+
*/
|
|
47
|
+
export function detectChangedRef(env = process.env) {
|
|
48
|
+
if (env.TESTGUARD_CHANGED_REF) return { ref: env.TESTGUARD_CHANGED_REF, from: 'TESTGUARD_CHANGED_REF' };
|
|
49
|
+
// GitHub Actions, pull_request events.
|
|
50
|
+
if (env.GITHUB_BASE_REF) return { ref: `origin/${env.GITHUB_BASE_REF}`, from: 'GITHUB_BASE_REF' };
|
|
51
|
+
// GitLab merge request pipelines. The diff base sha is exact and needs no
|
|
52
|
+
// remote-tracking ref; the target branch name is the fallback (it needs
|
|
53
|
+
// `git fetch origin <branch>` or GIT_DEPTH: 0 on the job).
|
|
54
|
+
const sha = env.CI_MERGE_REQUEST_DIFF_BASE_SHA;
|
|
55
|
+
if (sha && /^[0-9a-f]{7,40}$/i.test(sha) && !/^0+$/.test(sha)) return { ref: sha, from: 'CI_MERGE_REQUEST_DIFF_BASE_SHA' };
|
|
56
|
+
if (env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME) return { ref: `origin/${env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME}`, from: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME' };
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Resolve the reference a command should measure against, and whether it may
|
|
62
|
+
* be dropped on failure. An explicit `--changed` is the user's word: if it
|
|
63
|
+
* does not resolve, that is an error. A reference detected from CI variables
|
|
64
|
+
* is a convenience: if it does not resolve (shallow clone, temp directory,
|
|
65
|
+
* no remote) the command says so on stderr and continues without a change
|
|
66
|
+
* measurement, because `status` and `brief` must keep working everywhere.
|
|
67
|
+
*/
|
|
68
|
+
export function resolveChangedRef({ explicit, env = process.env } = {}) {
|
|
69
|
+
if (explicit) return { ref: explicit, from: '--changed', required: true };
|
|
70
|
+
const d = detectChangedRef(env);
|
|
71
|
+
return d ? { ...d, required: false } : null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Run `compute(ref)` for a resolved reference. A required reference propagates
|
|
76
|
+
* its error; a detected one that fails yields `undefined` after `warn(message)`.
|
|
77
|
+
*/
|
|
78
|
+
export function withChangedRef(resolved, compute, warn = () => {}) {
|
|
79
|
+
if (!resolved) return compute(undefined);
|
|
80
|
+
try {
|
|
81
|
+
return compute(resolved.ref);
|
|
82
|
+
} catch (e) {
|
|
83
|
+
if (resolved.required || !(e instanceof GitError)) throw e;
|
|
84
|
+
warn(`change coverage skipped: ${resolved.ref} (from ${resolved.from}) does not resolve here — ${e.message.split('. ')[0]}. Pass --changed <ref> to make this an error.`);
|
|
85
|
+
return compute(undefined);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Files changed since the merge-base of `ref` and HEAD, repo-root-relative,
|
|
91
|
+
* deletions dropped. With `includeDirty` the working tree (tracked edits,
|
|
92
|
+
* staged or not, plus untracked non-ignored files) is compared instead of HEAD.
|
|
93
|
+
*/
|
|
94
|
+
export function changedFiles({ root, ref, includeDirty = false }) {
|
|
95
|
+
let base;
|
|
96
|
+
try {
|
|
97
|
+
base = git(['merge-base', ref, 'HEAD'], root);
|
|
98
|
+
} catch (e) {
|
|
99
|
+
throw new GitError(`cannot resolve --changed ${ref}: ${e.message.split('\n').pop()}. In CI the base must be present locally: GitHub Actions — actions/checkout with fetch-depth: 0; GitLab — GIT_DEPTH: 0 or \`git fetch origin <target branch>\` before the job (merge request pipelines also provide CI_MERGE_REQUEST_DIFF_BASE_SHA, which needs no fetch).`);
|
|
100
|
+
}
|
|
101
|
+
const head = headSha(root);
|
|
102
|
+
const out = new Set(git(['diff', '--name-only', '--diff-filter=d', '-z', base, ...(includeDirty ? [] : ['HEAD'])], root).split('\0').filter(Boolean));
|
|
103
|
+
if (includeDirty) for (const f of git(['ls-files', '--others', '--exclude-standard', '-z'], root).split('\0').filter(Boolean)) out.add(f);
|
|
104
|
+
return { base, head, files: [...out].sort() };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const isExpired = (entry, now) => Boolean(entry.expires) && new Date(entry.expires).getTime() <= now.getTime();
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The claim to attach to, if any: one whose fault files live in the same
|
|
111
|
+
* directory as the changed file. A claim that merely shares `src/` is not
|
|
112
|
+
* "nearest", it is noise, and a wrong `--claim` hint is worse than none.
|
|
113
|
+
*/
|
|
114
|
+
function nearestClaim(file, faultFiles) {
|
|
115
|
+
const dir = dirname(file);
|
|
116
|
+
for (const [ff, ids] of faultFiles) if (dirname(ff) === dir) return ids[0];
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Compute the gate for one change. Pure apart from reading the repository,
|
|
122
|
+
* the claims file and the ignore file; the caller decides what to print and
|
|
123
|
+
* what to write.
|
|
124
|
+
*/
|
|
125
|
+
export function computeChangedGate({ projectDir, ref, includeDirty = false, exclude = [], strict = false, now = new Date(), toolVersion = '0.0.0', generatedAt = now.toISOString(), claimsPath, ignorePath }) {
|
|
126
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
127
|
+
const root = repoRoot(projectDir);
|
|
128
|
+
const relProject = relative(root, projectDir).split('\\').join('/');
|
|
129
|
+
const { base, head, files } = changedFiles({ root, ref, includeDirty });
|
|
130
|
+
|
|
131
|
+
// Repo-root paths → project paths; a change outside the project directory is not this project's change.
|
|
132
|
+
const changed = [];
|
|
133
|
+
for (const f of files) {
|
|
134
|
+
if (!relProject) { changed.push(f); continue; }
|
|
135
|
+
if (f.startsWith(relProject + '/')) changed.push(f.slice(relProject.length + 1));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const cPath = claimsPath ?? defaultClaimsPath(projectDir);
|
|
139
|
+
const claims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
|
|
140
|
+
const iPath = ignorePath ?? defaultIgnorePath(projectDir);
|
|
141
|
+
const ignore = existsSync(iPath) ? readSpecDoc('ignore', iPath) : { entries: [] };
|
|
142
|
+
|
|
143
|
+
// Coverage index: which files carry a fault, which test files defend a claim.
|
|
144
|
+
const faultFiles = new Map(); // file → claimIds
|
|
145
|
+
const defenderFiles = new Map(); // test file → claimIds
|
|
146
|
+
for (const c of claims.claims) {
|
|
147
|
+
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
148
|
+
const defenders = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
|
|
149
|
+
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const pathEntries = ignore.entries.filter((e) => e.kind === 'path').map((e) => ({ entry: e, re: globToRegExp(e.pattern), expired: isExpired(e, now), files: [] }));
|
|
153
|
+
const userExcludes = exclude.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
154
|
+
const defaultExcludes = DEFAULT_EXCLUDES.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
155
|
+
|
|
156
|
+
const excluded = [];
|
|
157
|
+
const covered = [];
|
|
158
|
+
const uncovered = [];
|
|
159
|
+
for (const file of changed) {
|
|
160
|
+
if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
|
|
161
|
+
const ux = userExcludes.find((x) => x.re.test(file));
|
|
162
|
+
if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
|
|
163
|
+
const dx = defaultExcludes.find((x) => x.re.test(file));
|
|
164
|
+
if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
|
|
165
|
+
|
|
166
|
+
const isTest = TEST_RE.test(file);
|
|
167
|
+
if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
|
|
168
|
+
if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
|
|
169
|
+
const active = pathEntries.find((p) => !p.expired && p.re.test(file));
|
|
170
|
+
if (active) { active.files.push(file); covered.push({ file, by: 'ignore', pattern: active.entry.pattern }); continue; }
|
|
171
|
+
for (const p of pathEntries) if (p.expired && p.re.test(file)) p.files.push(file);
|
|
172
|
+
|
|
173
|
+
// Uncovered. Point at a same-directory claim so the author knows where to attach.
|
|
174
|
+
const nearest = nearestClaim(file, faultFiles);
|
|
175
|
+
const item = {
|
|
176
|
+
file,
|
|
177
|
+
kind: isTest ? 'test' : 'source',
|
|
178
|
+
...(nearest ? { nearestClaimId: nearest } : {}),
|
|
179
|
+
suggestion: isTest
|
|
180
|
+
? `add ${file} to the defendedBy of the claim it defends${nearest ? ` (nearest: ${nearest})` : ''}, or state a new claim for what it asserts`
|
|
181
|
+
: `testguard scaffold ${file}${nearest ? ` --claim ${nearest}` : ''}`,
|
|
182
|
+
};
|
|
183
|
+
uncovered.push(item);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const evaluated = covered.length + uncovered.length;
|
|
187
|
+
const reliedOn = pathEntries.filter((p) => !p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, ...(p.entry.expires ? { expires: p.entry.expires } : {}), files: p.files }));
|
|
188
|
+
const expired = pathEntries.filter((p) => p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, expires: p.entry.expires, files: p.files }));
|
|
189
|
+
const exitCode = uncovered.length > 0 ? 1 : strict && changed.length > 0 && evaluated === 0 ? 1 : 0;
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
schemaVersion: 1,
|
|
193
|
+
tool: { name: 'testguard', version: toolVersion },
|
|
194
|
+
generatedAt,
|
|
195
|
+
ref,
|
|
196
|
+
base,
|
|
197
|
+
head,
|
|
198
|
+
includeDirty,
|
|
199
|
+
strict,
|
|
200
|
+
changed: changed.length,
|
|
201
|
+
evaluated,
|
|
202
|
+
excluded,
|
|
203
|
+
covered,
|
|
204
|
+
uncovered,
|
|
205
|
+
reliedOn,
|
|
206
|
+
expired,
|
|
207
|
+
exitCode,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function renderGate(doc) {
|
|
212
|
+
const lines = [];
|
|
213
|
+
const where = doc.includeDirty ? 'working tree' : `HEAD ${doc.head.slice(0, 7)}`;
|
|
214
|
+
lines.push(`gate: ${doc.changed} file${doc.changed === 1 ? '' : 's'} changed since ${doc.ref} (merge-base ${doc.base.slice(0, 7)}) in the ${where}; ${doc.evaluated} evaluated, ${doc.excluded.length} excluded, ${doc.covered.length} covered, ${doc.uncovered.length} uncovered`);
|
|
215
|
+
for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
|
|
216
|
+
for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
217
|
+
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
|
|
218
|
+
if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
|
|
219
|
+
if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
|
|
220
|
+
if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
|
|
221
|
+
return lines.join('\n');
|
|
222
|
+
}
|
package/src/init/init.mjs
CHANGED
|
@@ -14,7 +14,7 @@ Before writing or changing tests, run \`testguard status --json\` and follow \`n
|
|
|
14
14
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
15
15
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
16
16
|
${AGENTS_END}`;
|
|
17
|
-
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/scaffold-*.json'];
|
|
17
|
+
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Install the agent operating layer into a consumer project: the skill, the
|
|
@@ -21,13 +21,31 @@ which files exist. `state` is one of:
|
|
|
21
21
|
| state | meaning | you do |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
|
+
| `unclaimed-changes` | files you changed carry no claim (only when a reference is known: `--changed <ref>`, or CI's base branch) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
24
25
|
| `unprobed` | claims never probed | `testguard probe` |
|
|
25
26
|
| `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
|
|
26
27
|
| `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
|
|
27
28
|
| `unproven` | a finding is not covered by the baseline | `next.target` names it; see the verdict table |
|
|
28
29
|
| `clean` | everything killed or baselined | `testguard baseline` if `next` says so; otherwise nothing |
|
|
29
30
|
|
|
30
|
-
Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
31
|
+
Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
32
|
+
|
|
33
|
+
## Every change needs a claim
|
|
34
|
+
|
|
35
|
+
`probe` verifies only the claims that exist; it is silent about unclaimed
|
|
36
|
+
code by construction. So, for every source file you create or change:
|
|
37
|
+
|
|
38
|
+
1. `testguard gate --changed HEAD --include-dirty` (the pre-commit shape; in a
|
|
39
|
+
PR, `--changed origin/<base>`). One unclaimed file exits `1`.
|
|
40
|
+
2. For each `UNCLAIMED` file: `testguard scaffold <file>` proposes faults;
|
|
41
|
+
state what the code guarantees; add the claim, its faults and its
|
|
42
|
+
`defendedBy` to `testguard.claims.json`. A new test file must be named in
|
|
43
|
+
some claim's `defendedBy`, or it is unclaimed too.
|
|
44
|
+
3. Only when a file genuinely carries nothing to claim (generated code, a
|
|
45
|
+
thin wrapper whose logic is claimed elsewhere): a `path` entry in
|
|
46
|
+
`testguard.ignore.json` with a reason and, where possible, an `expires`.
|
|
47
|
+
The gate prints every entry it relied on; a reviewer reads them.
|
|
48
|
+
4. Then `testguard probe --claim <ID> --include-dirty` for the new claim.
|
|
31
49
|
|
|
32
50
|
## Verdict → action
|
|
33
51
|
|
package/src/status/status.mjs
CHANGED
|
@@ -7,6 +7,7 @@ import { hashFile, sha256 } from '../util/hash.mjs';
|
|
|
7
7
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
8
8
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
9
|
import { sortForReport } from '../render.mjs';
|
|
10
|
+
import { computeChangedGate, defaultIgnorePath } from '../gate/changed.mjs';
|
|
10
11
|
|
|
11
12
|
export const faultContentHash = (fault) => sha256(`${fault.find}\n${fault.replace}`);
|
|
12
13
|
|
|
@@ -22,7 +23,7 @@ const P = (projectDir) => ({
|
|
|
22
23
|
* Reads the claims file, evidence, baseline and the working tree; never
|
|
23
24
|
* trusts a cached verdict whose inputs have changed.
|
|
24
25
|
*/
|
|
25
|
-
export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt = new Date().toISOString(), paths = P(projectDir), max = 20 }) {
|
|
26
|
+
export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt = new Date().toISOString(), paths = P(projectDir), max = 20, changedRef, includeDirty = false }) {
|
|
26
27
|
const rel = (p) => relative(projectDir, p) || '.';
|
|
27
28
|
const doc = {
|
|
28
29
|
schemaVersion: 1,
|
|
@@ -37,7 +38,32 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
37
38
|
findings: [],
|
|
38
39
|
paths: {},
|
|
39
40
|
};
|
|
40
|
-
|
|
41
|
+
// ── Claim coverage of the change, when a reference is known ──
|
|
42
|
+
// Computed before anything about evidence: a change that touches unclaimed
|
|
43
|
+
// code is the finding every field report shared, and TestGuard is silent
|
|
44
|
+
// about unclaimed code by construction. The claim is written first.
|
|
45
|
+
// The reference is the caller's: this function never reads the environment,
|
|
46
|
+
// so a library caller or a test in a temp directory is never surprised by CI.
|
|
47
|
+
const ref = changedRef;
|
|
48
|
+
let changes;
|
|
49
|
+
if (ref) {
|
|
50
|
+
const g = computeChangedGate({ projectDir, ref, includeDirty, toolVersion });
|
|
51
|
+
changes = { ref: g.ref, base: g.base, includeDirty: g.includeDirty, changed: g.changed, evaluated: g.evaluated, excluded: g.excluded.length, uncovered: g.uncovered, reliedOn: g.reliedOn, expired: g.expired };
|
|
52
|
+
doc.changes = changes;
|
|
53
|
+
if (existsSync(defaultIgnorePath(projectDir))) doc.paths.ignore = rel(defaultIgnorePath(projectDir));
|
|
54
|
+
}
|
|
55
|
+
const unclaimedWhy = () => {
|
|
56
|
+
const files = changes.uncovered.map((u) => u.file);
|
|
57
|
+
return `${files.length} changed file${files.length === 1 ? '' : 's'} since ${changes.ref} carr${files.length === 1 ? 'ies' : 'y'} no claim and no excusing ignore entry: ${files.slice(0, 3).join(', ')}${files.length > 3 ? ', …' : ''}. State the claim before writing more code — TestGuard is silent about unclaimed code by construction.`;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
if (!existsSync(paths.claims)) {
|
|
61
|
+
if (changes?.uncovered.length) {
|
|
62
|
+
const u = changes.uncovered[0];
|
|
63
|
+
doc.next = { action: 'scaffold', command: `testguard scaffold ${u.file}`, why: `No testguard.claims.json in this project, and ${unclaimedWhy()}`, file: u.file };
|
|
64
|
+
}
|
|
65
|
+
return doc;
|
|
66
|
+
}
|
|
41
67
|
doc.paths.claims = rel(paths.claims);
|
|
42
68
|
|
|
43
69
|
const claims = loadClaims(paths.claims);
|
|
@@ -46,6 +72,13 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
46
72
|
const faultIndex = new Map();
|
|
47
73
|
for (const c of claims.claims) for (const f of c.faults) faultIndex.set(`${c.id}/${f.id}`, { claim: c, fault: f });
|
|
48
74
|
|
|
75
|
+
if (changes && changes.uncovered.length > 0) {
|
|
76
|
+
const u = changes.uncovered[0];
|
|
77
|
+
doc.state = 'unclaimed-changes';
|
|
78
|
+
doc.next = { action: 'claim', command: u.suggestion, why: unclaimedWhy(), file: u.file };
|
|
79
|
+
return doc;
|
|
80
|
+
}
|
|
81
|
+
|
|
49
82
|
const hasEvidence = existsSync(paths.evidence);
|
|
50
83
|
const hasProvisional = existsSync(paths.provisional);
|
|
51
84
|
if (hasProvisional) doc.paths.provisionalEvidence = rel(paths.provisional);
|
|
@@ -155,6 +188,13 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
155
188
|
|
|
156
189
|
export function renderStatus(doc) {
|
|
157
190
|
const lines = [`state: ${doc.state}${doc.provisional ? ' (provisional)' : ''} — ${doc.counts.claims} claims / ${doc.counts.faults} faults` + (doc.counts.byVerdict ? `; ${Object.entries(doc.counts.byVerdict).map(([k, v]) => `${v} ${k}`).join(', ')}; ${doc.counts.new ?? 0} new, ${doc.counts.baselined ?? 0} baselined` : '')];
|
|
191
|
+
if (doc.changes) {
|
|
192
|
+
const c = doc.changes;
|
|
193
|
+
lines.push(`changes: ${c.changed} file${c.changed === 1 ? '' : 's'} since ${c.ref}; ${c.evaluated} evaluated, ${c.excluded} excluded, ${c.uncovered.length} unclaimed`);
|
|
194
|
+
for (const u of c.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
195
|
+
for (const r of c.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}`);
|
|
196
|
+
for (const e of c.expired) lines.push(`EXPIRED ignore "${e.pattern}" no longer excuses ${e.files.join(', ')}`);
|
|
197
|
+
}
|
|
158
198
|
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
159
199
|
for (const s of doc.stale.slice(0, 8)) lines.push(`stale ${s}`);
|
|
160
200
|
lines.push(`next: [${doc.next.action}] ${doc.next.command}`);
|