testguard-cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +30 -4
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +11 -1
- package/spec/README.md +2 -1
- package/spec/lib/validate.mjs +11 -1
- package/spec/schemas/brief.schema.json +11 -0
- package/spec/schemas/evidence.schema.json +9 -1
- package/spec/schemas/status.schema.json +110 -0
- package/src/brief/brief.mjs +5 -3
- package/src/cli.mjs +12 -2
- package/src/commands/baseline.mjs +11 -2
- package/src/commands/brief.mjs +10 -3
- package/src/commands/init.mjs +15 -0
- package/src/commands/probe.mjs +18 -7
- package/src/commands/status.mjs +10 -0
- package/src/init/init.mjs +78 -0
- package/src/init/templates/SKILL.md +63 -0
- package/src/probe/probe.mjs +2 -1
- package/src/render.mjs +7 -5
- package/src/status/status.mjs +163 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
The agent operating layer. TestGuard is meant to be run by AI agents; this
|
|
13
|
+
release gives them one source of truth, an installable loop, and makes the
|
|
14
|
+
one cheap way to game it visible.
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- **`testguard status --json`** — the machine entry point. A new spec kind
|
|
19
|
+
(`status.schema.json`): `state` (no-claims · unprobed · evidence-stale ·
|
|
20
|
+
provisional-only · unproven · clean) and the one `next` action, computed
|
|
21
|
+
from claims, evidence, baseline and the working tree. Exit 0 clean, 1
|
|
22
|
+
unproven/stale, 2 nothing to probe yet.
|
|
23
|
+
- **`testguard init`** — installs `.claude/skills/testguard/SKILL.md`, the
|
|
24
|
+
`brief --text` SessionStart hook (merged into an existing
|
|
25
|
+
`.claude/settings.json`), an `AGENTS.md` section and the `.gitignore`
|
|
26
|
+
lines. Idempotent; `--force` replaces the skill.
|
|
27
|
+
- **Fault edits are visible.** Evidence records `subject.contentHash`;
|
|
28
|
+
`status` lists `changedFaults` (a fault edited after it was probed, with
|
|
29
|
+
its previous verdict) and makes `review-fault-change` the next action when
|
|
30
|
+
the edited fault had survived. Gate rule 8. Self-claim
|
|
31
|
+
`TG-FAULT-EDIT-VISIBLE`.
|
|
32
|
+
- `--json` on every command; `probe` and `baseline` emit the status document
|
|
33
|
+
plus their result. `brief` carries `next` and prints it.
|
|
34
|
+
|
|
35
|
+
## [0.2.1] - 2026-09-17
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
|
|
39
|
+
- **Provisional verdicts** (#5). A run with `--confirm` below 3 declares
|
|
40
|
+
`run.provisional: true` — a spec field whose equivalence with
|
|
41
|
+
`confirmRuns < 3` the validator enforces both ways. Provisional verdicts
|
|
42
|
+
print with a `?`, the summary is prefixed `PROVISIONAL`, a warning is
|
|
43
|
+
printed on stderr, evidence is written to `.testguard/evidence-provisional.json`
|
|
44
|
+
so the canonical file only ever holds confirmed runs, `brief` warns at the
|
|
45
|
+
top, and `baseline` refuses provisional evidence unless
|
|
46
|
+
`--allow-provisional`. A confirmed prior verdict is never reused by a
|
|
47
|
+
provisional run, nor the reverse.
|
|
48
|
+
|
|
10
49
|
## [0.2.0] - 2026-09-17
|
|
11
50
|
|
|
12
51
|
### Added
|
package/README.md
CHANGED
|
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.3.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
58
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -62,6 +62,8 @@ less than that many days ago (`ENOVERSIONS`); install that one with
|
|
|
62
62
|
## How it works
|
|
63
63
|
|
|
64
64
|
```bash
|
|
65
|
+
npx testguard-cli init # install the agent layer: skill, session-start hook, AGENTS.md section
|
|
66
|
+
npx testguard-cli status --json # where the project is and the ONE next action — the machine entry point
|
|
65
67
|
npx testguard-cli claims # what does this project claim, and is every claim probeable?
|
|
66
68
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
67
69
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
@@ -112,7 +114,10 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
112
114
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
113
115
|
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
114
116
|
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
115
|
-
clean there; test files may be dirty)
|
|
117
|
+
clean there; test files may be dirty), optionally `--confirm 1` for a fast
|
|
118
|
+
**provisional** signal — verdicts print with a `?`, evidence goes to
|
|
119
|
+
`evidence-provisional.json`, and `baseline` refuses it; final pass with
|
|
120
|
+
defaults. By default
|
|
116
121
|
the stream shows only unproven faults plus a killed count — `--verbose`
|
|
117
122
|
shows every fault. A custom
|
|
118
123
|
runner (`pnpm --filter`, a specific config) goes in
|
|
@@ -136,6 +141,27 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
136
141
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
137
142
|
so the hook can never break a session.
|
|
138
143
|
|
|
144
|
+
### Built for agents to run
|
|
145
|
+
|
|
146
|
+
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
147
|
+
things make that safe:
|
|
148
|
+
|
|
149
|
+
- **One source of truth.** `testguard status --json` computes `state` and the
|
|
150
|
+
one `next` action from the claims file, the evidence, the baseline and the
|
|
151
|
+
working tree. Every human rendering — the CLI text, the session-start
|
|
152
|
+
brief, the skill — derives from it, so they cannot disagree. Every command
|
|
153
|
+
accepts `--json`.
|
|
154
|
+
- **An installable operating loop.** `testguard init` writes
|
|
155
|
+
`.claude/skills/testguard/SKILL.md` (state → action, verdict → the only
|
|
156
|
+
acceptable fix, the two-gate rule for any test the agent writes), the
|
|
157
|
+
`brief --text` session-start hook, an `AGENTS.md` section and the
|
|
158
|
+
`.gitignore` lines. Idempotent.
|
|
159
|
+
- **Gaming is visible.** The cheapest way to make a survivor disappear is to
|
|
160
|
+
weaken its fault, not to write a test. Evidence records every fault's
|
|
161
|
+
content hash; `status` lists any fault edited after it survived, with its
|
|
162
|
+
previous verdict, and makes reviewing that edit the next action. Editing a
|
|
163
|
+
claim is allowed — claims can be wrong — but it is never invisible.
|
|
164
|
+
|
|
139
165
|
### Authoring faults mechanically
|
|
140
166
|
|
|
141
167
|
Writing faults by hand means reading the code to find exact anchors. Two
|
|
@@ -189,8 +215,8 @@ through every verdict.
|
|
|
189
215
|
|
|
190
216
|
## Status
|
|
191
217
|
|
|
192
|
-
**v0.
|
|
193
|
-
mechanical scaffold
|
|
218
|
+
**v0.3.** Seven commands, vitest runner, hand-authored faults plus a
|
|
219
|
+
mechanical scaffold, and an agent operating layer (`status`, `init`). The contract
|
|
194
220
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
195
221
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
196
222
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -22,7 +22,11 @@ Rules that follow from the table:
|
|
|
22
22
|
1. **Green baseline first.** Defenders run N times unmodified before any fault
|
|
23
23
|
is applied. Anything short of N/N pass is `flaky-defender` and stops there.
|
|
24
24
|
2. **Confirm over N runs.** `killed` and `survived` both require exactly N
|
|
25
|
-
probe runs, all agreeing. Default N is 3.
|
|
25
|
+
probe runs, all agreeing. Default N is 3. **Fewer than three runs is
|
|
26
|
+
provisional**: the evidence declares `run.provisional: true`, every
|
|
27
|
+
rendering marks the verdicts as unconfirmed, and a provisional run is
|
|
28
|
+
never frozen into a baseline. Provisional runs exist for the fix loop —
|
|
29
|
+
a fast signal while writing a test — not for a gate.
|
|
26
30
|
3. **Only a test body rejecting the behaviour kills.** A test that fails by
|
|
27
31
|
assertion — or by an exception the fault provoked inside it — counts. A
|
|
28
32
|
timeout does not, and a suite that fails to load does not: neither is
|
|
@@ -46,6 +50,12 @@ Rules that follow from the table:
|
|
|
46
50
|
snapshot the working tree — otherwise the answer looks right and is not.
|
|
47
51
|
7. **Never a single global score.** Output is per claim, ranked. Blindness is
|
|
48
52
|
concentrated, and one number hides where.
|
|
53
|
+
8. **A changed fault is a finding.** Editing a claim is legitimate — claims
|
|
54
|
+
can be wrong — but the cheapest way to make a survivor disappear without
|
|
55
|
+
writing a test is to weaken its fault. Evidence records each fault's
|
|
56
|
+
content hash; the status document lists every fault whose content
|
|
57
|
+
changed since it was probed, with its previous verdict. The change is
|
|
58
|
+
allowed; it is never invisible.
|
|
49
59
|
|
|
50
60
|
## Baseline and delta
|
|
51
61
|
|
package/spec/README.md
CHANGED
|
@@ -20,7 +20,8 @@ The pattern all such tools run:
|
|
|
20
20
|
| `baseline` | [`schemas/baseline.schema.json`](schemas/baseline.schema.json) | Frozen fingerprints of existing debt. Gate only what is new. |
|
|
21
21
|
| `ignore` | [`schemas/ignore.schema.json`](schemas/ignore.schema.json) | Reviewable scoping. Every entry has a reason. |
|
|
22
22
|
| `calibration` | [`schemas/calibration.schema.json`](schemas/calibration.schema.json) | P(finding is real) per bucket, with a Wilson interval and the sample size behind it. |
|
|
23
|
-
| `brief` | [`schemas/brief.schema.json`](schemas/brief.schema.json) | What to tell an agent before it writes code — ranked, capped, never a single score. |
|
|
23
|
+
| `brief` | [`schemas/brief.schema.json`](schemas/brief.schema.json) | What to tell an agent before it writes code — ranked, capped, never a single score, and carrying the one next action. |
|
|
24
|
+
| `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed. |
|
|
24
25
|
|
|
25
26
|
Shared definitions (verdicts, fault classes, provenance, annotations) live in
|
|
26
27
|
[`schemas/common.schema.json`](schemas/common.schema.json). Gate behaviour —
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -6,7 +6,7 @@ import { fingerprint } from './fingerprint.mjs';
|
|
|
6
6
|
|
|
7
7
|
const schemaDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'schemas');
|
|
8
8
|
|
|
9
|
-
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief']);
|
|
9
|
+
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status']);
|
|
10
10
|
export const PASSING_VERDICTS = Object.freeze(new Set(['killed']));
|
|
11
11
|
|
|
12
12
|
const ajv = new Ajv2020({ strict: true, allErrors: true });
|
|
@@ -45,6 +45,8 @@ const semantic = {
|
|
|
45
45
|
evidence(doc) {
|
|
46
46
|
const errors = [];
|
|
47
47
|
const n = doc.run.confirmRuns;
|
|
48
|
+
if (n < 3 && doc.run.provisional !== true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is below 3; the run must declare provisional: true` });
|
|
49
|
+
if (n >= 3 && doc.run.provisional === true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is confirmed; provisional must be absent or false` });
|
|
48
50
|
doc.records.forEach((r, i) => {
|
|
49
51
|
const p = `/records/${i}`;
|
|
50
52
|
const expected = fingerprint({ claimId: r.claim.id, subjectId: r.subject.id, file: r.subject.file ?? '', verdict: r.verdict });
|
|
@@ -94,6 +96,14 @@ const semantic = {
|
|
|
94
96
|
return errors;
|
|
95
97
|
},
|
|
96
98
|
|
|
99
|
+
status(doc) {
|
|
100
|
+
const errors = [];
|
|
101
|
+
if (['write-test', 'review-fault-change'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
102
|
+
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
103
|
+
if (doc.state === 'clean' && (doc.counts.new ?? 0) > 0) errors.push({ path: '/state', message: 'clean with new findings' });
|
|
104
|
+
return errors;
|
|
105
|
+
},
|
|
106
|
+
|
|
97
107
|
brief(doc) {
|
|
98
108
|
const errors = [];
|
|
99
109
|
if (!doc.text.startsWith(doc.heading)) errors.push({ path: '/text', message: 'text must begin with heading' });
|
|
@@ -34,6 +34,17 @@
|
|
|
34
34
|
},
|
|
35
35
|
"additionalProperties": false
|
|
36
36
|
},
|
|
37
|
+
"next": {
|
|
38
|
+
"description": "The one action that moves the project forward, as computed by the status document; the brief carries it so an agent reading its session-start context knows what to do, not only what is wrong.",
|
|
39
|
+
"type": "object",
|
|
40
|
+
"required": ["action", "command", "why"],
|
|
41
|
+
"properties": {
|
|
42
|
+
"action": { "enum": ["scaffold", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
43
|
+
"command": { "type": "string", "minLength": 1, "maxLength": 1024 },
|
|
44
|
+
"why": { "type": "string", "minLength": 1, "maxLength": 2000 }
|
|
45
|
+
},
|
|
46
|
+
"additionalProperties": false
|
|
47
|
+
},
|
|
37
48
|
"items": {
|
|
38
49
|
"type": "array",
|
|
39
50
|
"maxItems": 50,
|
|
@@ -43,6 +43,10 @@
|
|
|
43
43
|
"type": "integer",
|
|
44
44
|
"minimum": 1
|
|
45
45
|
},
|
|
46
|
+
"provisional": {
|
|
47
|
+
"description": "True iff confirmRuns < 3. A provisional run's verdicts are not confirmed: they must not be frozen into a baseline and must be rendered as provisional wherever they appear. Validators enforce the equivalence in both directions.",
|
|
48
|
+
"type": "boolean"
|
|
49
|
+
},
|
|
46
50
|
"mode": {
|
|
47
51
|
"description": "`worktree` = faults were applied in a scratch git worktree (default). `in-place` = the user's working tree was mutated and restored.",
|
|
48
52
|
"enum": ["worktree", "in-place"]
|
|
@@ -122,7 +126,11 @@
|
|
|
122
126
|
"description": { "type": "string", "maxLength": 2000 },
|
|
123
127
|
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
|
|
124
128
|
"faultClass": { "$ref": "urn:guard-spec:v1:common#/$defs/faultClass" },
|
|
125
|
-
"producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" }
|
|
129
|
+
"producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
|
|
130
|
+
"contentHash": {
|
|
131
|
+
"description": "sha256 of the subject's content (for a fault: find + newline + replace). Excluded from the fingerprint on purpose, so repairing an anchor does not churn a baseline — and recorded here on purpose, so weakening a fault to make a finding disappear is visible.",
|
|
132
|
+
"$ref": "urn:guard-spec:v1:common#/$defs/sha256"
|
|
133
|
+
}
|
|
126
134
|
},
|
|
127
135
|
"additionalProperties": true
|
|
128
136
|
},
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "urn:guard-spec:v1:status",
|
|
4
|
+
"title": "Guard spec v1 — status",
|
|
5
|
+
"description": "The single machine-readable answer to 'where is this project and what should happen next'. Every human rendering (CLI text, agent brief, skill instructions) derives from this document so they cannot disagree. Computed from the claims file, the evidence, the baseline and the current working tree; never from a cached verdict alone.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schemaVersion", "tool", "generatedAt", "state", "next", "counts"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"$schema": { "type": "string" },
|
|
10
|
+
"schemaVersion": { "const": 1 },
|
|
11
|
+
"tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
|
|
12
|
+
"generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
|
|
13
|
+
"state": {
|
|
14
|
+
"description": "no-claims: no claims file. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
15
|
+
"enum": ["no-claims", "unprobed", "evidence-stale", "provisional-only", "unproven", "clean"]
|
|
16
|
+
},
|
|
17
|
+
"next": {
|
|
18
|
+
"description": "The one action that moves the project forward. `command` is a shell line when the action is a tool command, or an instruction when it is not (write-test).",
|
|
19
|
+
"type": "object",
|
|
20
|
+
"required": ["action", "command", "why"],
|
|
21
|
+
"properties": {
|
|
22
|
+
"action": { "enum": ["scaffold", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
23
|
+
"command": { "type": "string", "minLength": 1, "maxLength": 1024 },
|
|
24
|
+
"why": { "type": "string", "minLength": 1, "maxLength": 2000 },
|
|
25
|
+
"target": {
|
|
26
|
+
"description": "For write-test / review-fault-change: the finding the action is about.",
|
|
27
|
+
"type": "object",
|
|
28
|
+
"required": ["claimId", "subjectId"],
|
|
29
|
+
"properties": {
|
|
30
|
+
"claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
31
|
+
"subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
32
|
+
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
|
|
33
|
+
"verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" }
|
|
34
|
+
},
|
|
35
|
+
"additionalProperties": false
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"additionalProperties": false
|
|
39
|
+
},
|
|
40
|
+
"provisional": { "type": "boolean" },
|
|
41
|
+
"counts": {
|
|
42
|
+
"type": "object",
|
|
43
|
+
"required": ["claims", "faults"],
|
|
44
|
+
"properties": {
|
|
45
|
+
"claims": { "type": "integer", "minimum": 0 },
|
|
46
|
+
"faults": { "type": "integer", "minimum": 0 },
|
|
47
|
+
"byVerdict": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"propertyNames": { "enum": ["killed", "survived", "nocover", "unverifiable", "timeout", "fault-invalid", "flaky-defender"] },
|
|
50
|
+
"additionalProperties": { "type": "integer", "minimum": 0 }
|
|
51
|
+
},
|
|
52
|
+
"new": { "type": "integer", "minimum": 0 },
|
|
53
|
+
"baselined": { "type": "integer", "minimum": 0 }
|
|
54
|
+
},
|
|
55
|
+
"additionalProperties": false
|
|
56
|
+
},
|
|
57
|
+
"stale": {
|
|
58
|
+
"description": "Why the evidence no longer describes the working tree. Empty when it does.",
|
|
59
|
+
"type": "array",
|
|
60
|
+
"items": { "type": "string", "minLength": 1, "maxLength": 512 }
|
|
61
|
+
},
|
|
62
|
+
"changedFaults": {
|
|
63
|
+
"description": "Faults whose find/replace content differs from what the evidence probed. A changed fault that previously survived is the cheapest way to make a finding disappear without a test; it is surfaced, never hidden.",
|
|
64
|
+
"type": "array",
|
|
65
|
+
"items": {
|
|
66
|
+
"type": "object",
|
|
67
|
+
"required": ["claimId", "subjectId", "previousVerdict"],
|
|
68
|
+
"properties": {
|
|
69
|
+
"claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
70
|
+
"subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
71
|
+
"previousVerdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
|
|
72
|
+
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" }
|
|
73
|
+
},
|
|
74
|
+
"additionalProperties": false
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
"findings": {
|
|
78
|
+
"description": "Unproven findings from the current evidence, ranked, new first, capped.",
|
|
79
|
+
"type": "array",
|
|
80
|
+
"maxItems": 50,
|
|
81
|
+
"items": {
|
|
82
|
+
"type": "object",
|
|
83
|
+
"required": ["fingerprint", "claimId", "subjectId", "verdict", "severity", "isNew"],
|
|
84
|
+
"properties": {
|
|
85
|
+
"fingerprint": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" },
|
|
86
|
+
"claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
87
|
+
"subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
|
|
88
|
+
"verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
|
|
89
|
+
"reason": { "type": "string", "maxLength": 256 },
|
|
90
|
+
"severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
|
|
91
|
+
"file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
|
|
92
|
+
"rank": { "type": "number" },
|
|
93
|
+
"isNew": { "type": "boolean" }
|
|
94
|
+
},
|
|
95
|
+
"additionalProperties": false
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"paths": {
|
|
99
|
+
"type": "object",
|
|
100
|
+
"properties": {
|
|
101
|
+
"claims": { "type": "string" },
|
|
102
|
+
"evidence": { "type": "string" },
|
|
103
|
+
"provisionalEvidence": { "type": "string" },
|
|
104
|
+
"baseline": { "type": "string" }
|
|
105
|
+
},
|
|
106
|
+
"additionalProperties": false
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
"additionalProperties": false
|
|
110
|
+
}
|
package/src/brief/brief.mjs
CHANGED
|
@@ -31,7 +31,7 @@ function orderItems(a, b) {
|
|
|
31
31
|
return (b.isNew - a.isNew) || (ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict)) || ((b.rank ?? 0) - (a.rank ?? 0));
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString() } = {}) {
|
|
34
|
+
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString(), next } = {}) {
|
|
35
35
|
const g = gate(evidence.records, baseline);
|
|
36
36
|
const toItem = (r, isNew) => ({
|
|
37
37
|
fingerprint: r.fingerprint,
|
|
@@ -54,8 +54,8 @@ export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Dat
|
|
|
54
54
|
new: g.new.length + g.belowFloor.length,
|
|
55
55
|
baselined: g.baselined.length,
|
|
56
56
|
};
|
|
57
|
-
const doc = { schemaVersion: 1, tool: evidence.tool, generatedAt, head: evidence.run.repo.head, heading: HEADING, summary, items, text: '' };
|
|
58
|
-
doc.text = renderBriefText(doc, { hasBaseline: Boolean(baseline), total: evidence.records.length });
|
|
57
|
+
const doc = { schemaVersion: 1, tool: evidence.tool, generatedAt, head: evidence.run.repo.head, heading: HEADING, summary, ...(next ? { next: { action: next.action, command: next.command, why: next.why } } : {}), items, text: '' };
|
|
58
|
+
doc.text = renderBriefText({ ...doc, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length });
|
|
59
59
|
return doc;
|
|
60
60
|
}
|
|
61
61
|
|
|
@@ -64,9 +64,11 @@ export function renderBriefText(brief, { hasBaseline, total }) {
|
|
|
64
64
|
const lines = [
|
|
65
65
|
brief.heading,
|
|
66
66
|
'',
|
|
67
|
+
...(brief.provisional ? ['**PROVISIONAL** — this evidence came from fewer than three confirmation runs; treat every verdict below as unconfirmed and re-probe with --confirm 3 before acting on it.', ''] : []),
|
|
67
68
|
`testguard ${brief.tool.version}${brief.head ? ` @ ${brief.head.slice(0, 12)}` : ''} — ${brief.summary.claims} claims, ${total} faults probed, ${unproven} unproven` +
|
|
68
69
|
(hasBaseline ? ` (${brief.summary.new} new since baseline).` : ' (no baseline; everything is new).'),
|
|
69
70
|
];
|
|
71
|
+
if (brief.next) lines.push('', `NEXT [${brief.next.action}]: ${brief.next.command}`, ` why: ${brief.next.why}`);
|
|
70
72
|
if (brief.items.length === 0) {
|
|
71
73
|
lines.push('', 'Every probed claim is defended. Keep it that way: new claims need a fault and a test that fails on it.');
|
|
72
74
|
return lines.join('\n') + '\n';
|
package/src/cli.mjs
CHANGED
|
@@ -11,11 +11,15 @@ import { claimsCommand } from './commands/claims.mjs';
|
|
|
11
11
|
import { baselineCommand } from './commands/baseline.mjs';
|
|
12
12
|
import { briefCommand } from './commands/brief.mjs';
|
|
13
13
|
import { scaffoldCommand } from './commands/scaffold.mjs';
|
|
14
|
+
import { statusCommand } from './commands/status.mjs';
|
|
15
|
+
import { initCommand } from './commands/init.mjs';
|
|
14
16
|
|
|
15
17
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
16
18
|
|
|
17
19
|
const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a project makes
|
|
18
20
|
|
|
21
|
+
testguard status [dir] where the project is and the ONE next action; --json is the machine entry point
|
|
22
|
+
testguard init [dir] install the agent layer: skill, session-start hook, AGENTS.md section, .gitignore lines
|
|
19
23
|
testguard claims [dir] list the claims file and report drift against @claim annotations in code
|
|
20
24
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
21
25
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
@@ -33,6 +37,7 @@ probe
|
|
|
33
37
|
--claim <ID,ID> probe only these claims; writes .testguard/evidence-partial.json
|
|
34
38
|
--include-dirty probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
|
|
35
39
|
--verbose also print each killed fault (default: only unproven ones, plus a count)
|
|
40
|
+
--confirm below 3 is PROVISIONAL: verdicts print with "?", evidence goes to .testguard/evidence-provisional.json
|
|
36
41
|
--runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
|
|
37
42
|
--node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
|
|
38
43
|
--in-place mutate the working tree instead of a scratch worktree
|
|
@@ -43,14 +48,17 @@ probe
|
|
|
43
48
|
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
44
49
|
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
45
50
|
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
51
|
+
status --json (exit 0 clean · 1 unproven/stale · 2 nothing to probe yet)
|
|
52
|
+
init --force (replace an existing skill file) --json
|
|
53
|
+
every command accepts --json; probe/baseline emit the status document plus their own result
|
|
46
54
|
claims --json
|
|
47
|
-
baseline --evidence <path> --out <path>
|
|
55
|
+
baseline --evidence <path> --out <path> --allow-provisional (freeze unconfirmed evidence; normally refused)
|
|
48
56
|
brief --evidence <path> --baseline <path> --max <n> --text (print only; safe for hooks)
|
|
49
57
|
|
|
50
58
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
|
|
51
59
|
`;
|
|
52
60
|
|
|
53
|
-
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand };
|
|
61
|
+
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand };
|
|
54
62
|
|
|
55
63
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
56
64
|
let parsed;
|
|
@@ -70,6 +78,8 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
70
78
|
ref: { type: 'string', default: 'HEAD' },
|
|
71
79
|
claim: { type: 'string' },
|
|
72
80
|
'include-dirty': { type: 'boolean', default: false },
|
|
81
|
+
'allow-provisional': { type: 'boolean', default: false },
|
|
82
|
+
force: { type: 'boolean', default: false },
|
|
73
83
|
verbose: { type: 'boolean', default: false },
|
|
74
84
|
'runner-cmd': { type: 'string' },
|
|
75
85
|
'node-modules': { type: 'string' },
|
|
@@ -3,19 +3,28 @@ import { resolve } from 'node:path';
|
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
4
|
import { buildBaseline } from '../baseline/baseline.mjs';
|
|
5
5
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
6
|
+
import { computeStatus } from '../status/status.mjs';
|
|
6
7
|
|
|
7
|
-
export async function baselineCommand({ projectDir, values }, io) {
|
|
8
|
+
export async function baselineCommand({ projectDir, values, version }, io) {
|
|
8
9
|
const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
9
10
|
if (!existsSync(evPath)) {
|
|
10
11
|
io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
|
|
11
12
|
return 2;
|
|
12
13
|
}
|
|
13
14
|
const evidence = readSpecDoc('evidence', evPath);
|
|
15
|
+
if (evidence.run.provisional && !values['allow-provisional']) {
|
|
16
|
+
io.err(`error: ${evPath} is provisional (confirmRuns ${evidence.run.confirmRuns} < 3); a baseline must be frozen from confirmed evidence. Re-run probe with --confirm 3, or pass --allow-provisional if you accept unconfirmed findings as the frozen contract.`);
|
|
17
|
+
return 2;
|
|
18
|
+
}
|
|
14
19
|
const baseline = buildBaseline(evidence);
|
|
15
20
|
const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
|
|
16
21
|
writeSpecDoc('baseline', outPath, baseline);
|
|
17
22
|
const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
|
|
18
|
-
|
|
23
|
+
if (values.json) {
|
|
24
|
+
io.out(JSON.stringify({ ...computeStatus({ projectDir, toolVersion: version }), baseline: { path: outPath, frozen: n } }, null, 2));
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''}${evidence.run.provisional ? ' — FROM PROVISIONAL EVIDENCE (--allow-provisional)' : ''} → ${outPath}`);
|
|
19
28
|
io.out('Commit this file; from now on only new findings gate. Ignore the regenerated ones — add to .gitignore:');
|
|
20
29
|
io.out(' .testguard/evidence.json');
|
|
21
30
|
io.out(' .testguard/brief.json');
|
package/src/commands/brief.mjs
CHANGED
|
@@ -2,9 +2,10 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
4
|
import { buildBrief } from '../brief/brief.mjs';
|
|
5
|
+
import { computeStatus } from '../status/status.mjs';
|
|
5
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
6
7
|
|
|
7
|
-
export async function briefCommand({ projectDir, values }, io) {
|
|
8
|
+
export async function briefCommand({ projectDir, values, version }, io) {
|
|
8
9
|
const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
9
10
|
if (!existsSync(evPath)) {
|
|
10
11
|
// A missing brief must never break an agent's session start.
|
|
@@ -20,8 +21,14 @@ export async function briefCommand({ projectDir, values }, io) {
|
|
|
20
21
|
io.err('--max must be an integer from 1 to 50');
|
|
21
22
|
return 3;
|
|
22
23
|
}
|
|
23
|
-
|
|
24
|
+
let next;
|
|
25
|
+
try {
|
|
26
|
+
next = computeStatus({ projectDir, toolVersion: version }).next;
|
|
27
|
+
} catch {
|
|
28
|
+
next = undefined; // a brief must never fail because status could not be computed
|
|
29
|
+
}
|
|
30
|
+
const brief = buildBrief(evidence, baseline, { max, next });
|
|
24
31
|
if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
25
|
-
io.out(brief.text.trimEnd());
|
|
32
|
+
io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
26
33
|
return 0;
|
|
27
34
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { initProject } from '../init/init.mjs';
|
|
2
|
+
|
|
3
|
+
export async function initCommand({ projectDir, values }, io) {
|
|
4
|
+
const { done, skipped } = initProject({ projectDir, force: values.force });
|
|
5
|
+
if (values.json) {
|
|
6
|
+
io.out(JSON.stringify({ done, skipped }, null, 2));
|
|
7
|
+
return 0;
|
|
8
|
+
}
|
|
9
|
+
for (const d of done) io.out(`+ ${d}`);
|
|
10
|
+
for (const s of skipped) io.out(`= ${s}`);
|
|
11
|
+
io.out('');
|
|
12
|
+
io.out('Agents in this project now start with the blind-spot brief and can run `testguard status --json` to learn what to do next.');
|
|
13
|
+
io.out(done.length ? 'Commit these files.' : 'Nothing to change.');
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
package/src/commands/probe.mjs
CHANGED
|
@@ -4,7 +4,9 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
|
4
4
|
import { probe } from '../probe/probe.mjs';
|
|
5
5
|
import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { gate } from '../baseline/baseline.mjs';
|
|
7
|
-
import { renderRecord, renderSummary, sortForReport } from '../render.mjs';
|
|
7
|
+
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
8
|
+
import { computeStatus } from '../status/status.mjs';
|
|
9
|
+
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
8
10
|
|
|
9
11
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
10
12
|
export const baselinePath = (projectDir) => join(projectDir, '.testguard', 'baseline.json');
|
|
@@ -23,7 +25,10 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
23
25
|
}
|
|
24
26
|
const only = values.claim ? values.claim.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
|
|
25
27
|
// A --claim run is partial evidence; keep it away from the canonical file unless --out says otherwise.
|
|
26
|
-
const
|
|
28
|
+
const provisional = confirmRuns < 3;
|
|
29
|
+
// Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
|
|
30
|
+
const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
|
|
31
|
+
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
27
32
|
const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
|
|
28
33
|
const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
|
|
29
34
|
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
@@ -42,19 +47,24 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
42
47
|
escalate: !values['no-escalate'],
|
|
43
48
|
toolVersion: version,
|
|
44
49
|
includeDirty: values['include-dirty'],
|
|
45
|
-
onStage: !values.quiet && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
46
|
-
onProgress: values.quiet ? undefined : (r) => {
|
|
50
|
+
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
51
|
+
onProgress: values.quiet || values.json ? undefined : (r) => {
|
|
47
52
|
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
48
|
-
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r) + (r.reusedFrom ? ' (reused)' : ''));
|
|
53
|
+
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional }) + (r.reusedFrom ? ' (reused)' : ''));
|
|
49
54
|
},
|
|
50
55
|
});
|
|
51
56
|
writeSpecDoc('evidence', outPath, evidence);
|
|
52
57
|
|
|
53
58
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
59
|
+
if (values.json) {
|
|
60
|
+
const status = computeStatus({ projectDir, toolVersion: version });
|
|
61
|
+
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
|
|
62
|
+
return g.new.length > 0 ? 1 : 0;
|
|
63
|
+
}
|
|
54
64
|
if (!values.quiet && baseline) {
|
|
55
65
|
io.out('');
|
|
56
66
|
const tag = (r) => (g.new.includes(r) ? '[NEW] ' : g.baselined.includes(r) ? '[baseline] ' : '[below floor] ');
|
|
57
|
-
for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out(' ' + tag(r) + renderRecord(r));
|
|
67
|
+
for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out(' ' + tag(r) + renderRecord(r, { provisional }));
|
|
58
68
|
}
|
|
59
69
|
io.out('');
|
|
60
70
|
if (!values.quiet && !values.verbose) {
|
|
@@ -62,6 +72,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
62
72
|
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
63
73
|
}
|
|
64
74
|
io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
|
|
65
|
-
io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : ''}`);
|
|
75
|
+
io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : provisional ? ' (provisional; not the canonical evidence file)' : ''}`);
|
|
76
|
+
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
66
77
|
return g.new.length > 0 ? 1 : 0;
|
|
67
78
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { computeStatus, renderStatus } from '../status/status.mjs';
|
|
2
|
+
import { validate } from '../../spec/lib/validate.mjs';
|
|
3
|
+
|
|
4
|
+
export async function statusCommand({ projectDir, values, version }, io) {
|
|
5
|
+
const doc = computeStatus({ projectDir, toolVersion: version });
|
|
6
|
+
const result = validate('status', doc);
|
|
7
|
+
if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
8
|
+
io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
|
|
9
|
+
return doc.state === 'clean' ? 0 : doc.state === 'no-claims' || doc.state === 'unprobed' ? 2 : 1;
|
|
10
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join, dirname } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
|
|
5
|
+
const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
|
|
6
|
+
const HOOK_CMD = 'npx -y testguard-cli brief --text';
|
|
7
|
+
const AGENTS_BEGIN = '<!-- testguard:begin -->';
|
|
8
|
+
const AGENTS_END = '<!-- testguard:end -->';
|
|
9
|
+
const AGENTS_BLOCK = `${AGENTS_BEGIN}
|
|
10
|
+
## TestGuard
|
|
11
|
+
|
|
12
|
+
This project's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
|
|
13
|
+
Before writing or changing tests, run \`testguard status --json\` and follow \`next\`.
|
|
14
|
+
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
15
|
+
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
16
|
+
${AGENTS_END}`;
|
|
17
|
+
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/scaffold-*.json'];
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Install the agent operating layer into a consumer project: the skill, the
|
|
21
|
+
* session-start hook, an AGENTS.md section and the .gitignore lines.
|
|
22
|
+
* Idempotent: re-running changes nothing unless --force replaces the skill.
|
|
23
|
+
*/
|
|
24
|
+
export function initProject({ projectDir, force = false }) {
|
|
25
|
+
const done = [];
|
|
26
|
+
const skipped = [];
|
|
27
|
+
|
|
28
|
+
const skillDir = join(projectDir, '.claude', 'skills', 'testguard');
|
|
29
|
+
const skillPath = join(skillDir, 'SKILL.md');
|
|
30
|
+
if (!existsSync(skillPath) || force) {
|
|
31
|
+
mkdirSync(skillDir, { recursive: true });
|
|
32
|
+
writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
|
|
33
|
+
done.push(`.claude/skills/testguard/SKILL.md${force && existsSync(skillPath) ? ' (replaced)' : ''}`);
|
|
34
|
+
} else {
|
|
35
|
+
skipped.push('.claude/skills/testguard/SKILL.md exists (use --force to replace)');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const settingsPath = join(projectDir, '.claude', 'settings.json');
|
|
39
|
+
let settings = {};
|
|
40
|
+
if (existsSync(settingsPath)) {
|
|
41
|
+
try {
|
|
42
|
+
settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
43
|
+
} catch (e) {
|
|
44
|
+
throw new Error(`.claude/settings.json is not valid JSON (${e.message}); fix it before init can add the hook`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
settings.hooks ??= {};
|
|
48
|
+
settings.hooks.SessionStart ??= [];
|
|
49
|
+
const already = JSON.stringify(settings.hooks.SessionStart).includes(HOOK_CMD);
|
|
50
|
+
if (already) {
|
|
51
|
+
skipped.push('SessionStart hook already present in .claude/settings.json');
|
|
52
|
+
} else {
|
|
53
|
+
settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: HOOK_CMD }] });
|
|
54
|
+
mkdirSync(dirname(settingsPath), { recursive: true });
|
|
55
|
+
writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
56
|
+
done.push('.claude/settings.json: SessionStart hook → brief --text');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const agentsPath = join(projectDir, 'AGENTS.md');
|
|
60
|
+
const agents = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : '';
|
|
61
|
+
if (agents.includes(AGENTS_BEGIN)) {
|
|
62
|
+
skipped.push('AGENTS.md already has the TestGuard section');
|
|
63
|
+
} else {
|
|
64
|
+
writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + AGENTS_BLOCK + '\n');
|
|
65
|
+
done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const giPath = join(projectDir, '.gitignore');
|
|
69
|
+
const gi = existsSync(giPath) ? readFileSync(giPath, 'utf8') : '';
|
|
70
|
+
const missing = GITIGNORE_LINES.filter((l) => !gi.split('\n').some((x) => x.trim() === l));
|
|
71
|
+
if (missing.length) {
|
|
72
|
+
writeFileSync(giPath, (gi ? gi.replace(/\s*$/, '\n') : '') + '# TestGuard: regenerated per run (baseline.json IS committed)\n' + missing.join('\n') + '\n');
|
|
73
|
+
done.push(`.gitignore: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
|
|
74
|
+
} else {
|
|
75
|
+
skipped.push('.gitignore already ignores the regenerated files');
|
|
76
|
+
}
|
|
77
|
+
return { done, skipped };
|
|
78
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: testguard
|
|
3
|
+
description: Prove the test suite defends this project's claims. Use before writing or changing tests, after changing guarded code, and whenever a session starts with a TEST BLINDSPOT CONTEXT block.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# TestGuard — how an agent runs it
|
|
7
|
+
|
|
8
|
+
TestGuard injects the faults a project's claims forbid and reports every one
|
|
9
|
+
the tests fail to detect. It is a **claim verifier**, not a test generator.
|
|
10
|
+
You operate it through one loop and one source of truth.
|
|
11
|
+
|
|
12
|
+
## The one source of truth
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
testguard status --json
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Read `state` and `next`. Do what `next.command` says. Never infer state from
|
|
19
|
+
which files exist. `state` is one of:
|
|
20
|
+
|
|
21
|
+
| state | meaning | you do |
|
|
22
|
+
|---|---|---|
|
|
23
|
+
| `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
|
+
| `unprobed` | claims never probed | `testguard probe` |
|
|
25
|
+
| `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
|
|
26
|
+
| `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
|
|
27
|
+
| `unproven` | a finding is not covered by the baseline | `next.target` names it; see the verdict table |
|
|
28
|
+
| `clean` | everything killed or baselined | `testguard baseline` if `next` says so; otherwise nothing |
|
|
29
|
+
|
|
30
|
+
Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
31
|
+
|
|
32
|
+
## Verdict → action
|
|
33
|
+
|
|
34
|
+
| verdict | what it means | the only acceptable fix |
|
|
35
|
+
|---|---|---|
|
|
36
|
+
| `SURVIVED` | the defenders stayed green while the claim was false | **write a test** in the defender file that fails with the fault applied and passes on HEAD |
|
|
37
|
+
| `SURVIVED [killed-by-undeclared-tests: …]` | other tests catch it | add those files to the claim's `defendedBy` |
|
|
38
|
+
| `NOCOVER` | no test file imports the target | write a new test file that imports it and asserts the claim |
|
|
39
|
+
| `UNVERIFIABLE` | the fault's anchor no longer matches, or the defenders fail to load | fix the **fault definition** (or the defenders' import); never the code |
|
|
40
|
+
| `FAULT-INVALID` | the replacement does not compile | fix the fault definition |
|
|
41
|
+
| `TIMEOUT` | the fault makes the code hang | write an assertion that fails on it; a hang is not a detection |
|
|
42
|
+
| `FLAKY-DEFENDER` | defenders not green 3/3, or disagreed across runs | fix the flake first; no verdict is trustworthy until then |
|
|
43
|
+
|
|
44
|
+
## The fix loop
|
|
45
|
+
|
|
46
|
+
1. `testguard status --json` → take `next.target`.
|
|
47
|
+
2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
|
|
48
|
+
3. `testguard probe --claim <ID> --include-dirty` — probes your uncommitted test without touching the tree. `--confirm 1` gives a fast **provisional** signal; a `?` on a verdict means unconfirmed.
|
|
49
|
+
4. When it is `killed` at `--confirm 3`, commit the test.
|
|
50
|
+
5. When everything is killed or baselined, `testguard baseline` if `next` says so, and commit `.testguard/baseline.json`.
|
|
51
|
+
|
|
52
|
+
## Rules that are not yours to bend
|
|
53
|
+
|
|
54
|
+
- **Never make a fault die by editing the claims file.** Editing a claim is allowed when the claim was wrong — but the evidence records every fault's content hash, and `status` lists any fault edited after it survived, with its previous verdict, as `changedFaults`. The change is permitted; it is never invisible. Say why in the commit.
|
|
55
|
+
- **A test that asserts current behaviour is not a fix.** If the test would pass with the fault applied, it defends nothing.
|
|
56
|
+
- **Never re-implement the code under test inside the test.** Drive the real function; assert the outcome, not that a function was called.
|
|
57
|
+
- `--confirm 1` results are provisional: they print with `?`, go to `evidence-provisional.json`, and cannot be baselined.
|
|
58
|
+
- Worktree mode probes a **commit**. If `probe` refuses because defenders are dirty, use `--include-dirty`; do not commit half-written tests to satisfy it.
|
|
59
|
+
- Do not add `.testguard/evidence.json` or `brief.json` to git; do commit `baseline.json`.
|
|
60
|
+
|
|
61
|
+
## Reading the numbers
|
|
62
|
+
|
|
63
|
+
`N unproven faults across M claims` counts faults and the claims they belong to. There is no single score on purpose: blindness is concentrated, and one number hides where.
|
package/src/probe/probe.mjs
CHANGED
|
@@ -132,6 +132,7 @@ export async function probe({
|
|
|
132
132
|
repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
|
|
133
133
|
runner: { name: vitest.name, ...((runnerVersion ?? readRunnerVersion(projectDir)) ? { version: runnerVersion ?? readRunnerVersion(projectDir) } : {}) },
|
|
134
134
|
confirmRuns,
|
|
135
|
+
...(confirmRuns < 3 ? { provisional: true } : {}),
|
|
135
136
|
mode,
|
|
136
137
|
},
|
|
137
138
|
records,
|
|
@@ -232,7 +233,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
|
|
|
232
233
|
return {
|
|
233
234
|
fingerprint: fingerprint({ claimId: claim.id, subjectId: fault.id, file: fault.file, verdict }),
|
|
234
235
|
claim: { id: claim.id, statement: claim.statement, severity: claim.severity, source: claim.source, producedBy: claim.producedBy },
|
|
235
|
-
subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy },
|
|
236
|
+
subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy, contentHash: sha256(`${fault.find}\n${fault.replace}`) },
|
|
236
237
|
verdict,
|
|
237
238
|
detail,
|
|
238
239
|
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}) },
|
package/src/render.mjs
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender', 'killed'];
|
|
2
2
|
|
|
3
3
|
/** Non-passing verdicts shout; the one pass does not. */
|
|
4
|
-
export const formatVerdict = (v) => (v === 'killed' ? 'killed' : v.toUpperCase());
|
|
4
|
+
export const formatVerdict = (v, provisional = false) => (v === 'killed' ? 'killed' : v.toUpperCase()) + (provisional ? '?' : '');
|
|
5
5
|
|
|
6
|
-
export
|
|
7
|
-
|
|
6
|
+
export const PROVISIONAL_WARNING = (n) => `PROVISIONAL — confirmRuns ${n} (< 3): nothing below is confirmed. Verdicts carry a "?"; this evidence cannot be frozen into a baseline. Re-run with --confirm 3 before trusting it.`;
|
|
7
|
+
|
|
8
|
+
export function renderRecord(r, { provisional = false } = {}) {
|
|
9
|
+
const head = `${formatVerdict(r.verdict, provisional).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
|
|
8
10
|
let why = '';
|
|
9
11
|
if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
|
|
10
12
|
const files = [...new Set(r.detail.undeclaredKillers.map((k) => k.split('::')[0]))];
|
|
@@ -26,11 +28,11 @@ export function summarize(records) {
|
|
|
26
28
|
|
|
27
29
|
export function renderSummary(records, run) {
|
|
28
30
|
const byVerdict = summarize(records);
|
|
29
|
-
const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v)}`);
|
|
31
|
+
const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v, run?.provisional)}`);
|
|
30
32
|
const unproven = records.filter((r) => r.verdict !== 'killed');
|
|
31
33
|
const claims = new Set(unproven.map((r) => r.claim.id)).size;
|
|
32
34
|
const where = run ? ` Probed ${run.repo.snapshot ? `working tree (snapshot ${run.repo.snapshot.slice(0, 7)} of ${run.repo.head.slice(0, 7)})` : run.mode === 'in-place' ? `in place at ${run.repo.head.slice(0, 7)}${run.repo.dirty ? ' (dirty)' : ''}` : run.repo.head.slice(0, 7)}.` : '';
|
|
33
|
-
return `${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
|
|
35
|
+
return `${run?.provisional ? 'PROVISIONAL: ' : ''}${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
|
|
34
36
|
}
|
|
35
37
|
|
|
36
38
|
/** Survivors first, then by rank score; killed last. */
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { gate } from '../baseline/baseline.mjs';
|
|
6
|
+
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
7
|
+
import { resolveDefenders } from '../probe/runner-vitest.mjs';
|
|
8
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
|
+
import { sortForReport } from '../render.mjs';
|
|
10
|
+
|
|
11
|
+
export const faultContentHash = (fault) => sha256(`${fault.find}\n${fault.replace}`);
|
|
12
|
+
|
|
13
|
+
const P = (projectDir) => ({
|
|
14
|
+
claims: defaultClaimsPath(projectDir),
|
|
15
|
+
evidence: join(projectDir, '.testguard', 'evidence.json'),
|
|
16
|
+
provisional: join(projectDir, '.testguard', 'evidence-provisional.json'),
|
|
17
|
+
baseline: join(projectDir, '.testguard', 'baseline.json'),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The single answer to "where is this project and what happens next".
|
|
22
|
+
* Reads the claims file, evidence, baseline and the working tree; never
|
|
23
|
+
* trusts a cached verdict whose inputs have changed.
|
|
24
|
+
*/
|
|
25
|
+
export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt = new Date().toISOString(), paths = P(projectDir), max = 20 }) {
|
|
26
|
+
const rel = (p) => relative(projectDir, p) || '.';
|
|
27
|
+
const doc = {
|
|
28
|
+
schemaVersion: 1,
|
|
29
|
+
tool: { name: 'testguard', version: toolVersion },
|
|
30
|
+
generatedAt,
|
|
31
|
+
state: 'no-claims',
|
|
32
|
+
next: { action: 'scaffold', command: 'testguard scaffold <source-file>', why: 'No testguard.claims.json in this project. Scaffold proposes faults for a file; state what each guarantees, then probe.' },
|
|
33
|
+
provisional: false,
|
|
34
|
+
counts: { claims: 0, faults: 0 },
|
|
35
|
+
stale: [],
|
|
36
|
+
changedFaults: [],
|
|
37
|
+
findings: [],
|
|
38
|
+
paths: {},
|
|
39
|
+
};
|
|
40
|
+
if (!existsSync(paths.claims)) return doc;
|
|
41
|
+
doc.paths.claims = rel(paths.claims);
|
|
42
|
+
|
|
43
|
+
const claims = loadClaims(paths.claims);
|
|
44
|
+
doc.counts.claims = claims.claims.length;
|
|
45
|
+
doc.counts.faults = claims.claims.reduce((n, c) => n + c.faults.length, 0);
|
|
46
|
+
const faultIndex = new Map();
|
|
47
|
+
for (const c of claims.claims) for (const f of c.faults) faultIndex.set(`${c.id}/${f.id}`, { claim: c, fault: f });
|
|
48
|
+
|
|
49
|
+
const hasEvidence = existsSync(paths.evidence);
|
|
50
|
+
const hasProvisional = existsSync(paths.provisional);
|
|
51
|
+
if (hasProvisional) doc.paths.provisionalEvidence = rel(paths.provisional);
|
|
52
|
+
if (!hasEvidence) {
|
|
53
|
+
if (hasProvisional) {
|
|
54
|
+
doc.state = 'provisional-only';
|
|
55
|
+
doc.provisional = true;
|
|
56
|
+
doc.next = { action: 'probe', command: 'testguard probe --confirm 3', why: 'Only provisional (sub-N) evidence exists; nothing is confirmed until the defenders agree three times.' };
|
|
57
|
+
} else {
|
|
58
|
+
doc.state = 'unprobed';
|
|
59
|
+
doc.next = { action: 'probe', command: 'testguard probe', why: `${doc.counts.claims} claims / ${doc.counts.faults} faults have never been probed.` };
|
|
60
|
+
}
|
|
61
|
+
return doc;
|
|
62
|
+
}
|
|
63
|
+
doc.paths.evidence = rel(paths.evidence);
|
|
64
|
+
const evidence = readSpecDoc('evidence', paths.evidence);
|
|
65
|
+
const baseline = existsSync(paths.baseline) ? readSpecDoc('baseline', paths.baseline) : undefined;
|
|
66
|
+
if (baseline) doc.paths.baseline = rel(paths.baseline);
|
|
67
|
+
|
|
68
|
+
// ── Staleness: does the evidence still describe this claims file and this tree? ──
|
|
69
|
+
const probed = new Map(evidence.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]));
|
|
70
|
+
for (const key of faultIndex.keys()) if (!probed.has(key)) doc.stale.push(`fault ${key} has never been probed`);
|
|
71
|
+
for (const [key, r] of probed) {
|
|
72
|
+
const cur = faultIndex.get(key);
|
|
73
|
+
if (!cur) { doc.stale.push(`fault ${key} was probed but is no longer in the claims file`); continue; }
|
|
74
|
+
if (r.subject.contentHash && r.subject.contentHash !== faultContentHash(cur.fault)) {
|
|
75
|
+
doc.changedFaults.push({ claimId: cur.claim.id, subjectId: cur.fault.id, previousVerdict: r.verdict, file: cur.fault.file });
|
|
76
|
+
doc.stale.push(`fault ${key} changed since it was probed (was ${r.verdict})`);
|
|
77
|
+
}
|
|
78
|
+
const target = join(projectDir, cur.fault.file);
|
|
79
|
+
if (existsSync(target) && hashFile(target) !== r.inputs.targetHash) doc.stale.push(`${cur.fault.file} changed since ${key} was probed`);
|
|
80
|
+
const defenders = cur.claim.defendedBy?.length ? resolveDefenders(projectDir, cur.claim.defendedBy) : discoverDefenders(projectDir, cur.fault.file);
|
|
81
|
+
for (const d of defenders) {
|
|
82
|
+
const h = r.inputs.defenderHashes[d];
|
|
83
|
+
if (!h) doc.stale.push(`${d} now defends ${key} but was not probed`);
|
|
84
|
+
else if (existsSync(join(projectDir, d)) && hashFile(join(projectDir, d)) !== h) doc.stale.push(`${d} changed since ${key} was probed`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
doc.stale = [...new Set(doc.stale)];
|
|
88
|
+
|
|
89
|
+
// ── Findings, gated against the baseline ──
|
|
90
|
+
const g = gate(evidence.records, baseline);
|
|
91
|
+
const counts = {};
|
|
92
|
+
for (const r of evidence.records) counts[r.verdict] = (counts[r.verdict] ?? 0) + 1;
|
|
93
|
+
doc.counts.byVerdict = counts;
|
|
94
|
+
doc.counts.new = g.new.length + g.belowFloor.length;
|
|
95
|
+
doc.counts.baselined = g.baselined.length;
|
|
96
|
+
const ranked = sortForReport([...g.new, ...g.belowFloor, ...g.baselined]);
|
|
97
|
+
doc.findings = ranked.slice(0, max).map((r) => ({
|
|
98
|
+
fingerprint: r.fingerprint, claimId: r.claim.id, subjectId: r.subject.id, verdict: r.verdict,
|
|
99
|
+
...(r.detail.reason ? { reason: r.detail.reason } : {}), severity: r.claim.severity,
|
|
100
|
+
...(r.subject.file ? { file: r.subject.file } : {}), ...(r.rank ? { rank: r.rank.score } : {}),
|
|
101
|
+
isNew: !g.baselined.includes(r),
|
|
102
|
+
}));
|
|
103
|
+
|
|
104
|
+
// ── State and next ──
|
|
105
|
+
const changedSurvivor = doc.changedFaults.find((c) => c.previousVerdict !== 'killed');
|
|
106
|
+
if (changedSurvivor) {
|
|
107
|
+
doc.state = 'evidence-stale';
|
|
108
|
+
doc.next = {
|
|
109
|
+
action: 'review-fault-change',
|
|
110
|
+
command: `testguard probe --claim ${changedSurvivor.claimId} --include-dirty`,
|
|
111
|
+
why: `Fault ${changedSurvivor.claimId}/${changedSurvivor.subjectId} was edited after it ${changedSurvivor.previousVerdict}. Editing a claim is allowed, but a weakened fault is the cheapest way to make a finding vanish without a test — confirm the edit is a correction, then re-probe.`,
|
|
112
|
+
target: { claimId: changedSurvivor.claimId, subjectId: changedSurvivor.subjectId, file: changedSurvivor.file, verdict: changedSurvivor.previousVerdict },
|
|
113
|
+
};
|
|
114
|
+
return doc;
|
|
115
|
+
}
|
|
116
|
+
if (doc.stale.length) {
|
|
117
|
+
doc.state = 'evidence-stale';
|
|
118
|
+
doc.next = { action: 'probe', command: 'testguard probe --include-dirty', why: `${doc.stale.length} input${doc.stale.length === 1 ? '' : 's'} changed since the evidence was taken: ${doc.stale.slice(0, 3).join('; ')}${doc.stale.length > 3 ? '; …' : ''}.` };
|
|
119
|
+
return doc;
|
|
120
|
+
}
|
|
121
|
+
const top = doc.findings.find((f) => f.isNew);
|
|
122
|
+
if (top) {
|
|
123
|
+
doc.state = 'unproven';
|
|
124
|
+
const cur = faultIndex.get(`${top.claimId}/${top.subjectId}`);
|
|
125
|
+
const defenders = cur.claim.defendedBy?.length ? resolveDefenders(projectDir, cur.claim.defendedBy) : discoverDefenders(projectDir, cur.fault.file);
|
|
126
|
+
const where = defenders[0] ?? `a new test file that imports ${cur.fault.file}`;
|
|
127
|
+
const whyByVerdict = {
|
|
128
|
+
survived: `${top.claimId}/${top.subjectId} (${top.severity}) survived: "${cur.claim.statement}" can be false with the suite green.`,
|
|
129
|
+
nocover: `${top.claimId}/${top.subjectId} (${top.severity}) has no defender: no test file imports ${cur.fault.file}.`,
|
|
130
|
+
unverifiable: `${top.claimId}/${top.subjectId} cannot be probed (${top.reason ?? 'anchor'}); fix the fault definition in testguard.claims.json, not the code.`,
|
|
131
|
+
'fault-invalid': `${top.claimId}/${top.subjectId}'s replacement does not load; fix the fault definition.`,
|
|
132
|
+
timeout: `${top.claimId}/${top.subjectId} makes the defenders hang; a hang is not a detection — write an assertion that fails on it.`,
|
|
133
|
+
'flaky-defender': `${top.claimId}/${top.subjectId}'s defenders are not reliably green (${top.reason ?? ''}); fix the flake before any verdict can be trusted.`,
|
|
134
|
+
};
|
|
135
|
+
const fixClaim = ['unverifiable', 'fault-invalid'].includes(top.verdict);
|
|
136
|
+
doc.next = {
|
|
137
|
+
action: fixClaim ? 'probe' : 'write-test',
|
|
138
|
+
command: fixClaim
|
|
139
|
+
? `edit ${top.claimId}/${top.subjectId} in testguard.claims.json, then: testguard probe --claim ${top.claimId} --include-dirty`
|
|
140
|
+
: `write a test in ${where} that fails on ${top.claimId}/${top.subjectId} and passes on HEAD, then: testguard probe --claim ${top.claimId} --include-dirty`,
|
|
141
|
+
why: whyByVerdict[top.verdict] ?? `${top.claimId}/${top.subjectId} is ${top.verdict}.`,
|
|
142
|
+
target: { claimId: top.claimId, subjectId: top.subjectId, ...(top.file ? { file: top.file } : {}), verdict: top.verdict },
|
|
143
|
+
};
|
|
144
|
+
return doc;
|
|
145
|
+
}
|
|
146
|
+
doc.state = 'clean';
|
|
147
|
+
const unprovenTotal = evidence.records.filter((r) => r.verdict !== 'killed').length;
|
|
148
|
+
if (unprovenTotal > 0 && !baseline) {
|
|
149
|
+
doc.next = { action: 'baseline', command: 'testguard baseline', why: `${unprovenTotal} unproven finding${unprovenTotal === 1 ? '' : 's'} with no baseline; freeze them so only new ones gate from here.` };
|
|
150
|
+
} else {
|
|
151
|
+
doc.next = { action: 'none', command: 'testguard probe', why: unprovenTotal ? `${unprovenTotal} baselined finding${unprovenTotal === 1 ? '' : 's'} remain; nothing new. Re-probe after changing code, tests or claims.` : 'Every claim is defended. Re-probe after changing code, tests or claims; add a claim for every new invariant.' };
|
|
152
|
+
}
|
|
153
|
+
return doc;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export function renderStatus(doc) {
|
|
157
|
+
const lines = [`state: ${doc.state}${doc.provisional ? ' (provisional)' : ''} — ${doc.counts.claims} claims / ${doc.counts.faults} faults` + (doc.counts.byVerdict ? `; ${Object.entries(doc.counts.byVerdict).map(([k, v]) => `${v} ${k}`).join(', ')}; ${doc.counts.new ?? 0} new, ${doc.counts.baselined ?? 0} baselined` : '')];
|
|
158
|
+
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
159
|
+
for (const s of doc.stale.slice(0, 8)) lines.push(`stale ${s}`);
|
|
160
|
+
lines.push(`next: [${doc.next.action}] ${doc.next.command}`);
|
|
161
|
+
lines.push(`why: ${doc.next.why}`);
|
|
162
|
+
return lines.join('\n');
|
|
163
|
+
}
|