testguard-cli 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +19 -8
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +20 -0
- package/spec/lib/validate.mjs +4 -1
- package/spec/schemas/brief.schema.json +1 -1
- package/spec/schemas/status.schema.json +20 -1
- package/src/claims/anchors.mjs +218 -0
- package/src/cli.mjs +5 -2
- package/src/commands/claims.mjs +10 -2
- package/src/init/templates/SKILL.md +5 -1
- package/src/probe/cost.mjs +43 -6
- package/src/status/status.mjs +19 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,131 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.9.2] - 2026-09-19
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.9.1`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- feat(claims): preflight fault anchors (#126)
|
|
17
|
+
- chore(homebrew): sha256 for v0.9.1 (#125)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- `claims --check-anchors` locates every exact fault anchor and syntax-checks
|
|
23
|
+
supported in-memory JavaScript and Python replacements without starting a
|
|
24
|
+
runner or touching the tree. CI runs the sub-second preflight before the
|
|
25
|
+
self-probe, while `status` reports rotted anchors as `invalid-anchors` with
|
|
26
|
+
the precise fault to repair. Repairs remain a reviewed judgement call:
|
|
27
|
+
there is deliberately no fuzzy match or `--fix`.
|
|
28
|
+
|
|
29
|
+
## [0.9.1] - 2026-09-19
|
|
30
|
+
|
|
31
|
+
Automated weekly release — everything merged since `v0.9.0`.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- docs: the brief catches up with the tool, and stops falling behind (#112)
|
|
36
|
+
- chore(homebrew): sha256 for v0.9.0 (#110)
|
|
37
|
+
- feat(cost): two ceilings, the report where it is read, and three claims off expensive defenders (#113)
|
|
38
|
+
- fix(release): approve the runs GITHUB_TOKEN's own PR leaves held (#111)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
### Added
|
|
42
|
+
- **A second cost ceiling, on the claim.** The total alone cannot see the
|
|
43
|
+
failure it was meant to catch: a new calibration claim cost 51 s for two
|
|
44
|
+
faults purely by being written into `replay.test.mjs` rather than
|
|
45
|
+
`calibration.test.mjs`, and re-pointing it took it to 4.9 s while the total
|
|
46
|
+
never moved. A per-fault average is worse — the regression that prompted
|
|
47
|
+
this budget raised the fault count 59 to 91 while per-fault cost barely
|
|
48
|
+
moved, so an average would have reported everything fine.
|
|
49
|
+
|
|
50
|
+
`perClaimSeconds` is 100 s, just above today's worst claim
|
|
51
|
+
(`TG-ADMIT-NEEDS-ALL-KILLED`, **87 s in CI** for one fault), so the rule
|
|
52
|
+
reads: no new claim may be worse than the worst we already have. Calibrated
|
|
53
|
+
against CI and not the local figure — 75 s, set from that claim's local
|
|
54
|
+
66.5 s, tripped on the first CI run. This file already warns about exactly
|
|
55
|
+
that for the total (950 s was 18% over local but 4.6% over CI, and the first
|
|
56
|
+
PR adding a claim tripped it); the same error applies one level down. A breach names the claim
|
|
57
|
+
and its remedy — a cheaper defender, not a bigger budget. The field is
|
|
58
|
+
optional and additive: a budget without one behaves exactly as before.
|
|
59
|
+
- **The cost report goes to the CI job summary, not only the log.** The gate
|
|
60
|
+
went from 9.6 to 23.6 minutes across one merged pull request and nothing
|
|
61
|
+
said a word, because `--cost` only ever printed into a log nobody opens
|
|
62
|
+
while it is passing. The summary is read without opening anything, so the
|
|
63
|
+
number is in front of a reviewer while the change is still a change.
|
|
64
|
+
|
|
65
|
+
### Performance
|
|
66
|
+
- **Three claims moved off expensive shared defenders: 65 s, ~30 s and ~30 s
|
|
67
|
+
down to under 2 s each.** All three faulted a *pure* function while naming
|
|
68
|
+
an integration test file, and a claim pays for every test in the file it
|
|
69
|
+
names. `TG-SURVIVOR-PROVES-THE-SUBJECT-RUNS` faults `classify()` but was
|
|
70
|
+
defended by `negative-control.test.mjs`, which spawns real runners against
|
|
71
|
+
fixture repositories; `TG-LABEL-NEVER-GUESSES` faults `labelDiff()` and
|
|
72
|
+
`TG-REPLAY-FLAKY-IS-NEVER-CAUGHT` faults `classifyReplay()`, both defended
|
|
73
|
+
by `replay.test.mjs`, which builds a scripted git corpus.
|
|
74
|
+
|
|
75
|
+
Two were a straight move of an already-isolated `describe` block into
|
|
76
|
+
`test/label.test.mjs` and `test/replay-verdict.test.mjs`; only the `classify`
|
|
77
|
+
case needed a test written. Each was proved with `admit` — passes on HEAD and
|
|
78
|
+
fails on every fault, 3/3 — *before* the expensive defender was dropped, and
|
|
79
|
+
never the other way round. The integration files keep their remaining claims,
|
|
80
|
+
where the fixture is the point.
|
|
81
|
+
|
|
82
|
+
### Changed
|
|
83
|
+
- **The cost budget is 1230 s, from 1350.** 1350 was set defensively against a
|
|
84
|
+
reuse-affected 1123 s, because this budget is defined against a from-scratch
|
|
85
|
+
probe and none had run yet. `release.yml` then measured **1081 s** at 129
|
|
86
|
+
faults on the v0.9.0 release commit with no reuse — so the six field-report
|
|
87
|
+
claims cost +82 s, not the +120 s a per-fault average predicted. 1230 is
|
|
88
|
+
~14% over 1081, the same headroom rule that set 1100 against 962. Leaving it
|
|
89
|
+
at 1350 would have let a 25% regression pass unnoticed.
|
|
90
|
+
|
|
91
|
+
### Fixed
|
|
92
|
+
- **The release no longer waits for a human to click Approve.** v0.9.0 sat with
|
|
93
|
+
every required check green and merged nothing. The `pull_request` runs a
|
|
94
|
+
release PR creates are held at `action_required`, because the repository
|
|
95
|
+
requires approval from first-time contributors and `github-actions[bot]` is
|
|
96
|
+
permanently one — it never authors a merged commit, so it can never
|
|
97
|
+
graduate. Held runs leave check suites QUEUED on the head sha, the status
|
|
98
|
+
rollup never reaches a terminal state, and auto-merge waits on it even though
|
|
99
|
+
`test (20/22/24)` are already green from the dispatched run.
|
|
100
|
+
|
|
101
|
+
`scheduled-release.yml` now approves those runs itself. `GITHUB_TOKEN` is
|
|
102
|
+
permitted to do this — `actions: write` carries the approve permission,
|
|
103
|
+
measured 201 against a bot PR's own held run — and the workflow already held
|
|
104
|
+
that permission, so nothing was granted and no secret was added. It may still
|
|
105
|
+
not approve a *review*, which is a different gate and remains ours alone.
|
|
106
|
+
A run left held after the poll is an `::error::` naming the run, never a
|
|
107
|
+
silent wait.
|
|
108
|
+
|
|
109
|
+
This is the third `GITHUB_TOKEN` wall in the same chain, after "raises no
|
|
110
|
+
`pull_request` event" (#105) and "its push raises no `push` event" (the
|
|
111
|
+
hourly sweep in release.yml). All three have the same root: an action taken
|
|
112
|
+
by `GITHUB_TOKEN` does not trigger the event that would normally follow it.
|
|
113
|
+
|
|
114
|
+
### Documentation
|
|
115
|
+
- **The technical brief describes the tool that exists.** It was last rendered
|
|
116
|
+
when it covered one verb, `probe`, and said nothing about `replay` and its
|
|
117
|
+
calibration — the empirical answer to whether an injected fault resembles a
|
|
118
|
+
real bug, which is the question a document arguing "evidence, not output"
|
|
119
|
+
can least afford to omit. A new page covers the eleven verbs and the
|
|
120
|
+
calibration, carrying the README's own framing: the greenfield transfer is
|
|
121
|
+
unproven, and `replay` is the instrument rather than the answer. The verdict
|
|
122
|
+
table, the cost formula and the prior-art section were checked against the
|
|
123
|
+
code and left alone.
|
|
124
|
+
- **The brief's PDF is rendered by the release instead of by memory.** Its
|
|
125
|
+
masthead read `v0.6` through three releases because nothing ever ran
|
|
126
|
+
`build-one-pager.mjs`. The HTML masthead is now a version surface and
|
|
127
|
+
`scheduled-release.yml` re-renders the PDF in the same step as the sync —
|
|
128
|
+
the same step, because an HTML and a PDF that disagree are worse than a
|
|
129
|
+
document uniformly out of date. `--require-chrome` makes a missing browser
|
|
130
|
+
fail the release rather than silently commit a stale PDF, and every render
|
|
131
|
+
must produce exactly one sheet per `.page` div: a section a few pixels too
|
|
132
|
+
tall spills onto a second sheet with no warning from Chrome, contradicting
|
|
133
|
+
the "Page N of M" footers the document prints about itself.
|
|
134
|
+
|
|
10
135
|
## [0.9.0] - 2026-09-18
|
|
11
136
|
|
|
12
137
|
Automated weekly release — everything merged since `v0.8.1`.
|
package/README.md
CHANGED
|
@@ -80,8 +80,9 @@ risk. TestGuard binds every fault to a **stated claim**, so the output is not
|
|
|
80
80
|
a score but a finding: *"Your project says a missing scope fails closed.
|
|
81
81
|
Nothing checks that."* That is the difference between a metric and an audit.
|
|
82
82
|
|
|
83
|
-
📄 **[Read the
|
|
84
|
-
— the idea on page one,
|
|
83
|
+
📄 **[Read the six-page technical brief (PDF)](docs/testguard-explained.pdf)**
|
|
84
|
+
— the idea on page one, then the field evidence, the anatomy of a run, the
|
|
85
|
+
verdicts, the loop and its calibration, and the prior art.
|
|
85
86
|
|
|
86
87
|
## Why
|
|
87
88
|
|
|
@@ -126,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
126
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
127
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
128
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
129
|
-
| GitHub Action | `uses: raccioly/testguard@v0.9.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.9.2` — see [`action.yml`](./action.yml) |
|
|
130
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
131
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.9.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.9.2/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
132
133
|
|
|
133
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
134
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -140,6 +141,7 @@ less than that many days ago (`ENOVERSIONS`); install that one with
|
|
|
140
141
|
npx testguard-cli init # install the agent layer at the git root
|
|
141
142
|
npx testguard-cli status --json # where the project is, and the ONE next action
|
|
142
143
|
npx testguard-cli claims # what does this project claim, and is each claim probeable?
|
|
144
|
+
npx testguard-cli claims --check-anchors # fail fast if an exact anchor moved or a replacement no longer parses
|
|
143
145
|
npx testguard-cli scaffold src/auth.ts # propose faults for a file, as a draft to keep or drop
|
|
144
146
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
145
147
|
npx testguard-cli admit test/auth.test.ts --claim AUTH-ADMIN # does this test satisfy the two-gate rule?
|
|
@@ -150,8 +152,9 @@ npx testguard-cli replay --since v1.0..HEAD # would this suite have caught the b
|
|
|
150
152
|
npx testguard-cli mcp # serve the read-only loop over MCP, on stdio
|
|
151
153
|
```
|
|
152
154
|
|
|
153
|
-
Exit codes are the contract: `0` nothing new to prove, `1` unproven claims
|
|
154
|
-
unclaimed changes, `2` a precondition failed and
|
|
155
|
+
Exit codes are the contract: `0` nothing new to prove, `1` unproven claims,
|
|
156
|
+
unclaimed changes or invalid fault anchors, `2` a precondition failed and
|
|
157
|
+
nothing was probed, `3` usage.
|
|
155
158
|
Every command accepts `--json`.
|
|
156
159
|
|
|
157
160
|
## How it works
|
|
@@ -176,6 +179,10 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
176
179
|
claim and every fault records who produced it. `testguard claims`
|
|
177
180
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
178
181
|
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
182
|
+
`claims --check-anchors` additionally locates every exact anchor and parses
|
|
183
|
+
each in-memory JavaScript or Python replacement. It runs no tests, creates
|
|
184
|
+
no worktree and changes no file, so a moved anchor is reported before an
|
|
185
|
+
expensive probe. It reports and stops; it never guesses a new anchor.
|
|
179
186
|
2. **Probe** confirms the defenders are green N times unmodified, applies
|
|
180
187
|
each fault in a scratch git worktree (your tree is never touched), runs
|
|
181
188
|
the defenders N times, re-runs survivors against the whole suite with
|
|
@@ -444,6 +451,10 @@ is reported as `EXPIRED`.
|
|
|
444
451
|
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
445
452
|
**before** any evidence state and makes the claim the next action; the brief
|
|
446
453
|
lists the unclaimed files first. The claim is written before more code.
|
|
454
|
+
After claim coverage, `status` reports `invalid-anchors` before any evidence
|
|
455
|
+
state when an exact fault anchor is missing or ambiguous, and points at the
|
|
456
|
+
fault definition to repair. Repair preserves the fault's meaning and requires
|
|
457
|
+
a re-probe; there is deliberately no `--fix`.
|
|
447
458
|
|
|
448
459
|
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
449
460
|
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
@@ -458,12 +469,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
458
469
|
# GitHub Actions
|
|
459
470
|
- uses: actions/checkout@v4
|
|
460
471
|
with: { fetch-depth: 0 }
|
|
461
|
-
- uses: raccioly/testguard@v0.9.
|
|
472
|
+
- uses: raccioly/testguard@v0.9.2
|
|
462
473
|
with: { command: gate }
|
|
463
474
|
|
|
464
475
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
465
476
|
include:
|
|
466
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.9.
|
|
477
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.9.2/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
467
478
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
468
479
|
```
|
|
469
480
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -395,6 +395,26 @@ changed); `1` at least one uncovered file (or strict mode over an
|
|
|
395
395
|
all-excluded change); `2` the change cannot be evaluated (unresolvable
|
|
396
396
|
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
397
397
|
|
|
398
|
+
## Anchor preflight
|
|
399
|
+
|
|
400
|
+
`claims --check-anchors` answers what is knowable before a runner starts. For
|
|
401
|
+
every fault-injection fault it reads the target, applies the same exact
|
|
402
|
+
`locate()` arithmetic as a probe, and reports `ok`, `anchor-missing` or
|
|
403
|
+
`anchor-ambiguous` with the observed and expected hit counts. When a language
|
|
404
|
+
has a cheap parser, it also parses the in-memory replacement (`node --check`
|
|
405
|
+
semantics for JavaScript modules, `compile()` for Python). Unsupported
|
|
406
|
+
languages are anchor-checked and otherwise skipped without inference.
|
|
407
|
+
|
|
408
|
+
One non-`ok` anchor or non-parsing supported replacement exits `1`. The check
|
|
409
|
+
runs no tests, creates no worktree, reads no git state and writes no source.
|
|
410
|
+
It is not evidence that a defender detects the fault; the self-probe still
|
|
411
|
+
runs after it. There is no automatic repair or fuzzy relocation: choosing a
|
|
412
|
+
new anchor is a semantic edit to the claim and must be reviewed and re-probed.
|
|
413
|
+
|
|
414
|
+
`invalid-anchors` precedes every evidence state in the status document (after
|
|
415
|
+
`unclaimed-changes`, when a change reference is known). Its next action is
|
|
416
|
+
`repair-fault`, naming the first fault and `testguard claims --check-anchors`.
|
|
417
|
+
|
|
398
418
|
## Severity floor
|
|
399
419
|
|
|
400
420
|
`--severity <level>` gates only findings whose claim severity is at or above
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -233,12 +233,15 @@ const semantic = {
|
|
|
233
233
|
|
|
234
234
|
status(doc) {
|
|
235
235
|
const errors = [];
|
|
236
|
-
if (['write-test', 'review-fault-change'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
236
|
+
if (['write-test', 'review-fault-change', 'repair-fault'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
237
237
|
if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
|
|
238
238
|
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
239
239
|
if (doc.state === 'clean' && (doc.counts.new ?? 0) > 0) errors.push({ path: '/state', message: 'clean with new findings' });
|
|
240
240
|
if (doc.state === 'unclaimed-changes' && !(doc.changes && doc.changes.uncovered.length > 0)) errors.push({ path: '/changes/uncovered', message: 'unclaimed-changes requires at least one uncovered changed file' });
|
|
241
241
|
if (doc.state === 'unclaimed-changes' && doc.next.action !== 'claim') errors.push({ path: '/next/action', message: 'unclaimed-changes requires next.action = claim' });
|
|
242
|
+
if (doc.state === 'invalid-anchors' && !(doc.invalidFaults?.length > 0)) errors.push({ path: '/invalidFaults', message: 'invalid-anchors requires at least one invalid fault' });
|
|
243
|
+
if (doc.state === 'invalid-anchors' && doc.next.action !== 'repair-fault') errors.push({ path: '/next/action', message: 'invalid-anchors requires next.action = repair-fault' });
|
|
244
|
+
if (doc.invalidFaults?.length > 0 && doc.state !== 'invalid-anchors') errors.push({ path: '/state', message: 'invalid fault anchors are hidden behind a later state; invalid-anchors precedes every evidence state' });
|
|
242
245
|
if (doc.evidenceSource && !doc.evidenceHead) errors.push({ path: '/evidenceHead', message: 'evidence was read, so the commit it describes must be recorded' });
|
|
243
246
|
if (doc.changes && doc.changes.uncovered.length > 0 && !['no-claims', 'unclaimed-changes'].includes(doc.state)) errors.push({ path: '/state', message: 'uncovered changed files are hidden behind a later state; unclaimed-changes precedes every evidence state' });
|
|
244
247
|
return errors;
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"type": "object",
|
|
40
40
|
"required": ["action", "command", "why"],
|
|
41
41
|
"properties": {
|
|
42
|
-
"action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "review-fault-change", "none"] },
|
|
42
|
+
"action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "repair-fault", "review-fault-change", "none"] },
|
|
43
43
|
"command": { "type": "string", "minLength": 1, "maxLength": 1024 },
|
|
44
44
|
"why": { "type": "string", "minLength": 1, "maxLength": 2000 }
|
|
45
45
|
},
|
|
@@ -26,10 +26,11 @@
|
|
|
26
26
|
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
27
27
|
},
|
|
28
28
|
"state": {
|
|
29
|
-
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
29
|
+
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
30
30
|
"enum": [
|
|
31
31
|
"no-claims",
|
|
32
32
|
"unclaimed-changes",
|
|
33
|
+
"invalid-anchors",
|
|
33
34
|
"unprobed",
|
|
34
35
|
"evidence-stale",
|
|
35
36
|
"provisional-only",
|
|
@@ -54,6 +55,7 @@
|
|
|
54
55
|
"write-test",
|
|
55
56
|
"baseline",
|
|
56
57
|
"review-fault-change",
|
|
58
|
+
"repair-fault",
|
|
57
59
|
"none"
|
|
58
60
|
]
|
|
59
61
|
},
|
|
@@ -188,6 +190,23 @@
|
|
|
188
190
|
"additionalProperties": false
|
|
189
191
|
}
|
|
190
192
|
},
|
|
193
|
+
"invalidFaults": {
|
|
194
|
+
"description": "Faults whose exact text anchor does not locate in the current target. These precede every evidence state: a probe cannot learn anything from a fault it cannot apply.",
|
|
195
|
+
"type": "array",
|
|
196
|
+
"items": {
|
|
197
|
+
"type": "object",
|
|
198
|
+
"required": ["claimId", "subjectId", "file", "reason", "hits", "expected"],
|
|
199
|
+
"properties": {
|
|
200
|
+
"claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
201
|
+
"subjectId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
202
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
203
|
+
"reason": { "enum": ["anchor-missing", "anchor-ambiguous"] },
|
|
204
|
+
"hits": { "type": "integer", "minimum": 0 },
|
|
205
|
+
"expected": { "type": "integer", "minimum": 1 }
|
|
206
|
+
},
|
|
207
|
+
"additionalProperties": false
|
|
208
|
+
}
|
|
209
|
+
},
|
|
191
210
|
"findings": {
|
|
192
211
|
"description": "Unproven findings from the current evidence, ranked, new first, capped.",
|
|
193
212
|
"type": "array",
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { dirname, extname, join, resolve } from 'node:path';
|
|
4
|
+
import { performance } from 'node:perf_hooks';
|
|
5
|
+
import { locate, mutate } from '../probe/inject.mjs';
|
|
6
|
+
import { resolveInterpreter } from '../probe/runners/python.mjs';
|
|
7
|
+
|
|
8
|
+
const JS_EXTENSIONS = new Set(['.js', '.mjs', '.cjs']);
|
|
9
|
+
|
|
10
|
+
const NODE_SYNTAX_CHECK = `
|
|
11
|
+
const vm = require('node:vm');
|
|
12
|
+
let input = '';
|
|
13
|
+
process.stdin.setEncoding('utf8');
|
|
14
|
+
process.stdin.on('data', (chunk) => { input += chunk; });
|
|
15
|
+
process.stdin.on('end', () => {
|
|
16
|
+
const results = JSON.parse(input).map(({ file, source, type }) => {
|
|
17
|
+
try {
|
|
18
|
+
if (type === 'module') new vm.SourceTextModule(source, { identifier: file });
|
|
19
|
+
else new vm.Script(source, { filename: file });
|
|
20
|
+
return { status: 'ok' };
|
|
21
|
+
} catch (error) {
|
|
22
|
+
return { status: 'invalid', message: error.name + ': ' + error.message };
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
process.stdout.write(JSON.stringify(results));
|
|
26
|
+
});
|
|
27
|
+
`;
|
|
28
|
+
|
|
29
|
+
const PYTHON_SYNTAX_CHECK = `
|
|
30
|
+
import json, sys
|
|
31
|
+
results = []
|
|
32
|
+
for item in json.load(sys.stdin):
|
|
33
|
+
try:
|
|
34
|
+
compile(item["source"], item["file"], "exec")
|
|
35
|
+
results.append({"status": "ok"})
|
|
36
|
+
except (SyntaxError, IndentationError, TabError) as error:
|
|
37
|
+
results.append({"status": "invalid", "message": type(error).__name__ + ": " + str(error)})
|
|
38
|
+
json.dump(results, sys.stdout)
|
|
39
|
+
`;
|
|
40
|
+
|
|
41
|
+
const isInjection = (fault) => (fault.method ?? 'fault-injection') === 'fault-injection';
|
|
42
|
+
const keyOf = (claimId, faultId) => `${claimId}\0${faultId}`;
|
|
43
|
+
|
|
44
|
+
/** Read each target once and locate every injection fault without touching disk. */
|
|
45
|
+
function inspectLocations(projectDir, claims) {
|
|
46
|
+
const sources = new Map();
|
|
47
|
+
const work = [];
|
|
48
|
+
for (const claim of claims.claims) {
|
|
49
|
+
for (const fault of claim.faults) {
|
|
50
|
+
if (!isInjection(fault)) {
|
|
51
|
+
work.push({ claim, fault, result: { claimId: claim.id, faultId: fault.id, file: fault.file, status: 'ok', check: 'not-applicable' } });
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const path = resolve(projectDir, fault.file);
|
|
55
|
+
let source;
|
|
56
|
+
if (sources.has(path)) source = sources.get(path);
|
|
57
|
+
else {
|
|
58
|
+
source = existsSync(path) ? readFileSync(path, 'utf8') : undefined;
|
|
59
|
+
sources.set(path, source);
|
|
60
|
+
}
|
|
61
|
+
const anchor = source === undefined
|
|
62
|
+
? { status: 'anchor-missing', hits: 0, expected: fault.expectHits ?? 1 }
|
|
63
|
+
: locate(source, fault);
|
|
64
|
+
work.push({
|
|
65
|
+
claim,
|
|
66
|
+
fault,
|
|
67
|
+
source,
|
|
68
|
+
result: { claimId: claim.id, faultId: fault.id, file: fault.file, status: anchor.status, hits: anchor.hits, expected: anchor.expected },
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return work;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const summarize = (results, durationMs) => ({
|
|
76
|
+
checked: results.length,
|
|
77
|
+
ok: results.filter((r) => r.status === 'ok').length,
|
|
78
|
+
invalid: results.filter((r) => r.status !== 'ok').length,
|
|
79
|
+
durationMs: Math.round(durationMs),
|
|
80
|
+
results,
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
/** Exact anchor checks only. Synchronous so status and MCP stay synchronous. */
|
|
84
|
+
export function checkAnchorLocations(projectDir, claims) {
|
|
85
|
+
const started = performance.now();
|
|
86
|
+
return summarize(inspectLocations(projectDir, claims).map((x) => x.result), performance.now() - started);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function firstError(stderr) {
|
|
90
|
+
const lines = stderr.split('\n').map((line) => line.trim()).filter(Boolean);
|
|
91
|
+
return lines.find((line) => /(?:Syntax|Indentation|Tab)Error:/.test(line))
|
|
92
|
+
?? lines.find((line) => /error/i.test(line))
|
|
93
|
+
?? lines.at(-1)
|
|
94
|
+
?? 'replacement does not compile';
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function runBatchCheck(command, args, items, timeoutMs = 5_000) {
|
|
98
|
+
if (!items.length) return Promise.resolve([]);
|
|
99
|
+
return new Promise((resolveRun, reject) => {
|
|
100
|
+
let child;
|
|
101
|
+
try {
|
|
102
|
+
child = spawn(command, args, { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, CI: '1' } });
|
|
103
|
+
} catch (error) {
|
|
104
|
+
reject(error);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
let stdout = '';
|
|
108
|
+
let stderr = '';
|
|
109
|
+
let timedOut = false;
|
|
110
|
+
child.stdout.on('data', (chunk) => {
|
|
111
|
+
if (stdout.length < 16_777_216) stdout += chunk;
|
|
112
|
+
});
|
|
113
|
+
child.stderr.on('data', (chunk) => {
|
|
114
|
+
if (stderr.length < 16_384) stderr += chunk;
|
|
115
|
+
});
|
|
116
|
+
child.on('error', reject);
|
|
117
|
+
child.stdin.on('error', () => {});
|
|
118
|
+
child.stdin.end(JSON.stringify(items));
|
|
119
|
+
const timer = setTimeout(() => {
|
|
120
|
+
timedOut = true;
|
|
121
|
+
child.kill('SIGKILL');
|
|
122
|
+
}, timeoutMs);
|
|
123
|
+
child.on('close', (code) => {
|
|
124
|
+
clearTimeout(timer);
|
|
125
|
+
if (timedOut) reject(new Error(`${command} syntax check timed out after ${timeoutMs}ms`));
|
|
126
|
+
else if (code !== 0) reject(new Error(`${command} syntax check failed: ${firstError(stderr)}`));
|
|
127
|
+
else {
|
|
128
|
+
try {
|
|
129
|
+
const results = JSON.parse(stdout);
|
|
130
|
+
if (!Array.isArray(results) || results.length !== items.length) throw new Error('unexpected result count');
|
|
131
|
+
resolveRun(results);
|
|
132
|
+
} catch (error) {
|
|
133
|
+
reject(new Error(`${command} syntax check returned invalid output: ${error.message}`));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function packageType(projectDir, file, cache) {
|
|
141
|
+
if (extname(file) === '.mjs') return 'module';
|
|
142
|
+
if (extname(file) === '.cjs') return 'commonjs';
|
|
143
|
+
let dir = dirname(resolve(projectDir, file));
|
|
144
|
+
const root = resolve(projectDir);
|
|
145
|
+
while (dir.startsWith(root)) {
|
|
146
|
+
if (cache.has(dir)) return cache.get(dir);
|
|
147
|
+
const pkg = join(dir, 'package.json');
|
|
148
|
+
if (existsSync(pkg)) {
|
|
149
|
+
const type = JSON.parse(readFileSync(pkg, 'utf8')).type === 'module' ? 'module' : 'commonjs';
|
|
150
|
+
cache.set(dir, type);
|
|
151
|
+
return type;
|
|
152
|
+
}
|
|
153
|
+
if (dir === root) break;
|
|
154
|
+
dir = dirname(dir);
|
|
155
|
+
}
|
|
156
|
+
return 'commonjs';
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Anchor every fault, then ask the language parser whether the in-memory
|
|
161
|
+
* replacement still compiles. Unsupported languages (and Python when no
|
|
162
|
+
* interpreter resolves) are intentionally left anchor-only.
|
|
163
|
+
*/
|
|
164
|
+
export async function checkAnchors(projectDir, claims, { python } = {}) {
|
|
165
|
+
const started = performance.now();
|
|
166
|
+
const work = inspectLocations(projectDir, claims);
|
|
167
|
+
const candidates = work.filter((x) => x.result.status === 'ok' && x.source !== undefined && isInjection(x.fault));
|
|
168
|
+
const hasPython = candidates.some((x) => extname(x.fault.file) === '.py');
|
|
169
|
+
const interpreter = hasPython ? await resolveInterpreter({ projectDir, python, budgetMs: 5_000 }) : undefined;
|
|
170
|
+
const packageTypes = new Map();
|
|
171
|
+
const javascript = [];
|
|
172
|
+
const pythonSources = [];
|
|
173
|
+
for (const candidate of candidates) {
|
|
174
|
+
const { fault, source } = candidate;
|
|
175
|
+
const extension = extname(fault.file);
|
|
176
|
+
const changed = mutate(source, fault);
|
|
177
|
+
if (JS_EXTENSIONS.has(extension)) {
|
|
178
|
+
javascript.push({ candidate, file: fault.file, source: changed, type: packageType(projectDir, fault.file, packageTypes) });
|
|
179
|
+
} else if (extension === '.py' && interpreter && !interpreter.error) {
|
|
180
|
+
pythonSources.push({ candidate, file: fault.file, source: changed });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// One parser process per language keeps this a cheap preflight even when a
|
|
185
|
+
// claims file has hundreds of faults. The Node VM constructors use the same
|
|
186
|
+
// V8 grammar as `node --check` and compile without linking or executing code.
|
|
187
|
+
const [javascriptResults, pythonResults] = await Promise.all([
|
|
188
|
+
runBatchCheck(process.execPath, ['--no-warnings', '--experimental-vm-modules', '-e', NODE_SYNTAX_CHECK], javascript),
|
|
189
|
+
interpreter && !interpreter.error
|
|
190
|
+
? runBatchCheck(interpreter.path, ['-c', PYTHON_SYNTAX_CHECK], pythonSources)
|
|
191
|
+
: [],
|
|
192
|
+
]);
|
|
193
|
+
|
|
194
|
+
const parsedByFault = new Map();
|
|
195
|
+
javascript.forEach(({ candidate }, index) => parsedByFault.set(keyOf(candidate.claim.id, candidate.fault.id), { language: 'javascript', ...javascriptResults[index] }));
|
|
196
|
+
pythonSources.forEach(({ candidate }, index) => parsedByFault.set(keyOf(candidate.claim.id, candidate.fault.id), { language: 'python', ...pythonResults[index] }));
|
|
197
|
+
|
|
198
|
+
const results = work.map(({ claim, fault, result }) => {
|
|
199
|
+
const parsed = parsedByFault.get(keyOf(claim.id, fault.id));
|
|
200
|
+
if (!parsed) return result;
|
|
201
|
+
if (parsed.status === 'invalid') return { ...result, status: 'fault-invalid', syntax: parsed };
|
|
202
|
+
return { ...result, syntax: parsed };
|
|
203
|
+
});
|
|
204
|
+
return summarize(results, performance.now() - started);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export function renderAnchorChecks(report) {
|
|
208
|
+
const lines = report.results.map((result) => {
|
|
209
|
+
const label = result.status.toUpperCase().padEnd(16);
|
|
210
|
+
const hits = result.hits === undefined ? 'no text anchor for this method' : `${result.hits} hit${result.hits === 1 ? '' : 's'}, expected ${result.expected}`;
|
|
211
|
+
const syntax = result.syntax?.status === 'ok'
|
|
212
|
+
? `; ${result.syntax.language} syntax ok`
|
|
213
|
+
: result.syntax?.message ? `; ${result.syntax.language}: ${result.syntax.message}` : '';
|
|
214
|
+
return `${label} ${result.claimId}/${result.faultId} ${result.file} — ${hits}${syntax}`;
|
|
215
|
+
});
|
|
216
|
+
lines.push(`anchor preflight: ${report.checked} faults checked, ${report.ok} ok, ${report.invalid} invalid in ${report.durationMs}ms`);
|
|
217
|
+
return lines.join('\n');
|
|
218
|
+
}
|
package/src/cli.mjs
CHANGED
|
@@ -26,7 +26,7 @@ export const USAGE = `testguard ${VERSION} — proves a test suite defends the c
|
|
|
26
26
|
|
|
27
27
|
testguard status [dir] where the project is and the ONE next action; --json is the machine entry point
|
|
28
28
|
testguard init [dir] install the agent layer at the git root (skill, session-start hook, AGENTS.md section) and the .gitignore lines in [dir]
|
|
29
|
-
testguard claims [dir] list
|
|
29
|
+
testguard claims [dir] list claims and drift; --check-anchors also verifies every exact anchor and replacement syntax
|
|
30
30
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
31
31
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
32
32
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
@@ -95,7 +95,9 @@ gate --changed <ref> measure the change since merge-base(ref, HEAD); aut
|
|
|
95
95
|
--strict a non-empty change that evaluates nothing is a failure, not a note
|
|
96
96
|
--ignore <path> ignore file (default: <dir>/testguard.ignore.json; kind=path entries excuse files, with a reason)
|
|
97
97
|
exit 0 every changed source file is claimed or excused · 1 unclaimed file · 2 cannot evaluate · 3 no reference
|
|
98
|
-
|
|
98
|
+
claims --check-anchors locate every fault without running tests; JS/MJS and Python replacements are syntax-checked in memory
|
|
99
|
+
--json includes anchorChecks; exit 1 when an anchor is missing/ambiguous or a supported replacement does not compile
|
|
100
|
+
status --json (exit 0 clean · 1 unproven/stale/unclaimed/invalid anchors · 2 nothing to probe yet)
|
|
99
101
|
--evidence <path> read this evidence instead of .testguard/evidence.json (e.g. CI's, fetched as an artifact); staleness is still computed from the recorded input hashes, and both commits are named
|
|
100
102
|
--changed <ref> also compute claim coverage of the change; unclaimed-changes then precedes every evidence state
|
|
101
103
|
init --force (replace an existing skill file) --here (keep the agent layer in [dir] instead of the git root) --json
|
|
@@ -163,6 +165,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
163
165
|
text: { type: 'boolean', default: false },
|
|
164
166
|
markdown: { type: 'boolean', default: false },
|
|
165
167
|
cost: { type: 'boolean', default: false },
|
|
168
|
+
'check-anchors': { type: 'boolean', default: false },
|
|
166
169
|
progress: { type: 'string' },
|
|
167
170
|
help: { type: 'boolean', short: 'h', default: false },
|
|
168
171
|
version: { type: 'boolean', short: 'v', default: false },
|
package/src/commands/claims.mjs
CHANGED
|
@@ -9,12 +9,16 @@ import { evidencePath } from './probe.mjs';
|
|
|
9
9
|
import { costReport, renderCost } from '../probe/cost.mjs';
|
|
10
10
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
11
11
|
import { existsSync } from 'node:fs';
|
|
12
|
+
import { checkAnchors, renderAnchorChecks } from '../claims/anchors.mjs';
|
|
12
13
|
|
|
13
14
|
export async function claimsCommand({ projectDir, values, version }, io) {
|
|
14
15
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
15
16
|
const claims = loadClaims(path);
|
|
16
17
|
const annotations = scanAnnotations(projectDir);
|
|
17
18
|
const drift = reconcile(claims, annotations);
|
|
19
|
+
const anchorChecks = values['check-anchors']
|
|
20
|
+
? await checkAnchors(projectDir, claims, { python: values.python ? resolve(values.python) : undefined })
|
|
21
|
+
: undefined;
|
|
18
22
|
|
|
19
23
|
// A claim that disappeared is invisible to every other command; compare
|
|
20
24
|
// identities against a reference when one is given.
|
|
@@ -31,7 +35,7 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
31
35
|
}
|
|
32
36
|
|
|
33
37
|
if (values.json) {
|
|
34
|
-
io.out(JSON.stringify({ path, claims, annotations, drift, ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
38
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
35
39
|
} else {
|
|
36
40
|
const annotated = new Set(drift.annotated);
|
|
37
41
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
@@ -74,6 +78,10 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
74
78
|
io.out('');
|
|
75
79
|
io.out(renderCost(cost));
|
|
76
80
|
}
|
|
81
|
+
if (anchorChecks) {
|
|
82
|
+
io.out('');
|
|
83
|
+
io.out(renderAnchorChecks(anchorChecks));
|
|
84
|
+
}
|
|
77
85
|
}
|
|
78
|
-
return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 ? 1 : 0;
|
|
86
|
+
return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 || (anchorChecks?.invalid ?? 0) > 0 ? 1 : 0;
|
|
79
87
|
}
|
|
@@ -22,6 +22,7 @@ which files exist. `state` is one of:
|
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
24
|
| `unclaimed-changes` | files you changed carry no claim (only when a reference is known: `--changed <ref>`, or CI's base branch) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
25
|
+
| `invalid-anchors` | an exact fault anchor moved or became ambiguous | run `testguard claims --check-anchors`; repair `next.target` in the claims file without changing what the fault means, then re-probe. Never guess or auto-fix an anchor. |
|
|
25
26
|
| `unprobed` | claims never probed | `testguard probe` |
|
|
26
27
|
| `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
|
|
27
28
|
| `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
|
|
@@ -45,7 +46,10 @@ code by construction. So, for every source file you create or change:
|
|
|
45
46
|
thin wrapper whose logic is claimed elsewhere): a `path` entry in
|
|
46
47
|
`testguard.ignore.json` with a reason and, where possible, an `expires`.
|
|
47
48
|
The gate prints every entry it relied on; a reviewer reads them.
|
|
48
|
-
4.
|
|
49
|
+
4. After editing guarded code, run `testguard claims --check-anchors`. Repair
|
|
50
|
+
any named fault definition without weakening its meaning; the check never
|
|
51
|
+
relocates an anchor for you.
|
|
52
|
+
5. Then `testguard admit <test-file> --claim <ID>` for the new claim's test (or `testguard probe --claim <ID> --include-dirty` for the full report).
|
|
49
53
|
|
|
50
54
|
## Verdict → action
|
|
51
55
|
|
package/src/probe/cost.mjs
CHANGED
|
@@ -163,18 +163,41 @@ export function renderCost(report, { limit = 10 } = {}) {
|
|
|
163
163
|
*
|
|
164
164
|
* Pure. Reads a report, returns a decision; the caller owns exit codes and I/O.
|
|
165
165
|
*/
|
|
166
|
-
export function checkCostBudget(report, { budgetSeconds, previousMs, worst = 5 } = {}) {
|
|
166
|
+
export function checkCostBudget(report, { budgetSeconds, perClaimSeconds, previousMs, worst = 5 } = {}) {
|
|
167
167
|
if (typeof budgetSeconds !== 'number' || !Number.isFinite(budgetSeconds) || budgetSeconds <= 0) {
|
|
168
168
|
throw new TypeError('checkCostBudget: budgetSeconds must be a positive number');
|
|
169
169
|
}
|
|
170
170
|
const budgetMs = budgetSeconds * 1000;
|
|
171
171
|
const totalMs = report.totalMs ?? 0;
|
|
172
|
+
|
|
173
|
+
// ── Why a SECOND ceiling, on the claim. ──
|
|
174
|
+
// The total alone is a treadmill: claims should grow forever, so it can only
|
|
175
|
+
// ever be raised, and a number that always goes up stops being read. A
|
|
176
|
+
// per-FAULT average is worse — the regression that prompted this budget
|
|
177
|
+
// raised the fault count 59 -> 91 while per-fault cost barely moved, so an
|
|
178
|
+
// average would have reported everything fine.
|
|
179
|
+
//
|
|
180
|
+
// The measured pathology is per-CLAIM, and it is a claim naming an expensive
|
|
181
|
+
// SHARED defender: a claim pays for every test in the file it names, so
|
|
182
|
+
// TG-ADMIT-NEEDS-ALL-KILLED costs 67 s for ONE fault. A new calibration
|
|
183
|
+
// claim landed at 51 s for two faults purely by being written into
|
|
184
|
+
// replay.test.mjs instead of calibration.test.mjs; re-pointing it took it to
|
|
185
|
+
// 4.9 s. The total absorbed that silently. This does not.
|
|
186
|
+
//
|
|
187
|
+
// So the rule is "no new claim may be worse than the worst we already have",
|
|
188
|
+
// which stays meaningful as claims are added and tightens as they get cheaper.
|
|
189
|
+
const perClaimMs = typeof perClaimSeconds === 'number' && perClaimSeconds > 0 ? perClaimSeconds * 1000 : undefined;
|
|
190
|
+
const overPerClaim = perClaimMs === undefined ? [] : (report.claims ?? [])
|
|
191
|
+
.filter((c) => c.ms > perClaimMs)
|
|
192
|
+
.map((c) => ({ id: c.claimId, ms: c.ms, runs: c.runs }));
|
|
193
|
+
|
|
172
194
|
return {
|
|
173
|
-
ok: totalMs <= budgetMs,
|
|
195
|
+
ok: totalMs <= budgetMs && overPerClaim.length === 0,
|
|
174
196
|
totalMs,
|
|
175
197
|
budgetMs,
|
|
176
198
|
overByMs: Math.max(0, totalMs - budgetMs),
|
|
177
199
|
headroomMs: Math.max(0, budgetMs - totalMs),
|
|
200
|
+
...(perClaimMs === undefined ? {} : { perClaimMs, overPerClaim }),
|
|
178
201
|
...(typeof previousMs === 'number' ? { previousMs, deltaMs: totalMs - previousMs } : {}),
|
|
179
202
|
// Named so a failure opens with which claims to look at, not just a number.
|
|
180
203
|
// claimCosts emits `claimId`; reading `id` here printed "undefined" for every
|
|
@@ -189,10 +212,24 @@ export function renderCostBudget(d) {
|
|
|
189
212
|
const delta = typeof d.deltaMs === 'number'
|
|
190
213
|
? ` (${d.deltaMs >= 0 ? '+' : ''}${secs(Math.abs(d.deltaMs))} against the previous run's ${secs(d.previousMs)})`
|
|
191
214
|
: '';
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
215
|
+
const overTotal = d.totalMs > d.budgetMs;
|
|
216
|
+
out.push(overTotal
|
|
217
|
+
? `COST BUDGET EXCEEDED: ${secs(d.totalMs)} against a ${secs(d.budgetMs)} budget, over by ${secs(d.overByMs)}${delta}.`
|
|
218
|
+
: `cost ${secs(d.totalMs)} of a ${secs(d.budgetMs)} budget — ${secs(d.headroomMs)} to spare${delta}.`);
|
|
219
|
+
|
|
220
|
+
// A per-claim breach is its own finding and names its own remedy: the claim
|
|
221
|
+
// is not too big, it is pointed at too expensive a defender file.
|
|
222
|
+
if (d.overPerClaim?.length) {
|
|
223
|
+
out.push('');
|
|
224
|
+
out.push(`PER-CLAIM BUDGET EXCEEDED: ${d.overPerClaim.length} claim${d.overPerClaim.length === 1 ? '' : 's'} over the ${secs(d.perClaimMs)} per-claim ceiling.`);
|
|
225
|
+
for (const c of d.overPerClaim) out.push(` ${secs(c.ms).padStart(7)} ${c.id} (${c.runs} runs)`);
|
|
226
|
+
out.push('');
|
|
227
|
+
out.push('A claim pays for every test in the file it names. Point it at a test that');
|
|
228
|
+
out.push('does not need the expensive setup, re-probe to confirm the fault still dies,');
|
|
229
|
+
out.push('and only then raise the ceiling if it is still over.');
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (overTotal && d.worst.length) {
|
|
196
233
|
out.push('');
|
|
197
234
|
out.push('most expensive claims');
|
|
198
235
|
for (const c of d.worst) out.push(` ${secs(c.ms).padStart(7)} ${c.id} (${c.runs} runs)`);
|
package/src/status/status.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { discoverDefenders } from '../probe/discover.mjs';
|
|
|
9
9
|
import { sortForReport } from '../render.mjs';
|
|
10
10
|
import { computeChangedGate, defaultIgnorePath } from '../gate/changed.mjs';
|
|
11
11
|
import { headSha, isAncestor } from '../git.mjs';
|
|
12
|
+
import { checkAnchorLocations } from '../claims/anchors.mjs';
|
|
12
13
|
|
|
13
14
|
export const faultContentHash = (fault) => sha256(`${fault.find}\n${fault.replace}`);
|
|
14
15
|
|
|
@@ -41,6 +42,7 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
41
42
|
provisional: false,
|
|
42
43
|
counts: { claims: 0, faults: 0 },
|
|
43
44
|
stale: [],
|
|
45
|
+
invalidFaults: [],
|
|
44
46
|
changedFaults: [],
|
|
45
47
|
findings: [],
|
|
46
48
|
notes: [],
|
|
@@ -87,6 +89,22 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
87
89
|
return doc;
|
|
88
90
|
}
|
|
89
91
|
|
|
92
|
+
const anchors = checkAnchorLocations(projectDir, claims);
|
|
93
|
+
doc.invalidFaults = anchors.results
|
|
94
|
+
.filter((result) => result.status !== 'ok')
|
|
95
|
+
.map(({ claimId, faultId, file, status: reason, hits, expected }) => ({ claimId, subjectId: faultId, file, reason, hits, expected }));
|
|
96
|
+
if (doc.invalidFaults.length) {
|
|
97
|
+
const first = doc.invalidFaults[0];
|
|
98
|
+
doc.state = 'invalid-anchors';
|
|
99
|
+
doc.next = {
|
|
100
|
+
action: 'repair-fault',
|
|
101
|
+
command: 'testguard claims --check-anchors',
|
|
102
|
+
why: `${doc.invalidFaults.length} fault anchor${doc.invalidFaults.length === 1 ? '' : 's'} no longer locate exactly. Repair ${first.claimId}/${first.subjectId} in testguard.claims.json without changing what the fault means, then re-probe.`,
|
|
103
|
+
target: { claimId: first.claimId, subjectId: first.subjectId, file: first.file },
|
|
104
|
+
};
|
|
105
|
+
return doc;
|
|
106
|
+
}
|
|
107
|
+
|
|
90
108
|
const evidenceFile = evidenceOverride ?? paths.evidence;
|
|
91
109
|
const hasEvidence = existsSync(evidenceFile);
|
|
92
110
|
const hasProvisional = existsSync(paths.provisional);
|
|
@@ -236,6 +254,7 @@ export function renderStatus(doc) {
|
|
|
236
254
|
for (const e of c.expired) lines.push(`EXPIRED ignore "${e.pattern}" no longer excuses ${e.files.join(', ')}`);
|
|
237
255
|
}
|
|
238
256
|
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
257
|
+
for (const f of doc.invalidFaults ?? []) lines.push(`INVALID ${f.claimId}/${f.subjectId} ${f.reason}: ${f.hits} hits, expected ${f.expected} (${f.file})`);
|
|
239
258
|
for (const s of doc.stale.slice(0, 8)) lines.push(`stale ${s}`);
|
|
240
259
|
for (const n of doc.notes ?? []) lines.push(`note ${n}`);
|
|
241
260
|
lines.push(`next: [${doc.next.action}] ${doc.next.command}`);
|