testguard-cli 0.14.2 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -1
- package/README.md +30 -4
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +99 -8
- package/spec/README.md +7 -0
- package/spec/lib/validate.mjs +26 -1
- package/spec/schemas/common.schema.json +10 -0
- package/spec/schemas/evidence.schema.json +14 -0
- package/spec/schemas/gate.schema.json +1 -0
- package/spec/schemas/status.schema.json +2 -1
- package/src/cli.mjs +34 -4
- package/src/command-budget.mjs +52 -0
- package/src/commands/probe.mjs +22 -0
- package/src/commands/replay.mjs +14 -1
- package/src/commands/sweep.mjs +15 -6
- package/src/gate/changed.mjs +40 -3
- package/src/git.mjs +4 -2
- package/src/init/templates/SKILL.md +1 -1
- package/src/probe/attribution.mjs +2 -0
- package/src/probe/discover.mjs +44 -8
- package/src/probe/imports.mjs +785 -0
- package/src/probe/mocks.mjs +13 -4
- package/src/probe/probe.mjs +177 -20
- package/src/probe/rank.mjs +28 -5
- package/src/probe/runners/discovery.mjs +636 -0
- package/src/probe/runners/index.mjs +51 -19
- package/src/probe/runners/jest.mjs +32 -1
- package/src/probe/runners/playwright.mjs +58 -2
- package/src/probe/runners/python.mjs +7 -7
- package/src/probe/runners/shared.mjs +71 -7
- package/src/probe/runners/vitest.mjs +21 -0
- package/src/replay/replay.mjs +108 -13
- package/src/status/status.mjs +6 -0
- package/src/status/surface.mjs +3 -2
- package/src/sweep/sweep.mjs +5 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,77 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.15.1] - 2026-09-30
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.15.0`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- fix: runner timeout identity and reported adoption regressions (#180)
|
|
17
|
+
- chore(homebrew): sha256 for v0.15.0 (#177)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- Assertion messages quoting "timed out" no longer become runner timeouts.
|
|
23
|
+
Native timeout headers and structured timeout failures remain non-detections
|
|
24
|
+
(#179).
|
|
25
|
+
- Configuration dependency hashing no longer rejects harmless absolute-looking
|
|
26
|
+
strings such as Vite URL bases, middleware routes, or `split('/')`. Existing
|
|
27
|
+
outside-project file dependencies and symlinks still fail closed (#178).
|
|
28
|
+
- Discovery failures no longer assert that dependencies are missing when the
|
|
29
|
+
runner resolved successfully but its configuration could not be discovered.
|
|
30
|
+
- Each subcommand's `--help` now shows its own options and an example; global
|
|
31
|
+
`--help` exits successfully and points to command-specific help (#120).
|
|
32
|
+
- Parent gates, status change reports, and claimed-surface counts respect valid
|
|
33
|
+
nested TestGuard project boundaries. Delegated files are explicitly reported
|
|
34
|
+
and require a separate child gate; invalid or symlinked markers fail (#150).
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- Explicit `probe --allow-empty` adoption mode, also available as an opt-in
|
|
39
|
+
GitHub Action input. A valid empty claims file skips verification without
|
|
40
|
+
writing evidence; JSON keeps the `no-claims` state. Default empty probes,
|
|
41
|
+
invalid claims, and explicit claim selections still fail (#176).
|
|
42
|
+
|
|
43
|
+
## [0.15.0] - 2026-09-29
|
|
44
|
+
|
|
45
|
+
Automated weekly release — everything merged since `v0.14.2`.
|
|
46
|
+
|
|
47
|
+
### Changed
|
|
48
|
+
|
|
49
|
+
- feat: harden discovery, replay, and command budgets (#174)
|
|
50
|
+
- chore(homebrew): sha256 for v0.14.2 (#173)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
### Added
|
|
54
|
+
|
|
55
|
+
- Runner-native test discovery for Vitest, Jest, and Playwright, with bounded
|
|
56
|
+
output, path validation, immutable universe manifests, exact mixed-runner
|
|
57
|
+
routing, and configuration-closure hashes (including workspace configs,
|
|
58
|
+
package helpers, setup files, and Python collection hooks) that prevent stale
|
|
59
|
+
evidence reuse.
|
|
60
|
+
- Symbol-aware JavaScript/TypeScript defender discovery through named,
|
|
61
|
+
default, namespace, CommonJS, and barrel re-exports. Ambiguity and bounded
|
|
62
|
+
resolver exhaustion fail closed as `defender-discovery-indeterminate`.
|
|
63
|
+
- `probe`, `sweep`, and `replay` accept `--command-budget <ms>` as a
|
|
64
|
+
cooperative measurement deadline distinct from the existing per-run
|
|
65
|
+
`--budget`. It caps asynchronous children and is checked at stage and write
|
|
66
|
+
boundaries; synchronous setup may overrun but cannot publish a partial
|
|
67
|
+
result, so an unattempted suffix can never be reported as clean.
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- Runner timeouts now hard-kill the original process group and descendants
|
|
72
|
+
still attributable at timeout. Diagnostics explicitly say cleanup is
|
|
73
|
+
unverified because a previously reparented daemon requires external OS or
|
|
74
|
+
container containment.
|
|
75
|
+
- Generated CommonJS export mutations and unresolved symbol/configuration
|
|
76
|
+
paths fail closed instead of allowing a false `nocover` result.
|
|
77
|
+
- Dirty-tree preflight now resolves the same full primary-plus-owned runner
|
|
78
|
+
universe as measurement, including custom Playwright names and imported
|
|
79
|
+
configuration helpers; discovery failure blocks rather than hiding edits.
|
|
80
|
+
|
|
10
81
|
## [0.14.2] - 2026-09-28
|
|
11
82
|
|
|
12
83
|
Automated weekly release — everything merged since `v0.14.1`.
|
|
@@ -15,7 +86,6 @@ Automated weekly release — everything merged since `v0.14.1`.
|
|
|
15
86
|
|
|
16
87
|
- chore(homebrew): sha256 for v0.14.1 (#171)
|
|
17
88
|
|
|
18
|
-
|
|
19
89
|
## [0.14.1] - 2026-09-27
|
|
20
90
|
|
|
21
91
|
Automated weekly release — everything merged since `v0.14.0`.
|
package/README.md
CHANGED
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.15.1` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.15.1/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -407,6 +407,20 @@ is missing. TestGuard itself still makes no network calls.
|
|
|
407
407
|
|
|
408
408
|
### Starting from zero claims
|
|
409
409
|
|
|
410
|
+
During gradual adoption, use `testguard probe --allow-empty` to skip a valid
|
|
411
|
+
empty claims file without blocking a push. It explicitly reports that no
|
|
412
|
+
verification was performed and writes no evidence; `--json` retains the
|
|
413
|
+
`no-claims` state. Missing or invalid claims files still fail. Run `gate`
|
|
414
|
+
separately to enforce changed-file coverage. For the pre-commit probe hook,
|
|
415
|
+
set `args: [--allow-empty]`; for the GitHub Action, set `allow-empty: 'true'`.
|
|
416
|
+
The default probe still exits 2 when no claims exist.
|
|
417
|
+
|
|
418
|
+
In a monorepo, a valid nested `testguard.claims.json` marks a separate project.
|
|
419
|
+
The parent gate reports its changed files as delegated and does not count them
|
|
420
|
+
as parent coverage. Run `gate` in each child project as well; a passing parent
|
|
421
|
+
gate does not establish that child coverage passed. Invalid child claims fail
|
|
422
|
+
evaluation instead of hiding files.
|
|
423
|
+
|
|
410
424
|
`gate` names the changed files that carry no claim, and stops there —
|
|
411
425
|
correctly, because stating a claim is a human act. But a project adopting this
|
|
412
426
|
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
@@ -663,12 +677,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
663
677
|
# GitHub Actions
|
|
664
678
|
- uses: actions/checkout@v4
|
|
665
679
|
with: { fetch-depth: 0 }
|
|
666
|
-
- uses: raccioly/testguard@v0.
|
|
680
|
+
- uses: raccioly/testguard@v0.15.1
|
|
667
681
|
with: { command: gate }
|
|
668
682
|
|
|
669
683
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
670
684
|
include:
|
|
671
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
685
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.15.1/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
672
686
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
673
687
|
```
|
|
674
688
|
|
|
@@ -694,6 +708,18 @@ talks to the network; only the job posts, and only when told to.
|
|
|
694
708
|
- **Never optimistic.** A timeout, a load failure, a mixed N-run result, a
|
|
695
709
|
flaky defender or a missing anchor is reported as unproven, never rounded
|
|
696
710
|
toward green. The one pass is `killed`; everything else gates.
|
|
711
|
+
- **Per-run and whole-command time are separate.** `--budget` limits each
|
|
712
|
+
runner invocation. `--command-budget` is a cooperative deadline for the
|
|
713
|
+
complete `probe`, `sweep`, or `replay` measurement: asynchronous children
|
|
714
|
+
receive only the remaining time, and stage/write boundaries re-check it.
|
|
715
|
+
Synchronous setup may overrun the nominal deadline, but once expiry is
|
|
716
|
+
observed TestGuard exits `2` and writes no new result document; a completed
|
|
717
|
+
prefix is not presented as though the unattempted suffix were clean.
|
|
718
|
+
- **Configured discovery is evidence.** Vitest, Jest, and Playwright enumerate
|
|
719
|
+
their own configured files; TestGuard validates and hashes that immutable
|
|
720
|
+
universe instead of guessing with filename globs. Automatic JavaScript and
|
|
721
|
+
TypeScript discovery follows the imported symbol through barrels. Ambiguous
|
|
722
|
+
or unbounded resolution is `unverifiable`, never a flattering `nocover`.
|
|
697
723
|
- **One exact-pinned runtime dependency** (`ajv`), Node ≥ 20, MIT.
|
|
698
724
|
|
|
699
725
|
### Prior art, and what was taken from where
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.1",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -12,7 +12,7 @@ The verdict set is closed. Only one value is a pass.
|
|
|
12
12
|
| `killed` | Fault applied; every one of N probe runs failed with a genuine assertion failure, on a defender set that was green N/N unmodified. | no |
|
|
13
13
|
| `survived` | Fault applied; every one of N probe runs passed. The claim is **unproven**. | **yes** |
|
|
14
14
|
| `nocover` | No defending test exists: the declared globs resolve to nothing, or no test imports the subject. Worse than `survived` — nothing was even tried. | **yes** |
|
|
15
|
-
| `unverifiable` | The claim could not be probed: the fault's anchor is missing or ambiguous, its defenders failed to load (`defenders-failed-to-load`), probing it threw (`probe-error`), or the defenders do not execute the subject at all (`subject-not-executed`). Carries a `reason`. A loud, gating verdict — a claim that cannot be probed is not "skipped", it is undefended until someone fixes the fault or the defenders. Never confused with `flaky-defender`, which requires tests that *ran*. | **yes** |
|
|
15
|
+
| `unverifiable` | The claim could not be probed: the fault's anchor is missing or ambiguous, configured or symbol-level defender discovery was indeterminate (`defender-discovery-indeterminate`), its defenders failed to load (`defenders-failed-to-load`), probing it threw (`probe-error`), or the defenders do not execute the subject at all (`subject-not-executed`). Carries a `reason`. A loud, gating verdict — a claim that cannot be probed is not "skipped", it is undefended until someone fixes the fault or the defenders. Never confused with `flaky-defender`, which requires tests that *ran*. | **yes** |
|
|
16
16
|
| `timeout` | Probe run exceeded its budget. Not counted as a kill; the pessimistic reading is the safe one because flakiness biases the metric optimistically. | **yes** |
|
|
17
17
|
| `fault-invalid` | The replacement does not load or compile. A bad fault, not a detection. | **yes** |
|
|
18
18
|
| `flaky-defender` | Defenders were not green N/N on unmodified source (`defenders-not-green`), or the N probe runs disagreed with each other (`inconsistent-probe`). Either way no verdict about the fault can be trusted; fix the defenders first. | **yes** |
|
|
@@ -36,7 +36,11 @@ Rules that follow from the table:
|
|
|
36
36
|
that failed and then passed on retry (Playwright's `flaky`) is a
|
|
37
37
|
**non-green run** even when the runner exits 0, and a test the runner
|
|
38
38
|
reports as timed out (`timedOut`) is a timeout, never an assertion
|
|
39
|
-
failure.
|
|
39
|
+
failure. Jest-compatible timeout messages are recognized by native runner
|
|
40
|
+
error headers or structured timeout identity, not by words quoted in an
|
|
41
|
+
assertion, stack frame, or source excerpt. Structured assertion identity
|
|
42
|
+
does not hide a separate timeout in the same failed test.
|
|
43
|
+
Python is read the same way: an `AssertionError` and any other
|
|
40
44
|
exception raised by the test body both count, because both are the suite
|
|
41
45
|
rejecting the behaviour; a collection error — a module that will not
|
|
42
46
|
import, which is what a replacement that does not parse looks like from
|
|
@@ -151,7 +155,48 @@ Rules that follow from the table:
|
|
|
151
155
|
can name the file the interpreter loaded) answers the same question more
|
|
152
156
|
precisely for one language; where both are present they must agree.
|
|
153
157
|
|
|
154
|
-
13. **
|
|
158
|
+
13. **The configured runner defines the test universe.** For JavaScript and
|
|
159
|
+
TypeScript, defender discovery, baseline runs and escalation use the exact
|
|
160
|
+
file set returned by the resolved runner's native listing protocol. A
|
|
161
|
+
resolvable runner whose configuration cannot load, whose listing times out
|
|
162
|
+
or is malformed, or whose reported paths escape the project is a
|
|
163
|
+
precondition failure. It never falls back to built-in filename globs and
|
|
164
|
+
never becomes `nocover`. An explicitly selected runner that lists zero
|
|
165
|
+
files is also a precondition failure; `auto` may continue past a successful
|
|
166
|
+
empty listing, but must fail when every eligible runner is empty.
|
|
167
|
+
`inputs.testUniverseHash` binds the normalized file set, runner identity,
|
|
168
|
+
and hashed configuration dependencies. Those dependencies include absent
|
|
169
|
+
root candidates, nested workspace configs, Python collection hooks, and the
|
|
170
|
+
bounded static closure of local or package-imported config helpers and setup
|
|
171
|
+
files; creating or editing any of them forces remeasurement. Evidence that
|
|
172
|
+
predates this hash, or whose hash
|
|
173
|
+
changed, is readable but never reusable for a current native discovery run.
|
|
174
|
+
When several configured runners list defenders, exact manifest membership
|
|
175
|
+
assigns each file. An owned runner takes precedence over the project runner;
|
|
176
|
+
competing owned runners or a declared defender absent from every manifest
|
|
177
|
+
are precondition failures, never heuristic routing decisions.
|
|
178
|
+
|
|
179
|
+
Static string literals are conservative dependency candidates, not proof
|
|
180
|
+
of a filesystem dependency. Missing candidates and ordinary directories
|
|
181
|
+
(including `/` in URL bases and string separators) are ignored. An existing
|
|
182
|
+
candidate file outside the project, or a candidate symlink, is rejected.
|
|
183
|
+
|
|
184
|
+
14. **Barrel resolution follows symbols, not module reachability.** Automatic
|
|
185
|
+
discovery may traverse a re-export chain only for the binding the test
|
|
186
|
+
actually imports. Importing an unrelated symbol from a broad barrel does
|
|
187
|
+
not defend every file the barrel exports. Renames, explicit exports and
|
|
188
|
+
star exports are resolved deterministically; type-only paths do not count.
|
|
189
|
+
Ambiguous origins, unsupported runtime-generated exports, graph-limit
|
|
190
|
+
exhaustion, and originless cycles are `unverifiable` with reason
|
|
191
|
+
`defender-discovery-indeterminate`, never `nocover`. A declared
|
|
192
|
+
`defendedBy` remains authoritative. `inputs.discoveryHashes` binds every
|
|
193
|
+
candidate test, intermediate barrel, and resolver-configuration file
|
|
194
|
+
consulted by automatic discovery. Negative candidates are dependencies too:
|
|
195
|
+
editing a previously unrelated test so it begins importing the target must
|
|
196
|
+
invalidate an earlier `nocover`. Absent or changed dependencies force
|
|
197
|
+
remeasurement.
|
|
198
|
+
|
|
199
|
+
15. **Fault injection is one method, not the definition.** A claim declares
|
|
155
200
|
what must be true; a probe declares how a tool would try to make it false,
|
|
156
201
|
and `method` says which way. `fault-injection` — mutate the source, run the
|
|
157
202
|
claim's defenders, confirm over N runs — is the only method specified in
|
|
@@ -184,6 +229,32 @@ Rules that follow from the table:
|
|
|
184
229
|
The verdict set stays closed and shared. What each method may legitimately
|
|
185
230
|
report is part of defining it.
|
|
186
231
|
|
|
232
|
+
## Per-run and whole-command budgets are different
|
|
233
|
+
|
|
234
|
+
`--budget` remains the wall-clock ceiling for one runner invocation. An
|
|
235
|
+
optional `--command-budget` is a cooperative deadline for the complete
|
|
236
|
+
`probe`, `sweep`, or `replay` measurement. Every asynchronous child runner and
|
|
237
|
+
discovery process is given no more than the command's remaining time, and the
|
|
238
|
+
deadline is checked again at stage boundaries and immediately before any
|
|
239
|
+
result is written. Synchronous filesystem, Git, parsing, and cleanup work is
|
|
240
|
+
not preemptible in portable Node.js and may finish after the nominal deadline;
|
|
241
|
+
that overrun grants no authority to publish a result.
|
|
242
|
+
|
|
243
|
+
Exhausting the whole-command budget is not a test verdict and is never a
|
|
244
|
+
successful partial measurement. The command exits `2` and writes no new
|
|
245
|
+
result document; any previous document remains untouched. Cleanup may finish
|
|
246
|
+
after the deadline, but no record from the incomplete operation becomes
|
|
247
|
+
evidence. This is deliberately stricter than a per-run timeout: publishing
|
|
248
|
+
the completed prefix would let the unattempted suffix look clean.
|
|
249
|
+
|
|
250
|
+
Timeout cleanup is best effort, not portable process containment. TestGuard
|
|
251
|
+
hard-kills the runner's original process group and every descendant still
|
|
252
|
+
attributable through the process table. A daemon that detached, reparented,
|
|
253
|
+
and disappeared from that ancestry before timeout may survive, so timeout
|
|
254
|
+
diagnostics state `cleanup-unverified`. Such a run remains a timeout and never
|
|
255
|
+
becomes evidence of detection. Daemonizing or untrusted commands require an
|
|
256
|
+
OS or container boundary that TestGuard does not claim to provide.
|
|
257
|
+
|
|
187
258
|
## Replay reports; it never gates
|
|
188
259
|
|
|
189
260
|
A replayed bug is history. It escaped, by definition, which means the suite
|
|
@@ -198,7 +269,7 @@ Its verdicts mirror the probe's, for the same reasons:
|
|
|
198
269
|
| `caught` | a remaining test failed **by assertion** on the reverted source, every run. The suite knew. |
|
|
199
270
|
| `blind` | the suite stayed green on known-broken code. |
|
|
200
271
|
| `nocover` | no test imports the reverted files, **and at least one test file remained to ask**. Worse than `blind`: nothing was even tried. |
|
|
201
|
-
| `unverifiable` | the revert did not apply, the suite failed to load, it timed out, or removing the fix's own test left **no test file at all** (`the-fix-shipped-the-only-test-file`). Carries a `reason`. |
|
|
272
|
+
| `unverifiable` | the revert did not apply, configured or symbol-level defender discovery was indeterminate (`defender-discovery-indeterminate`), the suite failed to load, it timed out, or removing the fix's own test left **no test file at all** (`the-fix-shipped-the-only-test-file`). Carries a `reason`. |
|
|
202
273
|
| `flaky` | the runs disagreed. A flaky failure reads as "the suite caught it", so flakiness biases this metric **optimistically** — mixed runs are never `caught`. |
|
|
203
274
|
|
|
204
275
|
Two rules that follow:
|
|
@@ -432,10 +503,22 @@ Rules:
|
|
|
432
503
|
`unclaimed-changes` and its next action is to write the claim, before any
|
|
433
504
|
`unproven` finding is surfaced. The brief renders them first. The claim is
|
|
434
505
|
written before more code.
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
506
|
+
7. **A descendant claims file defines a separate project.** The first
|
|
507
|
+
descendant directory on a changed file's path with a valid, regular,
|
|
508
|
+
non-symlinked `testguard.claims.json` owns that subtree. Invalid or unreadable
|
|
509
|
+
markers fail evaluation; they cannot hide unclaimed files. Parent gate and
|
|
510
|
+
status report the delegation as optional `nested` entries with project and
|
|
511
|
+
file paths relative to the parent, and instruct the operator to run that
|
|
512
|
+
project's gate. Delegation is not coverage, an ignore reliance, or proof
|
|
513
|
+
that the child passed. Parent source-surface counts omit these subtrees.
|
|
514
|
+
Removing the marker restores parent evaluation. The accounting identity is
|
|
515
|
+
`changed = evaluated + excluded.length + sum(nested[].files.length)`;
|
|
516
|
+
`--strict` still fails a non-empty change with zero parent evaluation.
|
|
517
|
+
|
|
518
|
+
Exit codes: `0` every changed parent-owned source file is claimed or excused
|
|
519
|
+
(or nothing changed); delegated projects still require separate child gates.
|
|
520
|
+
`1` at least one uncovered file (or strict mode over an
|
|
521
|
+
all-excluded or all-delegated change); `2` the change cannot be evaluated (unresolvable
|
|
439
522
|
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
440
523
|
|
|
441
524
|
## Anchor preflight
|
|
@@ -486,3 +569,11 @@ evidence; they simply do not turn CI red.
|
|
|
486
569
|
A tool must never exit `0` because it had nothing to check. If the claims
|
|
487
570
|
file is empty or every claim is out of scope, that is reported explicitly and
|
|
488
571
|
the exit code is `2`.
|
|
572
|
+
|
|
573
|
+
`probe --allow-empty` is an explicit adoption exception for a valid claims
|
|
574
|
+
file containing zero claims, with no `--claim` selection. It exits `0`, says
|
|
575
|
+
that verification was skipped, and writes no evidence or baseline. Its JSON
|
|
576
|
+
output is the existing `no-claims` status document, without run metadata;
|
|
577
|
+
it never reports a clean measurement. Missing or invalid claims still fail,
|
|
578
|
+
and selected claims cannot be silently skipped. Run `gate` separately to
|
|
579
|
+
enforce coverage of changed files during adoption. Integrations must opt in.
|
package/spec/README.md
CHANGED
|
@@ -39,6 +39,13 @@ Shared definitions (verdicts, fault classes, provenance, annotations) live in
|
|
|
39
39
|
which verdicts turn CI red, baseline and delta rules, exit codes — is in
|
|
40
40
|
[`GATE-SEMANTICS.md`](GATE-SEMANTICS.md).
|
|
41
41
|
|
|
42
|
+
Operational completeness is contractual too. `--budget` bounds one runner
|
|
43
|
+
invocation; `--command-budget` is a cooperative deadline for a complete probe,
|
|
44
|
+
sweep, or replay measurement. It caps asynchronous child processes, is checked
|
|
45
|
+
at stage and write boundaries, and produces no document once observed expired.
|
|
46
|
+
Synchronous setup may overrun but cannot authorize a partial result. The exact
|
|
47
|
+
rule is in `GATE-SEMANTICS.md`.
|
|
48
|
+
|
|
42
49
|
Schemas are JSON Schema 2020-12 and identified as `urn:claimspec:v1:<kind>`.
|
|
43
50
|
|
|
44
51
|
## Conformance
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -28,6 +28,24 @@ const schemaFor = (kind) => {
|
|
|
28
28
|
return v;
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
+
const nestedErrors = (nested = [], prefix = '/nested', occupied = []) => {
|
|
32
|
+
const errors = [];
|
|
33
|
+
const projects = new Set();
|
|
34
|
+
const files = new Set(occupied);
|
|
35
|
+
for (const [i, entry] of nested.entries()) {
|
|
36
|
+
const p = `${prefix}/${i}`;
|
|
37
|
+
if (entry.project.split('/').some((part) => !part || part === '.' || part === '..') || entry.project.includes('\\')) errors.push({ path: `${p}/project`, message: 'nested project must be a canonical descendant directory' });
|
|
38
|
+
if (projects.has(entry.project)) errors.push({ path: `${p}/project`, message: 'nested project is duplicated' });
|
|
39
|
+
projects.add(entry.project);
|
|
40
|
+
for (const file of entry.files) {
|
|
41
|
+
if (!file.startsWith(`${entry.project}/`) || file.split('/').some((part) => !part || part === '.' || part === '..') || file.includes('\\')) errors.push({ path: `${p}/files`, message: 'nested file must remain beneath its canonical project directory' });
|
|
42
|
+
if (files.has(file)) errors.push({ path: `${p}/files`, message: 'nested file is counted in more than one bucket' });
|
|
43
|
+
files.add(file);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return errors;
|
|
47
|
+
};
|
|
48
|
+
|
|
31
49
|
// Rules JSON Schema cannot express. Each returns an array of {path, message}.
|
|
32
50
|
const semantic = {
|
|
33
51
|
claims(doc) {
|
|
@@ -243,6 +261,11 @@ const semantic = {
|
|
|
243
261
|
|
|
244
262
|
status(doc) {
|
|
245
263
|
const errors = [];
|
|
264
|
+
if (doc.changes?.nested) {
|
|
265
|
+
errors.push(...nestedErrors(doc.changes.nested, '/changes/nested', doc.changes.uncovered.map((file) => file.file)));
|
|
266
|
+
const nestedCount = doc.changes.nested.reduce((count, entry) => count + entry.files.length, 0);
|
|
267
|
+
if (doc.changes.changed !== doc.changes.evaluated + doc.changes.excluded + nestedCount) errors.push({ path: '/changes/changed', message: 'changed must account for evaluated, excluded, and nested files' });
|
|
268
|
+
}
|
|
246
269
|
if (['write-test', 'review-fault-change', 'repair-fault'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
247
270
|
if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
|
|
248
271
|
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
@@ -311,8 +334,10 @@ const semantic = {
|
|
|
311
334
|
|
|
312
335
|
gate(doc) {
|
|
313
336
|
const errors = [];
|
|
337
|
+
const nestedCount = (doc.nested ?? []).reduce((count, entry) => count + entry.files.length, 0);
|
|
338
|
+
errors.push(...nestedErrors(doc.nested, '/nested', [...doc.covered, ...doc.uncovered, ...doc.excluded].map((entry) => entry.file)));
|
|
314
339
|
if (doc.evaluated !== doc.covered.length + doc.uncovered.length) errors.push({ path: '/evaluated', message: `evaluated (${doc.evaluated}) must equal covered (${doc.covered.length}) + uncovered (${doc.uncovered.length})` });
|
|
315
|
-
if (doc.changed !== doc.evaluated + doc.excluded.length) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length})` });
|
|
340
|
+
if (doc.changed !== doc.evaluated + doc.excluded.length + nestedCount) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length}) + nested (${nestedCount})` });
|
|
316
341
|
if (doc.uncovered.length > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'an uncovered changed file must exit 1; the gate never passes over unclaimed code' });
|
|
317
342
|
if (doc.uncovered.length === 0 && doc.exitCode === 1 && !(doc.strict && doc.changed > 0 && doc.evaluated === 0)) errors.push({ path: '/exitCode', message: 'exit 1 without uncovered files is only valid under --strict when a non-empty change evaluated nothing' });
|
|
318
343
|
doc.covered.forEach((c, i) => {
|
|
@@ -4,6 +4,16 @@
|
|
|
4
4
|
"title": "claimspec v1 \u2014 shared definitions",
|
|
5
5
|
"description": "Definitions shared by every Guard-spec format. Referenced as urn:claimspec:v1:common#/$defs/<name>.",
|
|
6
6
|
"$defs": {
|
|
7
|
+
"nestedProject": {
|
|
8
|
+
"description": "Changed files delegated to a separately validated descendant TestGuard project. This is not coverage or evidence; run that project's gate separately.",
|
|
9
|
+
"type": "object",
|
|
10
|
+
"required": ["project", "files"],
|
|
11
|
+
"properties": {
|
|
12
|
+
"project": { "$ref": "#/$defs/repoPath" },
|
|
13
|
+
"files": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "#/$defs/repoPath" } }
|
|
14
|
+
},
|
|
15
|
+
"additionalProperties": false
|
|
16
|
+
},
|
|
7
17
|
"sha256": {
|
|
8
18
|
"type": "string",
|
|
9
19
|
"pattern": "^[a-f0-9]{64}$"
|
|
@@ -641,6 +641,20 @@
|
|
|
641
641
|
"additionalProperties": {
|
|
642
642
|
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
643
643
|
}
|
|
644
|
+
},
|
|
645
|
+
"testUniverseHash": {
|
|
646
|
+
"description": "Canonical hash of the runner-native test universe and the configuration files that produced it. Optional for backward compatibility; current probes emit it, and evidence without it is never reusable against a configured discovery run.",
|
|
647
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
648
|
+
},
|
|
649
|
+
"discoveryHashes": {
|
|
650
|
+
"description": "Content hashes of candidate tests, intermediate barrels, and resolver-configuration files consulted by automatic defender discovery. This includes negative candidates: a test edited to begin importing the target must invalidate an earlier `nocover`. Optional for backward compatibility; a changed or absent dependency forces remeasurement.",
|
|
651
|
+
"type": "object",
|
|
652
|
+
"propertyNames": {
|
|
653
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
654
|
+
},
|
|
655
|
+
"additionalProperties": {
|
|
656
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
657
|
+
}
|
|
644
658
|
}
|
|
645
659
|
},
|
|
646
660
|
"additionalProperties": false
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
18
|
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
19
19
|
"evaluated": { "description": "covered + uncovered: the changed files that are source and not excluded.", "type": "integer", "minimum": 0 },
|
|
20
|
+
"nested": { "description": "Optional descendant-project delegations; absence means none. These files are neither covered nor evaluated by the parent.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
20
21
|
"excluded": {
|
|
21
22
|
"type": "array",
|
|
22
23
|
"items": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
27
27
|
},
|
|
28
28
|
"state": {
|
|
29
|
-
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
29
|
+
"description": "no-claims: no claims file or a valid empty claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
30
30
|
"enum": [
|
|
31
31
|
"no-claims",
|
|
32
32
|
"unclaimed-changes",
|
|
@@ -392,6 +392,7 @@
|
|
|
392
392
|
"expired"
|
|
393
393
|
],
|
|
394
394
|
"properties": {
|
|
395
|
+
"nested": { "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
395
396
|
"ref": {
|
|
396
397
|
"type": "string",
|
|
397
398
|
"minLength": 1,
|
package/src/cli.mjs
CHANGED
|
@@ -44,6 +44,7 @@ probe
|
|
|
44
44
|
--claims <path> claims file (default: <dir>/testguard.claims.json)
|
|
45
45
|
--confirm <n> runs per verdict (default: 3)
|
|
46
46
|
--budget <ms> wall clock per run (default: 120000)
|
|
47
|
+
--command-budget <ms> cooperative measurement deadline for probe, sweep, or replay. Caps async children; expiry exits 2 and writes no partial result
|
|
47
48
|
--out <path> evidence file (default: <dir>/.testguard/evidence.json)
|
|
48
49
|
--baseline <path> baseline to gate against (default: <dir>/.testguard/baseline.json if present)
|
|
49
50
|
--severity <level> gate only at or above (default: low)
|
|
@@ -74,6 +75,7 @@ probe
|
|
|
74
75
|
--in-place mutate the working tree instead of a scratch worktree
|
|
75
76
|
--no-escalate do not re-run survivors against the whole suite
|
|
76
77
|
--no-reuse re-probe claims whose inputs have not changed
|
|
78
|
+
--allow-empty adoption only: skip a valid empty claims file; writes no evidence; run gate separately
|
|
77
79
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
78
80
|
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
79
81
|
|
|
@@ -84,7 +86,7 @@ sweep --changed <ref> (required unless --save-paths; CI bases and a safe lo
|
|
|
84
86
|
--save-paths sweep every file that WRITES TO STORAGE instead of the diff, and report the surface as the
|
|
85
87
|
denominator: "120 writes across 38 files, swept 12". A clean sweep over an unstated
|
|
86
88
|
denominator is a sample of unknown size, not a guarantee that saving works
|
|
87
|
-
--include-dirty --exclude <glob> --confirm <n> --budget <ms> --max <n> --out <path> --json
|
|
89
|
+
--include-dirty --exclude <glob> --confirm <n> --budget <ms> --command-budget <ms> --max <n> --out <path> --json
|
|
88
90
|
the cold start: no claim and no concern needed. Everything it proposes is a DRAFT — it never writes testguard.claims.json,
|
|
89
91
|
and its evidence never replaces .testguard/evidence.json. exit 1 on a fault that survived, or a file no test imports
|
|
90
92
|
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
@@ -96,6 +98,7 @@ mcp no options. JSON-RPC 2.0 over stdio; five READ-ONLY tools (status, br
|
|
|
96
98
|
replay --since <range> commit range to search for fix commits (HEAD~50..HEAD, a tag, origin/main..HEAD)
|
|
97
99
|
--max <n> replay at most n fixes (they are slow: one worktree and N runs each)
|
|
98
100
|
--confirm <n> runs per verdict (default 3); a flaky failure reads as "the suite caught it", so mixed runs are never caught
|
|
101
|
+
--command-budget <ms> cooperative measurement deadline; expiry writes neither replay nor calibration, so an unmeasured suffix cannot look clean
|
|
99
102
|
--out <path> replay document (default: <dir>/.testguard/replay.json); the calibration goes beside it
|
|
100
103
|
reports, never gates: a bug that escaped is history, not a regression in this change
|
|
101
104
|
· one-line JSX element removed · on<Event> handler prop dropped
|
|
@@ -132,6 +135,27 @@ exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or uncl
|
|
|
132
135
|
|
|
133
136
|
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand, concerns: concernsCommand };
|
|
134
137
|
|
|
138
|
+
/** Derive command help from the same option descriptions as the overview. */
|
|
139
|
+
export function commandUsage(command) {
|
|
140
|
+
const lines = USAGE.split('\n');
|
|
141
|
+
const overview = lines.find((line) => line.startsWith(` testguard ${command} `)) ?? ` testguard ${command}`;
|
|
142
|
+
const sections = [];
|
|
143
|
+
let selected = false;
|
|
144
|
+
for (const line of lines) {
|
|
145
|
+
const heading = /^([a-z]+)(?:\s|$)/.exec(line)?.[1];
|
|
146
|
+
if (heading && Object.hasOwn(COMMANDS, heading)) selected = heading === command;
|
|
147
|
+
else if (line.startsWith('every command ') || line.startsWith('exit codes:')) selected = false;
|
|
148
|
+
if (selected) sections.push(line);
|
|
149
|
+
}
|
|
150
|
+
if (command === 'concerns') sections.push('concerns --concerns <path> read a custom concerns file; --json prints the validated inventory');
|
|
151
|
+
const examples = {
|
|
152
|
+
probe: 'probe . --confirm 3', gate: 'gate . --changed origin/main',
|
|
153
|
+
scaffold: 'scaffold src/example.mjs --json', replay: 'replay . --since HEAD~10..HEAD --max 3',
|
|
154
|
+
admit: 'admit test/example.test.mjs --claim EXAMPLE-001', sweep: 'sweep . --changed origin/main --cap 10',
|
|
155
|
+
};
|
|
156
|
+
return `${lines[0]}\n\n${overview}\n\n${sections.join('\n').trim()}\n\nCommon: --json, --help, --version\nExample: testguard ${examples[command] ?? `${command}${command === 'mcp' ? '' : ' .'}`}\n`;
|
|
157
|
+
}
|
|
158
|
+
|
|
135
159
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
136
160
|
let parsed;
|
|
137
161
|
try {
|
|
@@ -142,6 +166,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
142
166
|
claims: { type: 'string' },
|
|
143
167
|
confirm: { type: 'string', default: '3' },
|
|
144
168
|
budget: { type: 'string', default: '120000' },
|
|
169
|
+
'command-budget': { type: 'string' },
|
|
145
170
|
out: { type: 'string' },
|
|
146
171
|
evidence: { type: 'string' },
|
|
147
172
|
baseline: { type: 'string' },
|
|
@@ -177,6 +202,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
177
202
|
'in-place': { type: 'boolean', default: false },
|
|
178
203
|
'no-escalate': { type: 'boolean', default: false },
|
|
179
204
|
'no-reuse': { type: 'boolean', default: false },
|
|
205
|
+
'allow-empty': { type: 'boolean', default: false },
|
|
180
206
|
quiet: { type: 'boolean', default: false },
|
|
181
207
|
json: { type: 'boolean', default: false },
|
|
182
208
|
text: { type: 'boolean', default: false },
|
|
@@ -199,9 +225,9 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
199
225
|
return 0;
|
|
200
226
|
}
|
|
201
227
|
const [command, dirArg] = positionals;
|
|
202
|
-
if (
|
|
203
|
-
io.out(USAGE);
|
|
204
|
-
return
|
|
228
|
+
if (!command) {
|
|
229
|
+
io.out(USAGE + "\nRun 'testguard <command> --help' for that command's options.");
|
|
230
|
+
return values.help ? 0 : 3;
|
|
205
231
|
}
|
|
206
232
|
const handler = COMMANDS[command];
|
|
207
233
|
if (!handler) {
|
|
@@ -209,6 +235,10 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
209
235
|
io.err(USAGE);
|
|
210
236
|
return 3;
|
|
211
237
|
}
|
|
238
|
+
if (values.help) {
|
|
239
|
+
io.out(commandUsage(command));
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
212
242
|
if (![...RUNNER_NAMES, 'auto'].includes(values.runner)) {
|
|
213
243
|
io.err(`--runner must be one of ${RUNNER_NAMES.join(', ')} or auto`);
|
|
214
244
|
return 3;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { PreconditionError } from './probe/worktree.mjs';
|
|
2
|
+
import { performance } from 'node:perf_hooks';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A cooperative measurement deadline shared by every stage of one command.
|
|
6
|
+
*
|
|
7
|
+
* Per-run budgets answer "how long may this runner invocation take?". This
|
|
8
|
+
* deadline answers the different question "may the whole command still
|
|
9
|
+
* produce a result?". Async child processes receive no more than the
|
|
10
|
+
* remaining time; synchronous setup can overrun because JavaScript cannot
|
|
11
|
+
* preempt it, so every stage boundary and every write boundary checks again.
|
|
12
|
+
* Expiry is deliberately not a verdict: a partial result would make the
|
|
13
|
+
* unattempted suffix look clean.
|
|
14
|
+
*/
|
|
15
|
+
export class CommandBudget {
|
|
16
|
+
constructor(limitMs, { now = () => performance.now() } = {}) {
|
|
17
|
+
if (!Number.isInteger(limitMs) || limitMs < 1) throw new TypeError('command budget must be a positive integer');
|
|
18
|
+
this.limitMs = limitMs;
|
|
19
|
+
this.now = now;
|
|
20
|
+
this.startedAtMs = now();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
elapsedMs() {
|
|
24
|
+
return Math.max(0, this.now() - this.startedAtMs);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
remainingMs() {
|
|
28
|
+
return Math.max(0, this.limitMs - this.elapsedMs());
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
assertOpen() {
|
|
32
|
+
if (this.remainingMs() === 0) {
|
|
33
|
+
throw new PreconditionError(`command budget of ${this.limitMs}ms exhausted; no partial result was written`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Never let a child process outlive the command's remaining measurement window. */
|
|
38
|
+
runBudget(perRunMs) {
|
|
39
|
+
this.assertOpen();
|
|
40
|
+
return Math.max(1, Math.min(perRunMs, Math.floor(this.remainingMs())));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function createCommandBudget(limitMs, options) {
|
|
45
|
+
return limitMs === undefined ? undefined : new CommandBudget(limitMs, options);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function parseCommandBudget(value) {
|
|
49
|
+
if (value === undefined) return undefined;
|
|
50
|
+
const parsed = Number(value);
|
|
51
|
+
return Number.isInteger(parsed) && parsed >= 1000 ? parsed : null;
|
|
52
|
+
}
|