testguard-cli 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +62 -8
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +55 -5
- package/spec/lib/validate.mjs +35 -1
- package/spec/schemas/claims.schema.json +6 -0
- package/spec/schemas/common.schema.json +10 -0
- package/spec/schemas/evidence.schema.json +4 -0
- package/spec/schemas/gate.schema.json +1 -0
- package/spec/schemas/status.schema.json +27 -1
- package/src/cli.mjs +30 -3
- package/src/commands/admit.mjs +8 -3
- package/src/commands/claims.mjs +40 -4
- package/src/commands/probe.mjs +13 -0
- package/src/gate/changed.mjs +45 -4
- package/src/init/init.mjs +6 -0
- package/src/init/templates/SKILL.md +44 -1
- package/src/probe/attribution.mjs +9 -0
- package/src/probe/cost.mjs +16 -8
- package/src/probe/probe.mjs +28 -24
- package/src/probe/runners/discovery.mjs +4 -2
- package/src/probe/runners/shared.mjs +16 -1
- package/src/status/status.mjs +15 -2
- package/src/status/surface.mjs +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.16.0] - 2026-09-30
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.15.1`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- feat: fault-specific defenders, provenance and cost reporting (#184)
|
|
17
|
+
- Add advisory AI update guidance and stabilize cleanup regression (#183)
|
|
18
|
+
- chore(homebrew): sha256 for v0.15.1 (#182)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Fault-specific `defendedBy` overrides, with explicit empty-list discovery,
|
|
24
|
+
evidence selection provenance, per-fault cost reporting and prior-kill
|
|
25
|
+
narrowing warnings (#90). Selection changes invalidate reuse and status;
|
|
26
|
+
admission and changed-file coverage use the actual selected defenders.
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
|
|
30
|
+
- New AI integrations include permission-aware, once-per-session update advice
|
|
31
|
+
that identifies the actual CLI and requests approval before any upgrade.
|
|
32
|
+
CLI commands and session-start hooks remain offline. Existing customized
|
|
33
|
+
instructions remain untouched; safe manual refresh is documented.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- The daemon-cleanup regression test now requires explicit startup and
|
|
38
|
+
post-cleanup release signals instead of assuming a detached process starts
|
|
39
|
+
and writes within fixed sleeps. Timeout and cleanup semantics are unchanged.
|
|
40
|
+
|
|
41
|
+
## [0.15.1] - 2026-09-30
|
|
42
|
+
|
|
43
|
+
Automated weekly release — everything merged since `v0.15.0`.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- fix: runner timeout identity and reported adoption regressions (#180)
|
|
48
|
+
- chore(homebrew): sha256 for v0.15.0 (#177)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
|
|
53
|
+
- Assertion messages quoting "timed out" no longer become runner timeouts.
|
|
54
|
+
Native timeout headers and structured timeout failures remain non-detections
|
|
55
|
+
(#179).
|
|
56
|
+
- Configuration dependency hashing no longer rejects harmless absolute-looking
|
|
57
|
+
strings such as Vite URL bases, middleware routes, or `split('/')`. Existing
|
|
58
|
+
outside-project file dependencies and symlinks still fail closed (#178).
|
|
59
|
+
- Discovery failures no longer assert that dependencies are missing when the
|
|
60
|
+
runner resolved successfully but its configuration could not be discovered.
|
|
61
|
+
- Each subcommand's `--help` now shows its own options and an example; global
|
|
62
|
+
`--help` exits successfully and points to command-specific help (#120).
|
|
63
|
+
- Parent gates, status change reports, and claimed-surface counts respect valid
|
|
64
|
+
nested TestGuard project boundaries. Delegated files are explicitly reported
|
|
65
|
+
and require a separate child gate; invalid or symlinked markers fail (#150).
|
|
66
|
+
|
|
67
|
+
### Added
|
|
68
|
+
|
|
69
|
+
- Explicit `probe --allow-empty` adoption mode, also available as an opt-in
|
|
70
|
+
GitHub Action input. A valid empty claims file skips verification without
|
|
71
|
+
writing evidence; JSON keeps the `no-claims` state. Default empty probes,
|
|
72
|
+
invalid claims, and explicit claim selections still fail (#176).
|
|
73
|
+
|
|
10
74
|
## [0.15.0] - 2026-09-29
|
|
11
75
|
|
|
12
76
|
Automated weekly release — everything merged since `v0.14.2`.
|
package/README.md
CHANGED
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.16.0` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -221,6 +221,14 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
221
221
|
the evidence records them as `repo.ignoredDirty`, so the run says what it
|
|
222
222
|
did not look at. Every summary names the commit probed.
|
|
223
223
|
|
|
224
|
+
A fault may declare its own `defendedBy` to override the claim's set.
|
|
225
|
+
Omit the fault field to inherit; set it to `[]` to request discovery rather
|
|
226
|
+
than inheritance. Evidence names the selection origin and selected files.
|
|
227
|
+
Changing either invalidates reuse and makes status stale. A narrowed fault
|
|
228
|
+
set after a prior kill is warned about by `claims`; re-probe before trusting
|
|
229
|
+
it. For example, a unit-level fault can select `["test/unit.test.mjs"]`
|
|
230
|
+
while an integration fault inherits the claim's broader set.
|
|
231
|
+
|
|
224
232
|
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
225
233
|
files that import the fault's target, by relative path or resolved alias
|
|
226
234
|
(tsconfig `paths`, through `extends` and `references`, vite/vitest
|
|
@@ -328,6 +336,34 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
328
336
|
offline. The brief's first line says which install answered
|
|
329
337
|
(`local install` or `global`), so a stale one is visible.
|
|
330
338
|
|
|
339
|
+
### Advisory updates for AI sessions
|
|
340
|
+
|
|
341
|
+
New integrations created by `testguard init` tell the AI to identify the actual
|
|
342
|
+
CLI (project-local before global) and, once per session when network policy
|
|
343
|
+
permits, read the registry's stable version:
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
This is an optional check performed by the AI, not by TestGuard's CLI or hook.
|
|
350
|
+
A newer stable version produces a suggestion and a request for approval, never
|
|
351
|
+
an automatic install. Project pins and the original package manager still
|
|
352
|
+
apply; prereleases and development checkouts are not replaced. Offline or
|
|
353
|
+
failed checks do not change verdicts, evidence or exit codes, and do not prove
|
|
354
|
+
that an installation is current. AI instructions are guidance, not a guarantee
|
|
355
|
+
that every harness performs the check.
|
|
356
|
+
|
|
357
|
+
Upgrading the package does **not** refresh an existing copied skill or an
|
|
358
|
+
existing managed section already listing the project. Compare the upgraded
|
|
359
|
+
package's `src/init/templates/SKILL.md` with
|
|
360
|
+
`.claude/skills/testguard/SKILL.md` and merge approved changes, preserving local
|
|
361
|
+
customizations. For an existing `AGENTS.md` section, add the same advisory
|
|
362
|
+
policy after reviewing it. `testguard init --force` replaces the whole skill
|
|
363
|
+
but does not refresh an already-listed managed section; use it only with
|
|
364
|
+
explicit overwrite approval and review the diff. No new automatic refresh
|
|
365
|
+
mechanism is introduced.
|
|
366
|
+
|
|
331
367
|
### Would this suite have caught the bugs that already escaped?
|
|
332
368
|
|
|
333
369
|
An injected fault is a fault somebody thought of. A bug that actually shipped
|
|
@@ -407,6 +443,20 @@ is missing. TestGuard itself still makes no network calls.
|
|
|
407
443
|
|
|
408
444
|
### Starting from zero claims
|
|
409
445
|
|
|
446
|
+
During gradual adoption, use `testguard probe --allow-empty` to skip a valid
|
|
447
|
+
empty claims file without blocking a push. It explicitly reports that no
|
|
448
|
+
verification was performed and writes no evidence; `--json` retains the
|
|
449
|
+
`no-claims` state. Missing or invalid claims files still fail. Run `gate`
|
|
450
|
+
separately to enforce changed-file coverage. For the pre-commit probe hook,
|
|
451
|
+
set `args: [--allow-empty]`; for the GitHub Action, set `allow-empty: 'true'`.
|
|
452
|
+
The default probe still exits 2 when no claims exist.
|
|
453
|
+
|
|
454
|
+
In a monorepo, a valid nested `testguard.claims.json` marks a separate project.
|
|
455
|
+
The parent gate reports its changed files as delegated and does not count them
|
|
456
|
+
as parent coverage. Run `gate` in each child project as well; a passing parent
|
|
457
|
+
gate does not establish that child coverage passed. Invalid child claims fail
|
|
458
|
+
evaluation instead of hiding files.
|
|
459
|
+
|
|
410
460
|
`gate` names the changed files that carry no claim, and stops there —
|
|
411
461
|
correctly, because stating a claim is a human act. But a project adopting this
|
|
412
462
|
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
@@ -663,12 +713,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
663
713
|
# GitHub Actions
|
|
664
714
|
- uses: actions/checkout@v4
|
|
665
715
|
with: { fetch-depth: 0 }
|
|
666
|
-
- uses: raccioly/testguard@v0.
|
|
716
|
+
- uses: raccioly/testguard@v0.16.0
|
|
667
717
|
with: { command: gate }
|
|
668
718
|
|
|
669
719
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
670
720
|
include:
|
|
671
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
721
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
672
722
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
673
723
|
```
|
|
674
724
|
|
|
@@ -941,19 +991,23 @@ most expensive claims
|
|
|
941
991
|
4m 58s TG-FAULT-EDIT-VISIBLE 1 fault · 6 runs
|
|
942
992
|
|
|
943
993
|
defender files, by the time of the runs that included them
|
|
944
|
-
(a
|
|
994
|
+
(a fault runs its selected defender set at once, so these overlap and do not sum to the total)
|
|
945
995
|
24m 3s test/probe.fixture.test.mjs 5 claims
|
|
946
996
|
|
|
947
|
-
shared defenders — each
|
|
997
|
+
shared defenders — each fault selects the file; durations include recorded baseline sharing
|
|
948
998
|
24m 3s test/probe.fixture.test.mjs
|
|
949
|
-
named by TG-ESCALATION-N-RUNS, TG-FAULT-EDIT-VISIBLE, …
|
|
999
|
+
named by TG-ESCALATION-N-RUNS/F1, TG-FAULT-EDIT-VISIBLE/F1, …
|
|
950
1000
|
```
|
|
951
1001
|
|
|
952
1002
|
Every number is read back out of the `durationMs` the probe already records.
|
|
953
1003
|
`--cost` re-measures nothing, spawns nothing, and costs the price of reading
|
|
954
1004
|
one JSON file.
|
|
955
1005
|
|
|
956
|
-
|
|
1006
|
+
Cost output also lists individual faults, their selected defenders and selection
|
|
1007
|
+
origin. Sharing is counted across distinct fault identities, including siblings
|
|
1008
|
+
within one claim; repeated confirmation runs alone are not sharing.
|
|
1009
|
+
|
|
1010
|
+
**The per-file figures overlap.** A run executes a fault's whole defender set
|
|
957
1011
|
at once, so the runner never says how much of it belonged to which file. A
|
|
958
1012
|
file's `ms` is the time of every run that *included* it: an upper bound on
|
|
959
1013
|
what removing it could save, which is the decision you are making. Only
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Gate semantics
|
|
2
2
|
|
|
3
|
+
## Fault-specific defender selection
|
|
4
|
+
|
|
5
|
+
A fault's `defendedBy`, when present, overrides its claim's declaration.
|
|
6
|
+
An explicit empty fault list requests discovery and never falls back to the
|
|
7
|
+
claim. Without a fault override, a non-empty claim list is inherited; otherwise
|
|
8
|
+
defenders are discovered. New evidence records `defenders.selectionSource` as
|
|
9
|
+
`fault`, `claim`, or `discovery`, alongside the actual requested and resolved
|
|
10
|
+
sets. Empty requests use discovery and carry `discovered: true`, including an
|
|
11
|
+
explicit empty fault override. Non-empty requests are never marked discovered.
|
|
12
|
+
Dirty-input preflight, runner ownership, mock-awareness, input hashing and
|
|
13
|
+
baseline caching all operate on each fault's selected defender set.
|
|
14
|
+
|
|
15
|
+
Changing requested globs, resolved files or selection origin invalidates verdict
|
|
16
|
+
reuse. Legacy evidence remains readable but cannot authorize a new fault-level
|
|
17
|
+
override. A strict resolved subset after a previous kill is reported by `claims`
|
|
18
|
+
as a narrowing warning and requires re-probing; it is not a new verdict.
|
|
19
|
+
Cost reports include per-fault recorded run durations and selection origin.
|
|
20
|
+
Shared defenders identify the actual fault records that selected each file,
|
|
21
|
+
including multiple faults within one claim. File costs remain overlapping upper
|
|
22
|
+
bounds, not additive shares. Existing total/per-claim budget ceilings are unchanged.
|
|
23
|
+
|
|
3
24
|
How a Guard-spec tool decides whether CI goes red. Shared by every tool that
|
|
4
25
|
adopts the spec; each tool may add stricter rules, never looser ones.
|
|
5
26
|
|
|
@@ -36,7 +57,11 @@ Rules that follow from the table:
|
|
|
36
57
|
that failed and then passed on retry (Playwright's `flaky`) is a
|
|
37
58
|
**non-green run** even when the runner exits 0, and a test the runner
|
|
38
59
|
reports as timed out (`timedOut`) is a timeout, never an assertion
|
|
39
|
-
failure.
|
|
60
|
+
failure. Jest-compatible timeout messages are recognized by native runner
|
|
61
|
+
error headers or structured timeout identity, not by words quoted in an
|
|
62
|
+
assertion, stack frame, or source excerpt. Structured assertion identity
|
|
63
|
+
does not hide a separate timeout in the same failed test.
|
|
64
|
+
Python is read the same way: an `AssertionError` and any other
|
|
40
65
|
exception raised by the test body both count, because both are the suite
|
|
41
66
|
rejecting the behaviour; a collection error — a module that will not
|
|
42
67
|
import, which is what a replacement that does not parse looks like from
|
|
@@ -172,6 +197,11 @@ Rules that follow from the table:
|
|
|
172
197
|
competing owned runners or a declared defender absent from every manifest
|
|
173
198
|
are precondition failures, never heuristic routing decisions.
|
|
174
199
|
|
|
200
|
+
Static string literals are conservative dependency candidates, not proof
|
|
201
|
+
of a filesystem dependency. Missing candidates and ordinary directories
|
|
202
|
+
(including `/` in URL bases and string separators) are ignored. An existing
|
|
203
|
+
candidate file outside the project, or a candidate symlink, is rejected.
|
|
204
|
+
|
|
175
205
|
14. **Barrel resolution follows symbols, not module reachability.** Automatic
|
|
176
206
|
discovery may traverse a re-export chain only for the binding the test
|
|
177
207
|
actually imports. Importing an unrelated symbol from a broad barrel does
|
|
@@ -494,10 +524,22 @@ Rules:
|
|
|
494
524
|
`unclaimed-changes` and its next action is to write the claim, before any
|
|
495
525
|
`unproven` finding is surfaced. The brief renders them first. The claim is
|
|
496
526
|
written before more code.
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
527
|
+
7. **A descendant claims file defines a separate project.** The first
|
|
528
|
+
descendant directory on a changed file's path with a valid, regular,
|
|
529
|
+
non-symlinked `testguard.claims.json` owns that subtree. Invalid or unreadable
|
|
530
|
+
markers fail evaluation; they cannot hide unclaimed files. Parent gate and
|
|
531
|
+
status report the delegation as optional `nested` entries with project and
|
|
532
|
+
file paths relative to the parent, and instruct the operator to run that
|
|
533
|
+
project's gate. Delegation is not coverage, an ignore reliance, or proof
|
|
534
|
+
that the child passed. Parent source-surface counts omit these subtrees.
|
|
535
|
+
Removing the marker restores parent evaluation. The accounting identity is
|
|
536
|
+
`changed = evaluated + excluded.length + sum(nested[].files.length)`;
|
|
537
|
+
`--strict` still fails a non-empty change with zero parent evaluation.
|
|
538
|
+
|
|
539
|
+
Exit codes: `0` every changed parent-owned source file is claimed or excused
|
|
540
|
+
(or nothing changed); delegated projects still require separate child gates.
|
|
541
|
+
`1` at least one uncovered file (or strict mode over an
|
|
542
|
+
all-excluded or all-delegated change); `2` the change cannot be evaluated (unresolvable
|
|
501
543
|
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
502
544
|
|
|
503
545
|
## Anchor preflight
|
|
@@ -548,3 +590,11 @@ evidence; they simply do not turn CI red.
|
|
|
548
590
|
A tool must never exit `0` because it had nothing to check. If the claims
|
|
549
591
|
file is empty or every claim is out of scope, that is reported explicitly and
|
|
550
592
|
the exit code is `2`.
|
|
593
|
+
|
|
594
|
+
`probe --allow-empty` is an explicit adoption exception for a valid claims
|
|
595
|
+
file containing zero claims, with no `--claim` selection. It exits `0`, says
|
|
596
|
+
that verification was skipped, and writes no evidence or baseline. Its JSON
|
|
597
|
+
output is the existing `no-claims` status document, without run metadata;
|
|
598
|
+
it never reports a clean measurement. Missing or invalid claims still fail,
|
|
599
|
+
and selected claims cannot be silently skipped. Run `gate` separately to
|
|
600
|
+
enforce coverage of changed files during adoption. Integrations must opt in.
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -28,6 +28,24 @@ const schemaFor = (kind) => {
|
|
|
28
28
|
return v;
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
+
const nestedErrors = (nested = [], prefix = '/nested', occupied = []) => {
|
|
32
|
+
const errors = [];
|
|
33
|
+
const projects = new Set();
|
|
34
|
+
const files = new Set(occupied);
|
|
35
|
+
for (const [i, entry] of nested.entries()) {
|
|
36
|
+
const p = `${prefix}/${i}`;
|
|
37
|
+
if (entry.project.split('/').some((part) => !part || part === '.' || part === '..') || entry.project.includes('\\')) errors.push({ path: `${p}/project`, message: 'nested project must be a canonical descendant directory' });
|
|
38
|
+
if (projects.has(entry.project)) errors.push({ path: `${p}/project`, message: 'nested project is duplicated' });
|
|
39
|
+
projects.add(entry.project);
|
|
40
|
+
for (const file of entry.files) {
|
|
41
|
+
if (!file.startsWith(`${entry.project}/`) || file.split('/').some((part) => !part || part === '.' || part === '..') || file.includes('\\')) errors.push({ path: `${p}/files`, message: 'nested file must remain beneath its canonical project directory' });
|
|
42
|
+
if (files.has(file)) errors.push({ path: `${p}/files`, message: 'nested file is counted in more than one bucket' });
|
|
43
|
+
files.add(file);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return errors;
|
|
47
|
+
};
|
|
48
|
+
|
|
31
49
|
// Rules JSON Schema cannot express. Each returns an array of {path, message}.
|
|
32
50
|
const semantic = {
|
|
33
51
|
claims(doc) {
|
|
@@ -128,6 +146,15 @@ const semantic = {
|
|
|
128
146
|
if (r.detail.undeclaredKillers && r.detail.reason !== 'killed-by-undeclared-tests') {
|
|
129
147
|
errors.push({ path: `${p}/detail/undeclaredKillers`, message: 'undeclaredKillers is only meaningful with reason killed-by-undeclared-tests' });
|
|
130
148
|
}
|
|
149
|
+
if (r.defenders.selectionSource) {
|
|
150
|
+
const empty = r.defenders.requested.length === 0;
|
|
151
|
+
if ((r.defenders.selectionSource === 'discovery' && !empty)
|
|
152
|
+
|| (r.defenders.selectionSource === 'claim' && empty)
|
|
153
|
+
|| (empty && r.defenders.discovered !== true)
|
|
154
|
+
|| (!empty && r.defenders.discovered === true)) {
|
|
155
|
+
errors.push({ path: `${p}/defenders/selectionSource`, message: 'defender selection origin, requested set and discovery flag disagree' });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
131
158
|
if (r.defenders.byRunner) {
|
|
132
159
|
const resolved = new Set(r.defenders.resolved);
|
|
133
160
|
for (const [runner, files] of Object.entries(r.defenders.byRunner)) for (const f of files) if (!resolved.has(f)) errors.push({ path: `${p}/defenders/byRunner/${runner}`, message: `${f} ran under ${runner} but is not a resolved defender` });
|
|
@@ -243,6 +270,11 @@ const semantic = {
|
|
|
243
270
|
|
|
244
271
|
status(doc) {
|
|
245
272
|
const errors = [];
|
|
273
|
+
if (doc.changes?.nested) {
|
|
274
|
+
errors.push(...nestedErrors(doc.changes.nested, '/changes/nested', doc.changes.uncovered.map((file) => file.file)));
|
|
275
|
+
const nestedCount = doc.changes.nested.reduce((count, entry) => count + entry.files.length, 0);
|
|
276
|
+
if (doc.changes.changed !== doc.changes.evaluated + doc.changes.excluded + nestedCount) errors.push({ path: '/changes/changed', message: 'changed must account for evaluated, excluded, and nested files' });
|
|
277
|
+
}
|
|
246
278
|
if (['write-test', 'review-fault-change', 'repair-fault'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
247
279
|
if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
|
|
248
280
|
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
@@ -311,8 +343,10 @@ const semantic = {
|
|
|
311
343
|
|
|
312
344
|
gate(doc) {
|
|
313
345
|
const errors = [];
|
|
346
|
+
const nestedCount = (doc.nested ?? []).reduce((count, entry) => count + entry.files.length, 0);
|
|
347
|
+
errors.push(...nestedErrors(doc.nested, '/nested', [...doc.covered, ...doc.uncovered, ...doc.excluded].map((entry) => entry.file)));
|
|
314
348
|
if (doc.evaluated !== doc.covered.length + doc.uncovered.length) errors.push({ path: '/evaluated', message: `evaluated (${doc.evaluated}) must equal covered (${doc.covered.length}) + uncovered (${doc.uncovered.length})` });
|
|
315
|
-
if (doc.changed !== doc.evaluated + doc.excluded.length) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length})` });
|
|
349
|
+
if (doc.changed !== doc.evaluated + doc.excluded.length + nestedCount) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length}) + nested (${nestedCount})` });
|
|
316
350
|
if (doc.uncovered.length > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'an uncovered changed file must exit 1; the gate never passes over unclaimed code' });
|
|
317
351
|
if (doc.uncovered.length === 0 && doc.exitCode === 1 && !(doc.strict && doc.changed > 0 && doc.evaluated === 0)) errors.push({ path: '/exitCode', message: 'exit 1 without uncovered files is only valid under --strict when a non-empty change evaluated nothing' });
|
|
318
352
|
doc.covered.forEach((c, i) => {
|
|
@@ -121,6 +121,12 @@
|
|
|
121
121
|
"faultClass": {
|
|
122
122
|
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
123
123
|
},
|
|
124
|
+
"defendedBy": {
|
|
125
|
+
"description": "Overrides this fault's claim-level defenders when present. An explicit empty list requests discovery; absence inherits the claim. Selection changes require re-probing, never assumed detection.",
|
|
126
|
+
"type": "array",
|
|
127
|
+
"items": { "type": "string", "minLength": 1 },
|
|
128
|
+
"uniqueItems": true
|
|
129
|
+
},
|
|
124
130
|
"file": {
|
|
125
131
|
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
126
132
|
},
|
|
@@ -4,6 +4,16 @@
|
|
|
4
4
|
"title": "claimspec v1 \u2014 shared definitions",
|
|
5
5
|
"description": "Definitions shared by every Guard-spec format. Referenced as urn:claimspec:v1:common#/$defs/<name>.",
|
|
6
6
|
"$defs": {
|
|
7
|
+
"nestedProject": {
|
|
8
|
+
"description": "Changed files delegated to a separately validated descendant TestGuard project. This is not coverage or evidence; run that project's gate separately.",
|
|
9
|
+
"type": "object",
|
|
10
|
+
"required": ["project", "files"],
|
|
11
|
+
"properties": {
|
|
12
|
+
"project": { "$ref": "#/$defs/repoPath" },
|
|
13
|
+
"files": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "#/$defs/repoPath" } }
|
|
14
|
+
},
|
|
15
|
+
"additionalProperties": false
|
|
16
|
+
},
|
|
7
17
|
"sha256": {
|
|
8
18
|
"type": "string",
|
|
9
19
|
"pattern": "^[a-f0-9]{64}$"
|
|
@@ -520,6 +520,10 @@
|
|
|
520
520
|
"nocover"
|
|
521
521
|
],
|
|
522
522
|
"properties": {
|
|
523
|
+
"selectionSource": {
|
|
524
|
+
"description": "Origin of the requested defender selection: explicit fault override, inherited non-empty claim declaration, or automatic discovery. Optional only for legacy evidence; a changed origin invalidates reuse.",
|
|
525
|
+
"enum": ["fault", "claim", "discovery"]
|
|
526
|
+
},
|
|
523
527
|
"requested": {
|
|
524
528
|
"type": "array",
|
|
525
529
|
"items": {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
18
|
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
19
19
|
"evaluated": { "description": "covered + uncovered: the changed files that are source and not excluded.", "type": "integer", "minimum": 0 },
|
|
20
|
+
"nested": { "description": "Optional descendant-project delegations; absence means none. These files are neither covered nor evaluated by the parent.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
20
21
|
"excluded": {
|
|
21
22
|
"type": "array",
|
|
22
23
|
"items": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
27
27
|
},
|
|
28
28
|
"state": {
|
|
29
|
-
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
29
|
+
"description": "no-claims: no claims file or a valid empty claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
30
30
|
"enum": [
|
|
31
31
|
"no-claims",
|
|
32
32
|
"unclaimed-changes",
|
|
@@ -195,6 +195,24 @@
|
|
|
195
195
|
"totalRuns": { "type": "integer", "minimum": 0 },
|
|
196
196
|
"reusedRecords": { "type": "integer", "minimum": 0 },
|
|
197
197
|
"records": { "type": "integer", "minimum": 0 },
|
|
198
|
+
"faults": {
|
|
199
|
+
"description": "Recorded costs and selected defenders per fault; optional for legacy reports.",
|
|
200
|
+
"type": "array",
|
|
201
|
+
"items": {
|
|
202
|
+
"type": "object",
|
|
203
|
+
"required": ["claimId", "faultId", "ms", "runs", "reused", "defenders"],
|
|
204
|
+
"properties": {
|
|
205
|
+
"claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
206
|
+
"faultId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
207
|
+
"ms": { "type": "number", "minimum": 0 },
|
|
208
|
+
"runs": { "type": "integer", "minimum": 0 },
|
|
209
|
+
"reused": { "type": "boolean" },
|
|
210
|
+
"selectionSource": { "enum": ["fault", "claim", "discovery"] },
|
|
211
|
+
"defenders": { "type": "array", "uniqueItems": true, "items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" } }
|
|
212
|
+
},
|
|
213
|
+
"additionalProperties": false
|
|
214
|
+
}
|
|
215
|
+
},
|
|
198
216
|
"claims": {
|
|
199
217
|
"type": "array",
|
|
200
218
|
"items": {
|
|
@@ -392,6 +410,7 @@
|
|
|
392
410
|
"expired"
|
|
393
411
|
],
|
|
394
412
|
"properties": {
|
|
413
|
+
"nested": { "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
395
414
|
"ref": {
|
|
396
415
|
"type": "string",
|
|
397
416
|
"minLength": 1,
|
|
@@ -482,6 +501,13 @@
|
|
|
482
501
|
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
483
502
|
"ms": { "type": "number", "minimum": 0 },
|
|
484
503
|
"runs": { "type": "integer", "minimum": 0 },
|
|
504
|
+
"faults": {
|
|
505
|
+
"description": "Distinct claimId/faultId identities selecting this file, not confirmation runs.",
|
|
506
|
+
"type": "array",
|
|
507
|
+
"minItems": 1,
|
|
508
|
+
"uniqueItems": true,
|
|
509
|
+
"items": { "type": "string", "minLength": 1 }
|
|
510
|
+
},
|
|
485
511
|
"claims": {
|
|
486
512
|
"type": "array",
|
|
487
513
|
"minItems": 1,
|
package/src/cli.mjs
CHANGED
|
@@ -75,6 +75,7 @@ probe
|
|
|
75
75
|
--in-place mutate the working tree instead of a scratch worktree
|
|
76
76
|
--no-escalate do not re-run survivors against the whole suite
|
|
77
77
|
--no-reuse re-probe claims whose inputs have not changed
|
|
78
|
+
--allow-empty adoption only: skip a valid empty claims file; writes no evidence; run gate separately
|
|
78
79
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
79
80
|
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
80
81
|
|
|
@@ -134,6 +135,27 @@ exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or uncl
|
|
|
134
135
|
|
|
135
136
|
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand, concerns: concernsCommand };
|
|
136
137
|
|
|
138
|
+
/** Derive command help from the same option descriptions as the overview. */
|
|
139
|
+
export function commandUsage(command) {
|
|
140
|
+
const lines = USAGE.split('\n');
|
|
141
|
+
const overview = lines.find((line) => line.startsWith(` testguard ${command} `)) ?? ` testguard ${command}`;
|
|
142
|
+
const sections = [];
|
|
143
|
+
let selected = false;
|
|
144
|
+
for (const line of lines) {
|
|
145
|
+
const heading = /^([a-z]+)(?:\s|$)/.exec(line)?.[1];
|
|
146
|
+
if (heading && Object.hasOwn(COMMANDS, heading)) selected = heading === command;
|
|
147
|
+
else if (line.startsWith('every command ') || line.startsWith('exit codes:')) selected = false;
|
|
148
|
+
if (selected) sections.push(line);
|
|
149
|
+
}
|
|
150
|
+
if (command === 'concerns') sections.push('concerns --concerns <path> read a custom concerns file; --json prints the validated inventory');
|
|
151
|
+
const examples = {
|
|
152
|
+
probe: 'probe . --confirm 3', gate: 'gate . --changed origin/main',
|
|
153
|
+
scaffold: 'scaffold src/example.mjs --json', replay: 'replay . --since HEAD~10..HEAD --max 3',
|
|
154
|
+
admit: 'admit test/example.test.mjs --claim EXAMPLE-001', sweep: 'sweep . --changed origin/main --cap 10',
|
|
155
|
+
};
|
|
156
|
+
return `${lines[0]}\n\n${overview}\n\n${sections.join('\n').trim()}\n\nCommon: --json, --help, --version\nExample: testguard ${examples[command] ?? `${command}${command === 'mcp' ? '' : ' .'}`}\n`;
|
|
157
|
+
}
|
|
158
|
+
|
|
137
159
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
138
160
|
let parsed;
|
|
139
161
|
try {
|
|
@@ -180,6 +202,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
180
202
|
'in-place': { type: 'boolean', default: false },
|
|
181
203
|
'no-escalate': { type: 'boolean', default: false },
|
|
182
204
|
'no-reuse': { type: 'boolean', default: false },
|
|
205
|
+
'allow-empty': { type: 'boolean', default: false },
|
|
183
206
|
quiet: { type: 'boolean', default: false },
|
|
184
207
|
json: { type: 'boolean', default: false },
|
|
185
208
|
text: { type: 'boolean', default: false },
|
|
@@ -202,9 +225,9 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
202
225
|
return 0;
|
|
203
226
|
}
|
|
204
227
|
const [command, dirArg] = positionals;
|
|
205
|
-
if (
|
|
206
|
-
io.out(USAGE);
|
|
207
|
-
return
|
|
228
|
+
if (!command) {
|
|
229
|
+
io.out(USAGE + "\nRun 'testguard <command> --help' for that command's options.");
|
|
230
|
+
return values.help ? 0 : 3;
|
|
208
231
|
}
|
|
209
232
|
const handler = COMMANDS[command];
|
|
210
233
|
if (!handler) {
|
|
@@ -212,6 +235,10 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
212
235
|
io.err(USAGE);
|
|
213
236
|
return 3;
|
|
214
237
|
}
|
|
238
|
+
if (values.help) {
|
|
239
|
+
io.out(commandUsage(command));
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
215
242
|
if (![...RUNNER_NAMES, 'auto'].includes(values.runner)) {
|
|
216
243
|
io.err(`--runner must be one of ${RUNNER_NAMES.join(', ')} or auto`);
|
|
217
244
|
return 3;
|
package/src/commands/admit.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { probe } from '../probe/probe.mjs';
|
|
|
5
5
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
9
|
import { PreconditionError } from '../probe/worktree.mjs';
|
|
9
10
|
import { hintFor } from '../brief/brief.mjs';
|
|
10
11
|
import { formatVerdict, PROVISIONAL_WARNING } from '../render.mjs';
|
|
@@ -62,11 +63,15 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
62
63
|
|
|
63
64
|
// The named test must be a defender of the claim, declared or discovered;
|
|
64
65
|
// otherwise `killed` below would be someone else's test doing the work.
|
|
65
|
-
const
|
|
66
|
-
|
|
66
|
+
const selections = faults.map((f) => {
|
|
67
|
+
const { requested } = defenderSelection(claim, f);
|
|
68
|
+
return { requested, resolved: requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file) };
|
|
69
|
+
});
|
|
70
|
+
const declared = selections.every((s) => s.requested.length) ? selections[0].requested : null;
|
|
71
|
+
const isDefender = selections.some((s) => s.resolved.includes(testRel));
|
|
67
72
|
if (!isDefender) {
|
|
68
73
|
io.err(declared
|
|
69
|
-
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...
|
|
74
|
+
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...declared, testRel].map((d) => JSON.stringify(d)).join(', ')}] on the selected fault or its inherited claim in testguard.claims.json, then run admit again.`
|
|
70
75
|
: `${testRel} does not import ${[...new Set(faults.map((f) => f.file))].join(' or ')}, so it cannot be discovered as a defender of ${claimId}. Import the module under test, or declare "defendedBy": ["${testRel}"] on the claim.`);
|
|
71
76
|
return 3;
|
|
72
77
|
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -10,6 +10,27 @@ import { costReport, renderCost } from '../probe/cost.mjs';
|
|
|
10
10
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
11
11
|
import { existsSync } from 'node:fs';
|
|
12
12
|
import { checkAnchors, renderAnchorChecks } from '../claims/anchors.mjs';
|
|
13
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
14
|
+
|
|
15
|
+
/** A selection warning is not a verdict: only a fresh probe can defend it. */
|
|
16
|
+
export function defenderNarrowing(projectDir, claims, records) {
|
|
17
|
+
const prior = new Map(records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]));
|
|
18
|
+
const warnings = [];
|
|
19
|
+
for (const c of claims.claims) {
|
|
20
|
+
const inherited = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [];
|
|
21
|
+
for (const f of c.faults) {
|
|
22
|
+
const record = prior.get(`${c.id}/${f.id}`);
|
|
23
|
+
if (!Object.hasOwn(f, 'defendedBy') || record?.verdict !== 'killed') continue;
|
|
24
|
+
const { requested } = defenderSelection(c, f);
|
|
25
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
26
|
+
if (record.defenders.selectionSource === 'fault' && JSON.stringify(record.defenders.requested) === JSON.stringify(requested) && JSON.stringify([...record.defenders.resolved].sort()) === JSON.stringify([...resolved].sort())) continue;
|
|
27
|
+
if (resolved.length < inherited.length && resolved.every((d) => inherited.includes(d))) {
|
|
28
|
+
warnings.push({ claimId: c.id, faultId: f.id, inherited, resolved });
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return warnings;
|
|
33
|
+
}
|
|
13
34
|
|
|
14
35
|
export async function claimsCommand({ projectDir, values, version }, io) {
|
|
15
36
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
@@ -28,14 +49,17 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
28
49
|
|
|
29
50
|
// Cost is read back out of evidence the probe already wrote; it never runs a test.
|
|
30
51
|
let cost;
|
|
52
|
+
const ev = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
53
|
+
const hasOverrides = claims.claims.some((c) => c.faults.some((f) => Object.hasOwn(f, 'defendedBy')));
|
|
54
|
+
const prior = (values.cost || hasOverrides) && existsSync(ev) ? readSpecDoc('evidence', ev) : undefined;
|
|
55
|
+
const narrowedDefenders = hasOverrides && prior ? defenderNarrowing(projectDir, claims, prior.records) : [];
|
|
31
56
|
if (values.cost) {
|
|
32
|
-
|
|
33
|
-
cost = existsSync(ev) ? costReport(readSpecDoc('evidence', ev).records) : undefined;
|
|
57
|
+
cost = prior ? costReport(prior.records) : undefined;
|
|
34
58
|
if (!cost && !values.json) io.err(`no evidence at ${ev} — run \`testguard probe\` first; cost is derived from the run durations it records`);
|
|
35
59
|
}
|
|
36
60
|
|
|
37
61
|
if (values.json) {
|
|
38
|
-
io.out(JSON.stringify({ path, claims, annotations, drift, ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
62
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, ...(hasOverrides ? { narrowedDefenders } : {}), ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
39
63
|
} else {
|
|
40
64
|
const annotated = new Set(drift.annotated);
|
|
41
65
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
@@ -45,7 +69,18 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
45
69
|
const declared = c.defendedBy?.length > 0;
|
|
46
70
|
const targets = [...new Set(c.faults.map((f) => f.file))];
|
|
47
71
|
let cover;
|
|
48
|
-
if (
|
|
72
|
+
if (c.faults.some((f) => Object.hasOwn(f, 'defendedBy'))) {
|
|
73
|
+
const selected = new Set();
|
|
74
|
+
for (const f of c.faults) {
|
|
75
|
+
const selection = defenderSelection(c, f);
|
|
76
|
+
const defenders = selection.requested.length ? resolveDefenders(projectDir, selection.requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
77
|
+
defenders.forEach((d) => selected.add(d));
|
|
78
|
+
const m = classifyDefenders(projectDir, f.file, defenders);
|
|
79
|
+
for (const s of m.signals) signalLines.push({ claim: c.id, target: f.file, ...s });
|
|
80
|
+
io.out(` ${c.id}/${f.id}: ${selection.selectionSource}${selection.requested.length ? '' : ' (discovery)'} — ${defenders.join(', ') || 'NO DEFENDER'}`);
|
|
81
|
+
}
|
|
82
|
+
cover = `${selected.size} selected defender${selected.size === 1 ? '' : 's'}`;
|
|
83
|
+
} else if (declared) {
|
|
49
84
|
const defenders = resolveDefenders(projectDir, c.defendedBy);
|
|
50
85
|
const mocking = new Set();
|
|
51
86
|
for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
@@ -70,6 +105,7 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
70
105
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
71
106
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
|
72
107
|
for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
|
|
108
|
+
for (const w of narrowedDefenders) io.out(`NARROWED-DEFENDERS ${w.claimId}/${w.faultId} selects ${w.resolved.length} of ${w.inherited.length} inherited defenders after a prior kill — re-probe; the old kill does not prove the narrower selection.`);
|
|
73
109
|
if (removed) {
|
|
74
110
|
io.out('');
|
|
75
111
|
io.out(renderRemoved(removed));
|
package/src/commands/probe.mjs
CHANGED
|
@@ -57,6 +57,19 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
57
57
|
const stageOut = stageReporter(progress, (s) => progressTo.write(s));
|
|
58
58
|
const claims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
|
|
59
59
|
if (claims.claims.length === 0) {
|
|
60
|
+
if (values['allow-empty'] && values.claim === undefined) {
|
|
61
|
+
io.err('claims file declares no claims; verification skipped (--allow-empty); run gate to check changed-file coverage');
|
|
62
|
+
if (values.json) {
|
|
63
|
+
const doc = computeStatus({ projectDir, toolVersion: version, paths: {
|
|
64
|
+
claims: values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir),
|
|
65
|
+
evidence: evidencePath(projectDir), provisional: provisionalEvidencePath(projectDir), baseline: baselinePath(projectDir),
|
|
66
|
+
} });
|
|
67
|
+
const result = validate('status', doc);
|
|
68
|
+
if (!result.ok) throw new Error(`empty probe JSON document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
69
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
70
|
+
}
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
60
73
|
io.err('claims file declares no claims; nothing to verify');
|
|
61
74
|
return 2;
|
|
62
75
|
}
|
package/src/gate/changed.mjs
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { existsSync, realpathSync } from 'node:fs';
|
|
1
|
+
import { existsSync, realpathSync, lstatSync } from 'node:fs';
|
|
2
2
|
import { extname, join, relative, resolve, dirname } from 'node:path';
|
|
3
3
|
import { git, repoRoot, headSha, GitError } from '../git.mjs';
|
|
4
|
-
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath, ClaimsError } from '../claims/load.mjs';
|
|
5
5
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
9
|
import { IS_PY_TEST } from '../probe/pyimports.mjs';
|
|
9
10
|
import { globToRegExp } from '../util/glob.mjs';
|
|
10
11
|
|
|
@@ -49,6 +50,33 @@ export const isDefaultExcluded = (file) => DEFAULT_EXCLUDE_RES.some((re) => re.t
|
|
|
49
50
|
|
|
50
51
|
export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
|
|
51
52
|
|
|
53
|
+
/** Cache validated descendant boundaries for one measurement, never across runs. */
|
|
54
|
+
export function nestedProjectResolver(projectDir) {
|
|
55
|
+
const root = realpathSync(resolve(projectDir));
|
|
56
|
+
const markers = new Map();
|
|
57
|
+
return (file) => {
|
|
58
|
+
const parts = file.split('/').slice(0, -1);
|
|
59
|
+
for (let i = 1; i <= parts.length; i++) {
|
|
60
|
+
const project = parts.slice(0, i).join('/');
|
|
61
|
+
if (!markers.has(project)) {
|
|
62
|
+
const marker = join(root, project, 'testguard.claims.json');
|
|
63
|
+
let stat;
|
|
64
|
+
try { stat = lstatSync(marker); }
|
|
65
|
+
catch (error) {
|
|
66
|
+
if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') throw new ClaimsError(`cannot inspect nested claims file ${marker}: ${error.message}`);
|
|
67
|
+
}
|
|
68
|
+
if (stat) {
|
|
69
|
+
if (!stat.isFile() || stat.isSymbolicLink() || realpathSync(marker) !== marker) throw new ClaimsError(`nested claims file must be a regular file without symlink traversal: ${marker}`);
|
|
70
|
+
loadClaims(marker);
|
|
71
|
+
}
|
|
72
|
+
markers.set(project, Boolean(stat));
|
|
73
|
+
}
|
|
74
|
+
if (markers.get(project)) return project;
|
|
75
|
+
}
|
|
76
|
+
return null;
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
52
80
|
/**
|
|
53
81
|
* The reference the change is measured against. Environment overrides and CI
|
|
54
82
|
* bases are handled here; local Git discovery is separate because it needs a
|
|
@@ -201,7 +229,10 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
201
229
|
const defenderFiles = new Map(); // test file → claimIds
|
|
202
230
|
for (const c of claims.claims) {
|
|
203
231
|
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
204
|
-
const defenders =
|
|
232
|
+
const defenders = [...new Set(c.faults.flatMap((f) => {
|
|
233
|
+
const { requested } = defenderSelection(c, f);
|
|
234
|
+
return requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file);
|
|
235
|
+
}))];
|
|
205
236
|
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
206
237
|
}
|
|
207
238
|
|
|
@@ -212,7 +243,15 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
212
243
|
const excluded = [];
|
|
213
244
|
const covered = [];
|
|
214
245
|
const uncovered = [];
|
|
246
|
+
const nested = new Map();
|
|
247
|
+
const ownerOf = nestedProjectResolver(projectDir);
|
|
215
248
|
for (const file of changed) {
|
|
249
|
+
const owner = ownerOf(file);
|
|
250
|
+
if (owner) {
|
|
251
|
+
if (!nested.has(owner)) nested.set(owner, []);
|
|
252
|
+
nested.get(owner).push(file);
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
216
255
|
if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
|
|
217
256
|
const ux = userExcludes.find((x) => x.re.test(file));
|
|
218
257
|
if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
|
|
@@ -256,6 +295,7 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
256
295
|
changed: changed.length,
|
|
257
296
|
evaluated,
|
|
258
297
|
excluded,
|
|
298
|
+
...(nested.size ? { nested: [...nested].map(([project, files]) => ({ project, files })) } : {}),
|
|
259
299
|
covered,
|
|
260
300
|
uncovered,
|
|
261
301
|
reliedOn,
|
|
@@ -271,7 +311,8 @@ export function renderGate(doc) {
|
|
|
271
311
|
for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
|
|
272
312
|
for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
273
313
|
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
|
|
274
|
-
|
|
314
|
+
for (const n of doc.nested ?? []) lines.push(`delegated ${n.files.length} file(s) belong to the TestGuard project at ${n.project}/; run testguard gate ${n.project} --changed ${doc.ref}${doc.includeDirty ? ' --include-dirty' : ''}; child coverage has not been evaluated here`);
|
|
315
|
+
if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded or delegated (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
|
|
275
316
|
if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
|
|
276
317
|
if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
|
|
277
318
|
return lines.join('\n');
|
package/src/init/init.mjs
CHANGED
|
@@ -56,6 +56,12 @@ Before writing or changing tests, run \`testguard status --json [dir]\` and foll
|
|
|
56
56
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
57
57
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
58
58
|
|
|
59
|
+
Once per AI session, identify the exact CLI used (project-local first, then repository-root, then PATH) and read its \`--version\`.
|
|
60
|
+
If network policy permits, check \`npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000\` once; this is optional AI advice, not a CLI or hook network call.
|
|
61
|
+
Compare stable versions numerically by SemVer; do not downgrade or replace a prerelease/development checkout. If a newer stable version exists, suggest it and ask before updating, respecting pins and the original package manager.
|
|
62
|
+
Never install automatically or alter lockfiles/CI pins without approval. On an unavailable check, continue without claiming up-to-date; keep advice separate from evidence, verdicts and exit codes. Do not repeat a declined suggestion this session.
|
|
63
|
+
Existing copied skills and sections are preserved by ordinary init; review and merge template changes manually, or use \`init --force\` only with approval to overwrite the skill.
|
|
64
|
+
|
|
59
65
|
${lines.join('\n')}
|
|
60
66
|
${AGENTS_END}`;
|
|
61
67
|
}
|
|
@@ -9,6 +9,49 @@ TestGuard injects the faults a project's claims forbid and reports every one
|
|
|
9
9
|
the tests fail to detect. It is a **claim verifier**, not a test generator.
|
|
10
10
|
You operate it through one loop and one source of truth.
|
|
11
11
|
|
|
12
|
+
## Update advice (once per session, never an automatic install)
|
|
13
|
+
|
|
14
|
+
Before the operating loop, identify the CLI you will actually run: prefer the
|
|
15
|
+
project's installed `node_modules/.bin/testguard`, then the repository-root
|
|
16
|
+
install for a nested project, then `testguard` on PATH. Run that exact CLI with
|
|
17
|
+
`--version`, and use the same executable for verification. Do not use `npx` to
|
|
18
|
+
discover a version: it can fetch or install a different tool. When developing
|
|
19
|
+
TestGuard itself, use `node cli/testguard.mjs`; do not replace a source checkout
|
|
20
|
+
with a published version to make validation pass.
|
|
21
|
+
|
|
22
|
+
If the user's network policy permits registry access, make one bounded,
|
|
23
|
+
read-only check per session (not per command or project):
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Compare valid stable versions by numeric SemVer major, minor, then patch, not
|
|
30
|
+
lexicographically and not merely for inequality. Do not recommend a downgrade
|
|
31
|
+
or replacing a prerelease/development checkout. If the registry result is
|
|
32
|
+
missing, malformed, unavailable, or network access is prohibited, continue the
|
|
33
|
+
operating loop without claiming the installation is up to date. No retries or
|
|
34
|
+
repeated prompts after the user declines within this session.
|
|
35
|
+
|
|
36
|
+
For a newer stable version, say: "Using TestGuard X from <resolved executable>;
|
|
37
|
+
Y is available. Would you like to update?" Respect project pins, compatibility,
|
|
38
|
+
company policy and the original package manager; a registry's `latest` tag is
|
|
39
|
+
not proof that an upgrade is safe. Never install automatically, modify a
|
|
40
|
+
lockfile or CI pin, or switch between local and global installs without approval.
|
|
41
|
+
After an approved update, verify the same executable's version again.
|
|
42
|
+
|
|
43
|
+
Keep this advice in the AI conversation, separate from CLI JSON, evidence,
|
|
44
|
+
verdicts and exit codes. The CLI and session-start hook remain offline; the AI
|
|
45
|
+
performs this optional check, not TestGuard. Instructions cannot guarantee that
|
|
46
|
+
every AI checks for updates.
|
|
47
|
+
|
|
48
|
+
Package upgrades do not refresh an existing copied skill. Preserve customized
|
|
49
|
+
instructions: compare this file with the upgraded package's
|
|
50
|
+
`src/init/templates/SKILL.md` and merge approved changes. Use `testguard init
|
|
51
|
+
--force` only with explicit permission to replace the whole skill; review the
|
|
52
|
+
diff afterward. An ordinary `init` preserves existing skills and managed
|
|
53
|
+
sections already listing this project.
|
|
54
|
+
|
|
12
55
|
## The one source of truth
|
|
13
56
|
|
|
14
57
|
```bash
|
|
@@ -20,7 +63,7 @@ which files exist. `state` is one of:
|
|
|
20
63
|
|
|
21
64
|
| state | meaning | you do |
|
|
22
65
|
|---|---|---|
|
|
23
|
-
| `no-claims` |
|
|
66
|
+
| `no-claims` | missing or empty `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with the intended guarantee from a spec, ticket, or past bug; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
67
|
| `unclaimed-changes` | files you changed carry no claim (when a reference is explicit, supplied by CI, or safely inferred from local Git's remote default/upstream metadata) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
25
68
|
| `invalid-anchors` | an exact fault anchor moved or became ambiguous | run `testguard claims --check-anchors`; repair `next.target` in the claims file without changing what the fault means, then re-probe. Never guess or auto-fix an anchor. |
|
|
26
69
|
| `unprobed` | claims never probed | `testguard probe` |
|
|
@@ -109,11 +109,20 @@ export function subjectOf(fault, sha256) {
|
|
|
109
109
|
* The cost of re-measuring is a run; the cost of the other direction is a
|
|
110
110
|
* verdict nobody measured.
|
|
111
111
|
*/
|
|
112
|
+
/** Select declarations without resolving globs or running discovery. */
|
|
113
|
+
export function defenderSelection(claim, fault) {
|
|
114
|
+
if (Object.hasOwn(fault, 'defendedBy')) return { requested: [...fault.defendedBy], selectionSource: 'fault' };
|
|
115
|
+
if (claim.defendedBy?.length) return { requested: [...claim.defendedBy], selectionSource: 'claim' };
|
|
116
|
+
return { requested: [], selectionSource: 'discovery' };
|
|
117
|
+
}
|
|
118
|
+
|
|
112
119
|
export function isReusable(prior, current) {
|
|
113
120
|
if (!prior || !prior.subject?.contentHash) return false;
|
|
114
121
|
// A record that exists because the probe threw is a statement about the run,
|
|
115
122
|
// not about the code. Reusing it would make one disturbed run permanent.
|
|
116
123
|
if (prior.detail?.reason === 'probe-error') return false;
|
|
124
|
+
const origin = prior.defenders.selectionSource ?? (prior.defenders.requested.length ? 'claim' : 'discovery');
|
|
125
|
+
if (current.selectionSource && origin !== current.selectionSource) return false;
|
|
117
126
|
const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
|
|
118
127
|
return prior.inputs.targetHash === current.inputs.targetHash
|
|
119
128
|
&& same(prior.inputs.defenderHashes, current.inputs.defenderHashes)
|
package/src/probe/cost.mjs
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* already recorded. Nothing re-measures, nothing spawns, nothing guesses: a
|
|
13
13
|
* cost report costs the price of reading one JSON file.
|
|
14
14
|
*
|
|
15
|
-
* ATTRIBUTION, HONESTLY. Runs execute a
|
|
15
|
+
* ATTRIBUTION, HONESTLY. Runs execute a fault's selected defender set together,
|
|
16
16
|
* so the runner never says how much of a run belonged to which file. Per-file
|
|
17
17
|
* `ms` is therefore the time of every run that *included* that file: an upper
|
|
18
18
|
* bound on what removing it could save, and, when a claim names several
|
|
@@ -70,15 +70,16 @@ export function defenderCosts(records) {
|
|
|
70
70
|
for (const r of records) {
|
|
71
71
|
const c = recordCost(r);
|
|
72
72
|
for (const file of r.defenders?.resolved ?? []) {
|
|
73
|
-
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set() };
|
|
73
|
+
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set(), faults: new Set() };
|
|
74
74
|
e.ms += c.ms;
|
|
75
75
|
e.runs += c.runs;
|
|
76
76
|
e.claims.add(r.claim.id);
|
|
77
|
+
e.faults.add(`${r.claim.id}/${r.subject.id}`);
|
|
77
78
|
by.set(file, e);
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
return [...by.values()]
|
|
81
|
-
.map((e) => ({ ...e, claims: [...e.claims].sort() }))
|
|
82
|
+
.map((e) => ({ ...e, claims: [...e.claims].sort(), faults: [...e.faults].sort() }))
|
|
82
83
|
.sort((a, b) => b.ms - a.ms || a.file.localeCompare(b.file));
|
|
83
84
|
}
|
|
84
85
|
|
|
@@ -86,7 +87,7 @@ export function defenderCosts(records) {
|
|
|
86
87
|
* The whole cost model for one evidence document.
|
|
87
88
|
*
|
|
88
89
|
* `sharedDefenders` is the actionable part: a file named by more than one
|
|
89
|
-
*
|
|
90
|
+
* fault identity, including siblings within one claim. Splitting the
|
|
90
91
|
* behaviour it defends into a unit test that runs in milliseconds is what
|
|
91
92
|
* turns a twenty-four-minute gate into a one-minute one, and until a report
|
|
92
93
|
* names the file nobody knows which test to split.
|
|
@@ -100,8 +101,10 @@ export function costReport(records) {
|
|
|
100
101
|
reusedRecords: records.filter((r) => r.reusedFrom).length,
|
|
101
102
|
records: records.length,
|
|
102
103
|
claims,
|
|
104
|
+
faults: records.map((r) => ({ claimId: r.claim.id, faultId: r.subject.id, ...recordCost(r), defenders: [...(r.defenders?.resolved ?? [])], ...(r.defenders?.selectionSource ? { selectionSource: r.defenders.selectionSource } : {}) }))
|
|
105
|
+
.sort((a, b) => b.ms - a.ms || a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId)),
|
|
103
106
|
defenders,
|
|
104
|
-
sharedDefenders: defenders.filter((d) => d.
|
|
107
|
+
sharedDefenders: defenders.filter((d) => d.faults.length > 1),
|
|
105
108
|
};
|
|
106
109
|
}
|
|
107
110
|
|
|
@@ -123,19 +126,24 @@ export function renderCost(report, { limit = 10 } = {}) {
|
|
|
123
126
|
out.push(` ${secs(c.ms).padStart(7)} ${c.claimId.padEnd(34)} ${c.faults} fault${c.faults === 1 ? ' ' : 's'} · ${c.runs} runs${c.reused ? ` · ${c.reused} reused` : ''}`);
|
|
124
127
|
}
|
|
125
128
|
if (report.claims.length > limit) out.push(` … ${report.claims.length - limit} more`);
|
|
129
|
+
if (report.faults?.length) {
|
|
130
|
+
out.push('', 'most expensive faults');
|
|
131
|
+
for (const f of report.faults.slice(0, limit)) out.push(` ${secs(f.ms).padStart(7)} ${f.claimId}/${f.faultId} · ${f.runs} runs${f.selectionSource ? ` · ${f.selectionSource}` : ''} · ${f.defenders.join(', ') || 'no defenders'}`);
|
|
132
|
+
if (report.faults.length > limit) out.push(` … ${report.faults.length - limit} more`);
|
|
133
|
+
}
|
|
126
134
|
out.push('');
|
|
127
135
|
out.push('defender files, by the time of the runs that included them');
|
|
128
|
-
out.push(' (a
|
|
136
|
+
out.push(' (a fault runs its selected defender set at once, so these overlap and do not sum to the total)');
|
|
129
137
|
for (const d of report.defenders.slice(0, limit)) {
|
|
130
138
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file.padEnd(44)} ${d.claims.length} claim${d.claims.length === 1 ? '' : 's'}`);
|
|
131
139
|
}
|
|
132
140
|
if (report.defenders.length > limit) out.push(` … ${report.defenders.length - limit} more`);
|
|
133
141
|
if (report.sharedDefenders.length) {
|
|
134
142
|
out.push('');
|
|
135
|
-
out.push('shared defenders — each
|
|
143
|
+
out.push('shared defenders — each fault selects the file; durations include recorded baseline sharing');
|
|
136
144
|
for (const d of report.sharedDefenders.slice(0, limit)) {
|
|
137
145
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file}`);
|
|
138
|
-
out.push(` named by ${d.
|
|
146
|
+
out.push(` named by ${d.faults.join(', ')}`);
|
|
139
147
|
}
|
|
140
148
|
out.push('');
|
|
141
149
|
out.push(' To make a probe cheaper, split the behaviour a shared defender proves into a');
|
package/src/probe/probe.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import { selectRunner, discoverRunnerManifest, RUNNERS, OWNED_RUNNERS, partition
|
|
|
13
13
|
import { classify, shouldStopEarly } from './classify.mjs';
|
|
14
14
|
import { blastRadius, rank } from './rank.mjs';
|
|
15
15
|
import { classifyIndependence } from './independence.mjs';
|
|
16
|
-
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable } from './attribution.mjs';
|
|
16
|
+
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
|
|
17
17
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
18
18
|
import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
19
19
|
import { persistenceSignalsFor } from '../supply/persistence.mjs';
|
|
@@ -154,11 +154,14 @@ export async function probe({
|
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
for (const claim of scopedClaims) {
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
157
|
+
for (const fault of claim.faults) {
|
|
158
|
+
const { requested } = defenderSelection(claim, fault);
|
|
159
|
+
const discovered = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, configuredFiles);
|
|
160
|
+
const declared = requested.length ? resolveDefenders(projectDir, requested) : discovered.canDetect;
|
|
161
|
+
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
162
|
+
for (const dependency of discovered?.dependencies ?? []) watched.add(relative(root, join(projectDir, dependency)));
|
|
163
|
+
for (const g of requested) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
164
|
+
}
|
|
162
165
|
}
|
|
163
166
|
for (const config of [
|
|
164
167
|
'package.json', 'pnpm-workspace.yaml', 'pnpm-workspace.yml', 'lerna.json', 'nx.json', 'turbo.json', 'rush.json', 'workspace.json',
|
|
@@ -202,7 +205,7 @@ export async function probe({
|
|
|
202
205
|
commandBudget?.assertOpen();
|
|
203
206
|
if (sel.error) {
|
|
204
207
|
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
|
|
205
|
-
(mode === 'worktree' ? '
|
|
208
|
+
(mode === 'worktree' ? 'If dependencies are missing, pass --node-modules <path>, or run with --in-place. Otherwise fix the discovery error above.' : 'If dependencies are missing, install them first. Otherwise fix the discovery error above.'));
|
|
206
209
|
}
|
|
207
210
|
runner = sel.runner;
|
|
208
211
|
runnerVersion = sel.version;
|
|
@@ -315,20 +318,21 @@ export async function probe({
|
|
|
315
318
|
|
|
316
319
|
for (const claim of scopedClaims) {
|
|
317
320
|
commandBudget?.assertOpen();
|
|
318
|
-
const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
319
|
-
if (declared && !commandTemplate) {
|
|
320
|
-
const excluded = declared.filter((file) => !ownerByFile.has(file));
|
|
321
|
-
for (const file of excluded) {
|
|
322
|
-
const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
|
|
323
|
-
const check = expectedOwner && ownedChecked.get(expectedOwner);
|
|
324
|
-
if (expectedOwner && check && !check.ok) {
|
|
325
|
-
throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
if (excluded.length) throw new PreconditionError(`${claim.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
|
|
329
|
-
}
|
|
330
321
|
for (const fault of claim.faults) {
|
|
331
322
|
commandBudget?.assertOpen();
|
|
323
|
+
const { requested } = defenderSelection(claim, fault);
|
|
324
|
+
const declared = requested.length ? resolveDefenders(iso.projectDir, requested) : null;
|
|
325
|
+
if (declared && !commandTemplate) {
|
|
326
|
+
const excluded = declared.filter((file) => !ownerByFile.has(file));
|
|
327
|
+
for (const file of excluded) {
|
|
328
|
+
const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
|
|
329
|
+
const check = expectedOwner && ownedChecked.get(expectedOwner);
|
|
330
|
+
if (expectedOwner && check && !check.ok) {
|
|
331
|
+
throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (excluded.length) throw new PreconditionError(`${claim.id}/${fault.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
|
|
335
|
+
}
|
|
332
336
|
// Mock-awareness: a discovered file that mocks the target is not a
|
|
333
337
|
// defender; a declared one that mocks it stays (the author named it)
|
|
334
338
|
// but is listed, because a declared defender that mocks the subject
|
|
@@ -413,7 +417,7 @@ export function errorRecord({ claim, fault, defenders, discovered, byRunner, moc
|
|
|
413
417
|
subject: subjectOf(fault, sha256),
|
|
414
418
|
verdict: 'unverifiable',
|
|
415
419
|
detail: { baselineRuns: [], probeRuns: [], reason: 'probe-error', message },
|
|
416
|
-
defenders: {
|
|
420
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
|
|
417
421
|
inputs,
|
|
418
422
|
};
|
|
419
423
|
}
|
|
@@ -428,7 +432,7 @@ export function discoveryIndeterminateRecord({ claim, fault, defenders, discover
|
|
|
428
432
|
subject: subjectOf(fault, sha256),
|
|
429
433
|
verdict: 'unverifiable',
|
|
430
434
|
detail: { baselineRuns: [], probeRuns: [], reason: 'defender-discovery-indeterminate', message },
|
|
431
|
-
defenders: {
|
|
435
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: false, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
|
|
432
436
|
inputs,
|
|
433
437
|
};
|
|
434
438
|
}
|
|
@@ -516,8 +520,8 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
516
520
|
const subject = subjectOf(fault, sha256);
|
|
517
521
|
|
|
518
522
|
// Same source, same defenders, same N, same fault: the verdict cannot have changed.
|
|
519
|
-
if (isReusable(prior, { inputs,
|
|
520
|
-
return { ...prior, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
523
|
+
if (isReusable(prior, { inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
524
|
+
return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
521
525
|
}
|
|
522
526
|
|
|
523
527
|
const detail = { baselineRuns: [], probeRuns: [] };
|
|
@@ -655,7 +659,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
655
659
|
subject,
|
|
656
660
|
verdict,
|
|
657
661
|
detail,
|
|
658
|
-
defenders: {
|
|
662
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(allSignals.length ? { signals: allSignals } : {}) },
|
|
659
663
|
inputs,
|
|
660
664
|
rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast, independence: detail.independence?.class }),
|
|
661
665
|
};
|
|
@@ -275,7 +275,6 @@ const inside = (root, path) => {
|
|
|
275
275
|
const relativeConfigPath = (root, path) => relative(root, path).split(sep).join('/');
|
|
276
276
|
|
|
277
277
|
const existingConfigCandidate = (root, path) => {
|
|
278
|
-
if (!inside(root, path)) throw new DiscoveryError(`discovery config dependency resolves outside the project: ${path}`);
|
|
279
278
|
let stat;
|
|
280
279
|
try {
|
|
281
280
|
stat = lstatSync(path);
|
|
@@ -285,6 +284,7 @@ const existingConfigCandidate = (root, path) => {
|
|
|
285
284
|
}
|
|
286
285
|
if (stat.isSymbolicLink()) throw new DiscoveryError(`discovery config dependency is not readable: symbolic links are not allowed`);
|
|
287
286
|
if (!stat.isFile()) return null;
|
|
287
|
+
if (!inside(root, path)) throw new DiscoveryError(`discovery config dependency resolves outside the project: ${path}`);
|
|
288
288
|
return path;
|
|
289
289
|
};
|
|
290
290
|
|
|
@@ -305,7 +305,9 @@ const resolveConfigReference = (root, fromPath, rawReference) => {
|
|
|
305
305
|
|
|
306
306
|
const resolved = new Set();
|
|
307
307
|
for (const base of bases) {
|
|
308
|
-
|
|
308
|
+
// Strings in executable configs also describe URLs and separators. Only
|
|
309
|
+
// existing file candidates are dependencies; the boundary check follows
|
|
310
|
+
// that distinction, while symlinks remain refused even when dangling.
|
|
309
311
|
const candidates = extname(base)
|
|
310
312
|
? [base]
|
|
311
313
|
: [base, ...RESOLVABLE_CONFIG_EXTENSIONS.map((extension) => `${base}${extension}`),
|
|
@@ -184,6 +184,21 @@ export function firstInformativeLine(message) {
|
|
|
184
184
|
return lines.find((l) => /error|syntax|unexpected|expected .+ but found|cannot find|failed to parse|transform failed/i.test(l) && !/^●/.test(l)) ?? lines[0] ?? '';
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
+
function isRunnerTimeout(test) {
|
|
188
|
+
if (test.failureType === 'testTimeoutFailure') return true;
|
|
189
|
+
const details = test.failureDetails ?? [];
|
|
190
|
+
if (details.some((error) => error?.failureType === 'testTimeoutFailure')) return true;
|
|
191
|
+
return (test.failureMessages ?? []).some((message, index) => {
|
|
192
|
+
const error = details[index];
|
|
193
|
+
// Assertion text may quote a timeout header verbatim. Inspect only the
|
|
194
|
+
// error header, never stack frames, source excerpts, or expected values.
|
|
195
|
+
if (error?.name === 'AssertionError' || error?.code === 'ERR_ASSERTION' || error?.operator !== undefined) return false;
|
|
196
|
+
const header = String(message).replace(/\x1b\[[0-9;]*m/g, '').trimStart().split('\n')[0].trimEnd();
|
|
197
|
+
return /^(?:Error:\s*)?(?:thrown:\s*")?(?:Exceeded timeout of \d+(?:\.\d+)?\s*ms for a (?:test|hook)\b|(?:Test|Hook) timed out in \d+(?:\.\d+)?ms\b|test timed out after \d+(?:\.\d+)?ms\b)/i.test(header)
|
|
198
|
+
|| /^(?:(?:Error|AroundHookSetupError|AroundHookTeardownError):\s*)?The (?:setup|teardown) phase of "around(?:Each|All)" hook timed out after \d+(?:\.\d+)?ms\b/.test(header);
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
187
202
|
export function parseReport(report, durationMs) {
|
|
188
203
|
// A report that parsed as JSON but carries no `testResults` is not a suite
|
|
189
204
|
// that ran zero tests — it is a report of a shape this parser does not
|
|
@@ -195,7 +210,7 @@ export function parseReport(report, durationMs) {
|
|
|
195
210
|
const files = report.testResults ?? [];
|
|
196
211
|
const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
|
|
197
212
|
const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
|
|
198
|
-
const timeouts = results.filter(
|
|
213
|
+
const timeouts = results.filter(isRunnerTimeout).length;
|
|
199
214
|
const failedTests = results.map((t) => t.id);
|
|
200
215
|
const assertionFailures = results.length - timeouts;
|
|
201
216
|
const tests = { total: report.numTotalTests ?? 0, passed: report.numPassedTests ?? 0, failed: report.numFailedTests ?? 0 };
|
package/src/status/status.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { gate } from '../baseline/baseline.mjs';
|
|
|
6
6
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
7
7
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
8
8
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
9
10
|
import { coAuthorshipWarning, sortForReport } from '../render.mjs';
|
|
10
11
|
import { computeChangedGate, defaultIgnorePath } from '../gate/changed.mjs';
|
|
11
12
|
import { headSha, isAncestor } from '../git.mjs';
|
|
@@ -64,6 +65,7 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
64
65
|
if (ref) {
|
|
65
66
|
const g = computeChangedGate({ projectDir, ref, includeDirty, toolVersion });
|
|
66
67
|
changes = { ref: g.ref, base: g.base, includeDirty: g.includeDirty, changed: g.changed, evaluated: g.evaluated, excluded: g.excluded.length, uncovered: g.uncovered, reliedOn: g.reliedOn, expired: g.expired };
|
|
68
|
+
if (g.nested?.length) changes.nested = g.nested;
|
|
67
69
|
doc.changes = changes;
|
|
68
70
|
if (existsSync(defaultIgnorePath(projectDir))) doc.paths.ignore = rel(defaultIgnorePath(projectDir));
|
|
69
71
|
}
|
|
@@ -90,6 +92,10 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
90
92
|
doc.counts.claims = claims.claims.length;
|
|
91
93
|
doc.counts.faults = claims.claims.reduce((n, c) => n + c.faults.length, 0);
|
|
92
94
|
doc.surface = computeClaimedSurface({ projectDir, claims });
|
|
95
|
+
if (claims.claims.length === 0 && !changes?.uncovered.length) {
|
|
96
|
+
doc.next = { action: 'scaffold', command: 'testguard scaffold <source-file>', why: 'No claims have been declared; no verification was performed. State a claim before probing.' };
|
|
97
|
+
return doc;
|
|
98
|
+
}
|
|
93
99
|
const faultIndex = new Map();
|
|
94
100
|
for (const c of claims.claims) for (const f of c.faults) faultIndex.set(`${c.id}/${f.id}`, { claim: c, fault: f });
|
|
95
101
|
|
|
@@ -166,7 +172,12 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
166
172
|
}
|
|
167
173
|
const target = join(projectDir, cur.fault.file);
|
|
168
174
|
if (existsSync(target) && hashFile(target) !== r.inputs.targetHash) doc.stale.push(`${cur.fault.file} changed since ${key} was probed`);
|
|
169
|
-
const
|
|
175
|
+
const selection = defenderSelection(cur.claim, cur.fault);
|
|
176
|
+
const defenders = selection.requested.length ? resolveDefenders(projectDir, selection.requested) : discoverDefenders(projectDir, cur.fault.file);
|
|
177
|
+
const priorOrigin = r.defenders.selectionSource ?? (r.defenders.requested.length ? 'claim' : 'discovery');
|
|
178
|
+
if (priorOrigin !== selection.selectionSource || JSON.stringify(r.defenders.requested) !== JSON.stringify(selection.requested) || JSON.stringify([...r.defenders.resolved].sort()) !== JSON.stringify([...defenders].sort())) {
|
|
179
|
+
doc.stale.push(`defender selection for ${key} changed since it was probed`);
|
|
180
|
+
}
|
|
170
181
|
for (const d of defenders) {
|
|
171
182
|
const h = r.inputs.defenderHashes[d];
|
|
172
183
|
if (!h) doc.stale.push(`${d} now defends ${key} but was not probed`);
|
|
@@ -219,7 +230,8 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
219
230
|
if (top) {
|
|
220
231
|
doc.state = 'unproven';
|
|
221
232
|
const cur = faultIndex.get(`${top.claimId}/${top.subjectId}`);
|
|
222
|
-
const
|
|
233
|
+
const { requested } = defenderSelection(cur.claim, cur.fault);
|
|
234
|
+
const defenders = requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, cur.fault.file);
|
|
223
235
|
const where = defenders[0] ?? `a new test file that imports ${cur.fault.file}`;
|
|
224
236
|
const whyByVerdict = {
|
|
225
237
|
survived: `${top.claimId}/${top.subjectId} (${top.severity}) survived: "${cur.claim.statement}" can be false with the suite green.`,
|
|
@@ -281,6 +293,7 @@ export function renderStatus(doc) {
|
|
|
281
293
|
for (const u of c.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
282
294
|
for (const r of c.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}`);
|
|
283
295
|
for (const e of c.expired) lines.push(`EXPIRED ignore "${e.pattern}" no longer excuses ${e.files.join(', ')}`);
|
|
296
|
+
for (const n of c.nested ?? []) lines.push(`delegated ${n.files.length} file(s) belong to ${n.project}/; run testguard gate ${n.project} --changed ${c.ref}${c.includeDirty ? ' --include-dirty' : ''}; child coverage has not been evaluated here`);
|
|
284
297
|
}
|
|
285
298
|
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
286
299
|
for (const f of doc.invalidFaults ?? []) lines.push(`INVALID ${f.claimId}/${f.subjectId} ${f.reason}: ${f.hits} hits, expected ${f.expected} (${f.file})`);
|
package/src/status/surface.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { extname, join } from 'node:path';
|
|
3
3
|
import { git } from '../git.mjs';
|
|
4
|
-
import { SOURCE_EXT, isDefaultExcluded, isTestFile } from '../gate/changed.mjs';
|
|
4
|
+
import { SOURCE_EXT, isDefaultExcluded, isTestFile, nestedProjectResolver } from '../gate/changed.mjs';
|
|
5
5
|
import { walk } from '../util/glob.mjs';
|
|
6
6
|
|
|
7
7
|
export const HISTORY_COMMITS = 200;
|
|
@@ -24,11 +24,12 @@ function listedFiles(projectDir) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
function claimableModules(projectDir) {
|
|
27
|
+
const ownerOf = nestedProjectResolver(projectDir);
|
|
27
28
|
return listedFiles(projectDir).filter((file) =>
|
|
28
29
|
existsSync(join(projectDir, file)) &&
|
|
29
30
|
SOURCE_EXT.has(extname(file)) &&
|
|
30
31
|
!isTestFile(file) &&
|
|
31
|
-
!isDefaultExcluded(file));
|
|
32
|
+
!isDefaultExcluded(file) && !ownerOf(file));
|
|
32
33
|
}
|
|
33
34
|
|
|
34
35
|
function recentChanges(projectDir, sourceModules) {
|