testguard-cli 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +48 -8
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +21 -0
- package/spec/lib/validate.mjs +9 -0
- package/spec/schemas/claims.schema.json +6 -0
- package/spec/schemas/evidence.schema.json +4 -0
- package/spec/schemas/status.schema.json +25 -0
- package/src/commands/admit.mjs +8 -3
- package/src/commands/claims.mjs +40 -4
- package/src/gate/changed.mjs +5 -1
- package/src/init/init.mjs +6 -0
- package/src/init/templates/SKILL.md +43 -0
- package/src/probe/attribution.mjs +9 -0
- package/src/probe/cost.mjs +16 -8
- package/src/probe/probe.mjs +27 -23
- package/src/status/status.mjs +9 -2
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.16.0] - 2026-09-30
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.15.1`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- feat: fault-specific defenders, provenance and cost reporting (#184)
|
|
17
|
+
- Add advisory AI update guidance and stabilize cleanup regression (#183)
|
|
18
|
+
- chore(homebrew): sha256 for v0.15.1 (#182)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Fault-specific `defendedBy` overrides, with explicit empty-list discovery,
|
|
24
|
+
evidence selection provenance, per-fault cost reporting and prior-kill
|
|
25
|
+
narrowing warnings (#90). Selection changes invalidate reuse and status;
|
|
26
|
+
admission and changed-file coverage use the actual selected defenders.
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
|
|
30
|
+
- New AI integrations include permission-aware, once-per-session update advice
|
|
31
|
+
that identifies the actual CLI and requests approval before any upgrade.
|
|
32
|
+
CLI commands and session-start hooks remain offline. Existing customized
|
|
33
|
+
instructions remain untouched; safe manual refresh is documented.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- The daemon-cleanup regression test now requires explicit startup and
|
|
38
|
+
post-cleanup release signals instead of assuming a detached process starts
|
|
39
|
+
and writes within fixed sleeps. Timeout and cleanup semantics are unchanged.
|
|
40
|
+
|
|
10
41
|
## [0.15.1] - 2026-09-30
|
|
11
42
|
|
|
12
43
|
Automated weekly release — everything merged since `v0.15.0`.
|
package/README.md
CHANGED
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.16.0` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -221,6 +221,14 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
221
221
|
the evidence records them as `repo.ignoredDirty`, so the run says what it
|
|
222
222
|
did not look at. Every summary names the commit probed.
|
|
223
223
|
|
|
224
|
+
A fault may declare its own `defendedBy` to override the claim's set.
|
|
225
|
+
Omit the fault field to inherit; set it to `[]` to request discovery rather
|
|
226
|
+
than inheritance. Evidence names the selection origin and selected files.
|
|
227
|
+
Changing either invalidates reuse and makes status stale. A narrowed fault
|
|
228
|
+
set after a prior kill is warned about by `claims`; re-probe before trusting
|
|
229
|
+
it. For example, a unit-level fault can select `["test/unit.test.mjs"]`
|
|
230
|
+
while an integration fault inherits the claim's broader set.
|
|
231
|
+
|
|
224
232
|
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
225
233
|
files that import the fault's target, by relative path or resolved alias
|
|
226
234
|
(tsconfig `paths`, through `extends` and `references`, vite/vitest
|
|
@@ -328,6 +336,34 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
328
336
|
offline. The brief's first line says which install answered
|
|
329
337
|
(`local install` or `global`), so a stale one is visible.
|
|
330
338
|
|
|
339
|
+
### Advisory updates for AI sessions
|
|
340
|
+
|
|
341
|
+
New integrations created by `testguard init` tell the AI to identify the actual
|
|
342
|
+
CLI (project-local before global) and, once per session when network policy
|
|
343
|
+
permits, read the registry's stable version:
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
This is an optional check performed by the AI, not by TestGuard's CLI or hook.
|
|
350
|
+
A newer stable version produces a suggestion and a request for approval, never
|
|
351
|
+
an automatic install. Project pins and the original package manager still
|
|
352
|
+
apply; prereleases and development checkouts are not replaced. Offline or
|
|
353
|
+
failed checks do not change verdicts, evidence or exit codes, and do not prove
|
|
354
|
+
that an installation is current. AI instructions are guidance, not a guarantee
|
|
355
|
+
that every harness performs the check.
|
|
356
|
+
|
|
357
|
+
Upgrading the package does **not** refresh an existing copied skill or an
|
|
358
|
+
existing managed section already listing the project. Compare the upgraded
|
|
359
|
+
package's `src/init/templates/SKILL.md` with
|
|
360
|
+
`.claude/skills/testguard/SKILL.md` and merge approved changes, preserving local
|
|
361
|
+
customizations. For an existing `AGENTS.md` section, add the same advisory
|
|
362
|
+
policy after reviewing it. `testguard init --force` replaces the whole skill
|
|
363
|
+
but does not refresh an already-listed managed section; use it only with
|
|
364
|
+
explicit overwrite approval and review the diff. No new automatic refresh
|
|
365
|
+
mechanism is introduced.
|
|
366
|
+
|
|
331
367
|
### Would this suite have caught the bugs that already escaped?
|
|
332
368
|
|
|
333
369
|
An injected fault is a fault somebody thought of. A bug that actually shipped
|
|
@@ -677,12 +713,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
677
713
|
# GitHub Actions
|
|
678
714
|
- uses: actions/checkout@v4
|
|
679
715
|
with: { fetch-depth: 0 }
|
|
680
|
-
- uses: raccioly/testguard@v0.
|
|
716
|
+
- uses: raccioly/testguard@v0.16.0
|
|
681
717
|
with: { command: gate }
|
|
682
718
|
|
|
683
719
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
684
720
|
include:
|
|
685
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
721
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
686
722
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
687
723
|
```
|
|
688
724
|
|
|
@@ -955,19 +991,23 @@ most expensive claims
|
|
|
955
991
|
4m 58s TG-FAULT-EDIT-VISIBLE 1 fault · 6 runs
|
|
956
992
|
|
|
957
993
|
defender files, by the time of the runs that included them
|
|
958
|
-
(a
|
|
994
|
+
(a fault runs its selected defender set at once, so these overlap and do not sum to the total)
|
|
959
995
|
24m 3s test/probe.fixture.test.mjs 5 claims
|
|
960
996
|
|
|
961
|
-
shared defenders — each
|
|
997
|
+
shared defenders — each fault selects the file; durations include recorded baseline sharing
|
|
962
998
|
24m 3s test/probe.fixture.test.mjs
|
|
963
|
-
named by TG-ESCALATION-N-RUNS, TG-FAULT-EDIT-VISIBLE, …
|
|
999
|
+
named by TG-ESCALATION-N-RUNS/F1, TG-FAULT-EDIT-VISIBLE/F1, …
|
|
964
1000
|
```
|
|
965
1001
|
|
|
966
1002
|
Every number is read back out of the `durationMs` the probe already records.
|
|
967
1003
|
`--cost` re-measures nothing, spawns nothing, and costs the price of reading
|
|
968
1004
|
one JSON file.
|
|
969
1005
|
|
|
970
|
-
|
|
1006
|
+
Cost output also lists individual faults, their selected defenders and selection
|
|
1007
|
+
origin. Sharing is counted across distinct fault identities, including siblings
|
|
1008
|
+
within one claim; repeated confirmation runs alone are not sharing.
|
|
1009
|
+
|
|
1010
|
+
**The per-file figures overlap.** A run executes a fault's whole defender set
|
|
971
1011
|
at once, so the runner never says how much of it belonged to which file. A
|
|
972
1012
|
file's `ms` is the time of every run that *included* it: an upper bound on
|
|
973
1013
|
what removing it could save, which is the decision you are making. Only
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Gate semantics
|
|
2
2
|
|
|
3
|
+
## Fault-specific defender selection
|
|
4
|
+
|
|
5
|
+
A fault's `defendedBy`, when present, overrides its claim's declaration.
|
|
6
|
+
An explicit empty fault list requests discovery and never falls back to the
|
|
7
|
+
claim. Without a fault override, a non-empty claim list is inherited; otherwise
|
|
8
|
+
defenders are discovered. New evidence records `defenders.selectionSource` as
|
|
9
|
+
`fault`, `claim`, or `discovery`, alongside the actual requested and resolved
|
|
10
|
+
sets. Empty requests use discovery and carry `discovered: true`, including an
|
|
11
|
+
explicit empty fault override. Non-empty requests are never marked discovered.
|
|
12
|
+
Dirty-input preflight, runner ownership, mock-awareness, input hashing and
|
|
13
|
+
baseline caching all operate on each fault's selected defender set.
|
|
14
|
+
|
|
15
|
+
Changing requested globs, resolved files or selection origin invalidates verdict
|
|
16
|
+
reuse. Legacy evidence remains readable but cannot authorize a new fault-level
|
|
17
|
+
override. A strict resolved subset after a previous kill is reported by `claims`
|
|
18
|
+
as a narrowing warning and requires re-probing; it is not a new verdict.
|
|
19
|
+
Cost reports include per-fault recorded run durations and selection origin.
|
|
20
|
+
Shared defenders identify the actual fault records that selected each file,
|
|
21
|
+
including multiple faults within one claim. File costs remain overlapping upper
|
|
22
|
+
bounds, not additive shares. Existing total/per-claim budget ceilings are unchanged.
|
|
23
|
+
|
|
3
24
|
How a Guard-spec tool decides whether CI goes red. Shared by every tool that
|
|
4
25
|
adopts the spec; each tool may add stricter rules, never looser ones.
|
|
5
26
|
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -146,6 +146,15 @@ const semantic = {
|
|
|
146
146
|
if (r.detail.undeclaredKillers && r.detail.reason !== 'killed-by-undeclared-tests') {
|
|
147
147
|
errors.push({ path: `${p}/detail/undeclaredKillers`, message: 'undeclaredKillers is only meaningful with reason killed-by-undeclared-tests' });
|
|
148
148
|
}
|
|
149
|
+
if (r.defenders.selectionSource) {
|
|
150
|
+
const empty = r.defenders.requested.length === 0;
|
|
151
|
+
if ((r.defenders.selectionSource === 'discovery' && !empty)
|
|
152
|
+
|| (r.defenders.selectionSource === 'claim' && empty)
|
|
153
|
+
|| (empty && r.defenders.discovered !== true)
|
|
154
|
+
|| (!empty && r.defenders.discovered === true)) {
|
|
155
|
+
errors.push({ path: `${p}/defenders/selectionSource`, message: 'defender selection origin, requested set and discovery flag disagree' });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
149
158
|
if (r.defenders.byRunner) {
|
|
150
159
|
const resolved = new Set(r.defenders.resolved);
|
|
151
160
|
for (const [runner, files] of Object.entries(r.defenders.byRunner)) for (const f of files) if (!resolved.has(f)) errors.push({ path: `${p}/defenders/byRunner/${runner}`, message: `${f} ran under ${runner} but is not a resolved defender` });
|
|
@@ -121,6 +121,12 @@
|
|
|
121
121
|
"faultClass": {
|
|
122
122
|
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
123
123
|
},
|
|
124
|
+
"defendedBy": {
|
|
125
|
+
"description": "Overrides this fault's claim-level defenders when present. An explicit empty list requests discovery; absence inherits the claim. Selection changes require re-probing, never assumed detection.",
|
|
126
|
+
"type": "array",
|
|
127
|
+
"items": { "type": "string", "minLength": 1 },
|
|
128
|
+
"uniqueItems": true
|
|
129
|
+
},
|
|
124
130
|
"file": {
|
|
125
131
|
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
126
132
|
},
|
|
@@ -520,6 +520,10 @@
|
|
|
520
520
|
"nocover"
|
|
521
521
|
],
|
|
522
522
|
"properties": {
|
|
523
|
+
"selectionSource": {
|
|
524
|
+
"description": "Origin of the requested defender selection: explicit fault override, inherited non-empty claim declaration, or automatic discovery. Optional only for legacy evidence; a changed origin invalidates reuse.",
|
|
525
|
+
"enum": ["fault", "claim", "discovery"]
|
|
526
|
+
},
|
|
523
527
|
"requested": {
|
|
524
528
|
"type": "array",
|
|
525
529
|
"items": {
|
|
@@ -195,6 +195,24 @@
|
|
|
195
195
|
"totalRuns": { "type": "integer", "minimum": 0 },
|
|
196
196
|
"reusedRecords": { "type": "integer", "minimum": 0 },
|
|
197
197
|
"records": { "type": "integer", "minimum": 0 },
|
|
198
|
+
"faults": {
|
|
199
|
+
"description": "Recorded costs and selected defenders per fault; optional for legacy reports.",
|
|
200
|
+
"type": "array",
|
|
201
|
+
"items": {
|
|
202
|
+
"type": "object",
|
|
203
|
+
"required": ["claimId", "faultId", "ms", "runs", "reused", "defenders"],
|
|
204
|
+
"properties": {
|
|
205
|
+
"claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
206
|
+
"faultId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
207
|
+
"ms": { "type": "number", "minimum": 0 },
|
|
208
|
+
"runs": { "type": "integer", "minimum": 0 },
|
|
209
|
+
"reused": { "type": "boolean" },
|
|
210
|
+
"selectionSource": { "enum": ["fault", "claim", "discovery"] },
|
|
211
|
+
"defenders": { "type": "array", "uniqueItems": true, "items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" } }
|
|
212
|
+
},
|
|
213
|
+
"additionalProperties": false
|
|
214
|
+
}
|
|
215
|
+
},
|
|
198
216
|
"claims": {
|
|
199
217
|
"type": "array",
|
|
200
218
|
"items": {
|
|
@@ -483,6 +501,13 @@
|
|
|
483
501
|
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
484
502
|
"ms": { "type": "number", "minimum": 0 },
|
|
485
503
|
"runs": { "type": "integer", "minimum": 0 },
|
|
504
|
+
"faults": {
|
|
505
|
+
"description": "Distinct claimId/faultId identities selecting this file, not confirmation runs.",
|
|
506
|
+
"type": "array",
|
|
507
|
+
"minItems": 1,
|
|
508
|
+
"uniqueItems": true,
|
|
509
|
+
"items": { "type": "string", "minLength": 1 }
|
|
510
|
+
},
|
|
486
511
|
"claims": {
|
|
487
512
|
"type": "array",
|
|
488
513
|
"minItems": 1,
|
package/src/commands/admit.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { probe } from '../probe/probe.mjs';
|
|
|
5
5
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
9
|
import { PreconditionError } from '../probe/worktree.mjs';
|
|
9
10
|
import { hintFor } from '../brief/brief.mjs';
|
|
10
11
|
import { formatVerdict, PROVISIONAL_WARNING } from '../render.mjs';
|
|
@@ -62,11 +63,15 @@ export async function admitCommand({ file, values, version }, io) {
|
|
|
62
63
|
|
|
63
64
|
// The named test must be a defender of the claim, declared or discovered;
|
|
64
65
|
// otherwise `killed` below would be someone else's test doing the work.
|
|
65
|
-
const
|
|
66
|
-
|
|
66
|
+
const selections = faults.map((f) => {
|
|
67
|
+
const { requested } = defenderSelection(claim, f);
|
|
68
|
+
return { requested, resolved: requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file) };
|
|
69
|
+
});
|
|
70
|
+
const declared = selections.every((s) => s.requested.length) ? selections[0].requested : null;
|
|
71
|
+
const isDefender = selections.some((s) => s.resolved.includes(testRel));
|
|
67
72
|
if (!isDefender) {
|
|
68
73
|
io.err(declared
|
|
69
|
-
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...
|
|
74
|
+
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...declared, testRel].map((d) => JSON.stringify(d)).join(', ')}] on the selected fault or its inherited claim in testguard.claims.json, then run admit again.`
|
|
70
75
|
: `${testRel} does not import ${[...new Set(faults.map((f) => f.file))].join(' or ')}, so it cannot be discovered as a defender of ${claimId}. Import the module under test, or declare "defendedBy": ["${testRel}"] on the claim.`);
|
|
71
76
|
return 3;
|
|
72
77
|
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -10,6 +10,27 @@ import { costReport, renderCost } from '../probe/cost.mjs';
|
|
|
10
10
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
11
11
|
import { existsSync } from 'node:fs';
|
|
12
12
|
import { checkAnchors, renderAnchorChecks } from '../claims/anchors.mjs';
|
|
13
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
14
|
+
|
|
15
|
+
/** A selection warning is not a verdict: only a fresh probe can defend it. */
|
|
16
|
+
export function defenderNarrowing(projectDir, claims, records) {
|
|
17
|
+
const prior = new Map(records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]));
|
|
18
|
+
const warnings = [];
|
|
19
|
+
for (const c of claims.claims) {
|
|
20
|
+
const inherited = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [];
|
|
21
|
+
for (const f of c.faults) {
|
|
22
|
+
const record = prior.get(`${c.id}/${f.id}`);
|
|
23
|
+
if (!Object.hasOwn(f, 'defendedBy') || record?.verdict !== 'killed') continue;
|
|
24
|
+
const { requested } = defenderSelection(c, f);
|
|
25
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
26
|
+
if (record.defenders.selectionSource === 'fault' && JSON.stringify(record.defenders.requested) === JSON.stringify(requested) && JSON.stringify([...record.defenders.resolved].sort()) === JSON.stringify([...resolved].sort())) continue;
|
|
27
|
+
if (resolved.length < inherited.length && resolved.every((d) => inherited.includes(d))) {
|
|
28
|
+
warnings.push({ claimId: c.id, faultId: f.id, inherited, resolved });
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return warnings;
|
|
33
|
+
}
|
|
13
34
|
|
|
14
35
|
export async function claimsCommand({ projectDir, values, version }, io) {
|
|
15
36
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
@@ -28,14 +49,17 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
28
49
|
|
|
29
50
|
// Cost is read back out of evidence the probe already wrote; it never runs a test.
|
|
30
51
|
let cost;
|
|
52
|
+
const ev = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
53
|
+
const hasOverrides = claims.claims.some((c) => c.faults.some((f) => Object.hasOwn(f, 'defendedBy')));
|
|
54
|
+
const prior = (values.cost || hasOverrides) && existsSync(ev) ? readSpecDoc('evidence', ev) : undefined;
|
|
55
|
+
const narrowedDefenders = hasOverrides && prior ? defenderNarrowing(projectDir, claims, prior.records) : [];
|
|
31
56
|
if (values.cost) {
|
|
32
|
-
|
|
33
|
-
cost = existsSync(ev) ? costReport(readSpecDoc('evidence', ev).records) : undefined;
|
|
57
|
+
cost = prior ? costReport(prior.records) : undefined;
|
|
34
58
|
if (!cost && !values.json) io.err(`no evidence at ${ev} — run \`testguard probe\` first; cost is derived from the run durations it records`);
|
|
35
59
|
}
|
|
36
60
|
|
|
37
61
|
if (values.json) {
|
|
38
|
-
io.out(JSON.stringify({ path, claims, annotations, drift, ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
62
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, ...(hasOverrides ? { narrowedDefenders } : {}), ...(anchorChecks ? { anchorChecks } : {}), ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
39
63
|
} else {
|
|
40
64
|
const annotated = new Set(drift.annotated);
|
|
41
65
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
@@ -45,7 +69,18 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
45
69
|
const declared = c.defendedBy?.length > 0;
|
|
46
70
|
const targets = [...new Set(c.faults.map((f) => f.file))];
|
|
47
71
|
let cover;
|
|
48
|
-
if (
|
|
72
|
+
if (c.faults.some((f) => Object.hasOwn(f, 'defendedBy'))) {
|
|
73
|
+
const selected = new Set();
|
|
74
|
+
for (const f of c.faults) {
|
|
75
|
+
const selection = defenderSelection(c, f);
|
|
76
|
+
const defenders = selection.requested.length ? resolveDefenders(projectDir, selection.requested) : discoverDefendersDetailed(projectDir, f.file).canDetect;
|
|
77
|
+
defenders.forEach((d) => selected.add(d));
|
|
78
|
+
const m = classifyDefenders(projectDir, f.file, defenders);
|
|
79
|
+
for (const s of m.signals) signalLines.push({ claim: c.id, target: f.file, ...s });
|
|
80
|
+
io.out(` ${c.id}/${f.id}: ${selection.selectionSource}${selection.requested.length ? '' : ' (discovery)'} — ${defenders.join(', ') || 'NO DEFENDER'}`);
|
|
81
|
+
}
|
|
82
|
+
cover = `${selected.size} selected defender${selected.size === 1 ? '' : 's'}`;
|
|
83
|
+
} else if (declared) {
|
|
49
84
|
const defenders = resolveDefenders(projectDir, c.defendedBy);
|
|
50
85
|
const mocking = new Set();
|
|
51
86
|
for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
@@ -70,6 +105,7 @@ export async function claimsCommand({ projectDir, values, version }, io) {
|
|
|
70
105
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
71
106
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
|
72
107
|
for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
|
|
108
|
+
for (const w of narrowedDefenders) io.out(`NARROWED-DEFENDERS ${w.claimId}/${w.faultId} selects ${w.resolved.length} of ${w.inherited.length} inherited defenders after a prior kill — re-probe; the old kill does not prove the narrower selection.`);
|
|
73
109
|
if (removed) {
|
|
74
110
|
io.out('');
|
|
75
111
|
io.out(renderRemoved(removed));
|
package/src/gate/changed.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { loadClaims, defaultClaimsPath, ClaimsError } from '../claims/load.mjs';
|
|
|
5
5
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
8
9
|
import { IS_PY_TEST } from '../probe/pyimports.mjs';
|
|
9
10
|
import { globToRegExp } from '../util/glob.mjs';
|
|
10
11
|
|
|
@@ -228,7 +229,10 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
228
229
|
const defenderFiles = new Map(); // test file → claimIds
|
|
229
230
|
for (const c of claims.claims) {
|
|
230
231
|
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
231
|
-
const defenders =
|
|
232
|
+
const defenders = [...new Set(c.faults.flatMap((f) => {
|
|
233
|
+
const { requested } = defenderSelection(c, f);
|
|
234
|
+
return requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, f.file);
|
|
235
|
+
}))];
|
|
232
236
|
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
233
237
|
}
|
|
234
238
|
|
package/src/init/init.mjs
CHANGED
|
@@ -56,6 +56,12 @@ Before writing or changing tests, run \`testguard status --json [dir]\` and foll
|
|
|
56
56
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
57
57
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
58
58
|
|
|
59
|
+
Once per AI session, identify the exact CLI used (project-local first, then repository-root, then PATH) and read its \`--version\`.
|
|
60
|
+
If network policy permits, check \`npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000\` once; this is optional AI advice, not a CLI or hook network call.
|
|
61
|
+
Compare stable versions numerically by SemVer; do not downgrade or replace a prerelease/development checkout. If a newer stable version exists, suggest it and ask before updating, respecting pins and the original package manager.
|
|
62
|
+
Never install automatically or alter lockfiles/CI pins without approval. On an unavailable check, continue without claiming up-to-date; keep advice separate from evidence, verdicts and exit codes. Do not repeat a declined suggestion this session.
|
|
63
|
+
Existing copied skills and sections are preserved by ordinary init; review and merge template changes manually, or use \`init --force\` only with approval to overwrite the skill.
|
|
64
|
+
|
|
59
65
|
${lines.join('\n')}
|
|
60
66
|
${AGENTS_END}`;
|
|
61
67
|
}
|
|
@@ -9,6 +9,49 @@ TestGuard injects the faults a project's claims forbid and reports every one
|
|
|
9
9
|
the tests fail to detect. It is a **claim verifier**, not a test generator.
|
|
10
10
|
You operate it through one loop and one source of truth.
|
|
11
11
|
|
|
12
|
+
## Update advice (once per session, never an automatic install)
|
|
13
|
+
|
|
14
|
+
Before the operating loop, identify the CLI you will actually run: prefer the
|
|
15
|
+
project's installed `node_modules/.bin/testguard`, then the repository-root
|
|
16
|
+
install for a nested project, then `testguard` on PATH. Run that exact CLI with
|
|
17
|
+
`--version`, and use the same executable for verification. Do not use `npx` to
|
|
18
|
+
discover a version: it can fetch or install a different tool. When developing
|
|
19
|
+
TestGuard itself, use `node cli/testguard.mjs`; do not replace a source checkout
|
|
20
|
+
with a published version to make validation pass.
|
|
21
|
+
|
|
22
|
+
If the user's network policy permits registry access, make one bounded,
|
|
23
|
+
read-only check per session (not per command or project):
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Compare valid stable versions by numeric SemVer major, minor, then patch, not
|
|
30
|
+
lexicographically and not merely for inequality. Do not recommend a downgrade
|
|
31
|
+
or replacing a prerelease/development checkout. If the registry result is
|
|
32
|
+
missing, malformed, unavailable, or network access is prohibited, continue the
|
|
33
|
+
operating loop without claiming the installation is up to date. No retries or
|
|
34
|
+
repeated prompts after the user declines within this session.
|
|
35
|
+
|
|
36
|
+
For a newer stable version, say: "Using TestGuard X from <resolved executable>;
|
|
37
|
+
Y is available. Would you like to update?" Respect project pins, compatibility,
|
|
38
|
+
company policy and the original package manager; a registry's `latest` tag is
|
|
39
|
+
not proof that an upgrade is safe. Never install automatically, modify a
|
|
40
|
+
lockfile or CI pin, or switch between local and global installs without approval.
|
|
41
|
+
After an approved update, verify the same executable's version again.
|
|
42
|
+
|
|
43
|
+
Keep this advice in the AI conversation, separate from CLI JSON, evidence,
|
|
44
|
+
verdicts and exit codes. The CLI and session-start hook remain offline; the AI
|
|
45
|
+
performs this optional check, not TestGuard. Instructions cannot guarantee that
|
|
46
|
+
every AI checks for updates.
|
|
47
|
+
|
|
48
|
+
Package upgrades do not refresh an existing copied skill. Preserve customized
|
|
49
|
+
instructions: compare this file with the upgraded package's
|
|
50
|
+
`src/init/templates/SKILL.md` and merge approved changes. Use `testguard init
|
|
51
|
+
--force` only with explicit permission to replace the whole skill; review the
|
|
52
|
+
diff afterward. An ordinary `init` preserves existing skills and managed
|
|
53
|
+
sections already listing this project.
|
|
54
|
+
|
|
12
55
|
## The one source of truth
|
|
13
56
|
|
|
14
57
|
```bash
|
|
@@ -109,11 +109,20 @@ export function subjectOf(fault, sha256) {
|
|
|
109
109
|
* The cost of re-measuring is a run; the cost of the other direction is a
|
|
110
110
|
* verdict nobody measured.
|
|
111
111
|
*/
|
|
112
|
+
/** Select declarations without resolving globs or running discovery. */
|
|
113
|
+
export function defenderSelection(claim, fault) {
|
|
114
|
+
if (Object.hasOwn(fault, 'defendedBy')) return { requested: [...fault.defendedBy], selectionSource: 'fault' };
|
|
115
|
+
if (claim.defendedBy?.length) return { requested: [...claim.defendedBy], selectionSource: 'claim' };
|
|
116
|
+
return { requested: [], selectionSource: 'discovery' };
|
|
117
|
+
}
|
|
118
|
+
|
|
112
119
|
export function isReusable(prior, current) {
|
|
113
120
|
if (!prior || !prior.subject?.contentHash) return false;
|
|
114
121
|
// A record that exists because the probe threw is a statement about the run,
|
|
115
122
|
// not about the code. Reusing it would make one disturbed run permanent.
|
|
116
123
|
if (prior.detail?.reason === 'probe-error') return false;
|
|
124
|
+
const origin = prior.defenders.selectionSource ?? (prior.defenders.requested.length ? 'claim' : 'discovery');
|
|
125
|
+
if (current.selectionSource && origin !== current.selectionSource) return false;
|
|
117
126
|
const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
|
|
118
127
|
return prior.inputs.targetHash === current.inputs.targetHash
|
|
119
128
|
&& same(prior.inputs.defenderHashes, current.inputs.defenderHashes)
|
package/src/probe/cost.mjs
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* already recorded. Nothing re-measures, nothing spawns, nothing guesses: a
|
|
13
13
|
* cost report costs the price of reading one JSON file.
|
|
14
14
|
*
|
|
15
|
-
* ATTRIBUTION, HONESTLY. Runs execute a
|
|
15
|
+
* ATTRIBUTION, HONESTLY. Runs execute a fault's selected defender set together,
|
|
16
16
|
* so the runner never says how much of a run belonged to which file. Per-file
|
|
17
17
|
* `ms` is therefore the time of every run that *included* that file: an upper
|
|
18
18
|
* bound on what removing it could save, and, when a claim names several
|
|
@@ -70,15 +70,16 @@ export function defenderCosts(records) {
|
|
|
70
70
|
for (const r of records) {
|
|
71
71
|
const c = recordCost(r);
|
|
72
72
|
for (const file of r.defenders?.resolved ?? []) {
|
|
73
|
-
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set() };
|
|
73
|
+
const e = by.get(file) ?? { file, ms: 0, runs: 0, claims: new Set(), faults: new Set() };
|
|
74
74
|
e.ms += c.ms;
|
|
75
75
|
e.runs += c.runs;
|
|
76
76
|
e.claims.add(r.claim.id);
|
|
77
|
+
e.faults.add(`${r.claim.id}/${r.subject.id}`);
|
|
77
78
|
by.set(file, e);
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
return [...by.values()]
|
|
81
|
-
.map((e) => ({ ...e, claims: [...e.claims].sort() }))
|
|
82
|
+
.map((e) => ({ ...e, claims: [...e.claims].sort(), faults: [...e.faults].sort() }))
|
|
82
83
|
.sort((a, b) => b.ms - a.ms || a.file.localeCompare(b.file));
|
|
83
84
|
}
|
|
84
85
|
|
|
@@ -86,7 +87,7 @@ export function defenderCosts(records) {
|
|
|
86
87
|
* The whole cost model for one evidence document.
|
|
87
88
|
*
|
|
88
89
|
* `sharedDefenders` is the actionable part: a file named by more than one
|
|
89
|
-
*
|
|
90
|
+
* fault identity, including siblings within one claim. Splitting the
|
|
90
91
|
* behaviour it defends into a unit test that runs in milliseconds is what
|
|
91
92
|
* turns a twenty-four-minute gate into a one-minute one, and until a report
|
|
92
93
|
* names the file nobody knows which test to split.
|
|
@@ -100,8 +101,10 @@ export function costReport(records) {
|
|
|
100
101
|
reusedRecords: records.filter((r) => r.reusedFrom).length,
|
|
101
102
|
records: records.length,
|
|
102
103
|
claims,
|
|
104
|
+
faults: records.map((r) => ({ claimId: r.claim.id, faultId: r.subject.id, ...recordCost(r), defenders: [...(r.defenders?.resolved ?? [])], ...(r.defenders?.selectionSource ? { selectionSource: r.defenders.selectionSource } : {}) }))
|
|
105
|
+
.sort((a, b) => b.ms - a.ms || a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId)),
|
|
103
106
|
defenders,
|
|
104
|
-
sharedDefenders: defenders.filter((d) => d.
|
|
107
|
+
sharedDefenders: defenders.filter((d) => d.faults.length > 1),
|
|
105
108
|
};
|
|
106
109
|
}
|
|
107
110
|
|
|
@@ -123,19 +126,24 @@ export function renderCost(report, { limit = 10 } = {}) {
|
|
|
123
126
|
out.push(` ${secs(c.ms).padStart(7)} ${c.claimId.padEnd(34)} ${c.faults} fault${c.faults === 1 ? ' ' : 's'} · ${c.runs} runs${c.reused ? ` · ${c.reused} reused` : ''}`);
|
|
124
127
|
}
|
|
125
128
|
if (report.claims.length > limit) out.push(` … ${report.claims.length - limit} more`);
|
|
129
|
+
if (report.faults?.length) {
|
|
130
|
+
out.push('', 'most expensive faults');
|
|
131
|
+
for (const f of report.faults.slice(0, limit)) out.push(` ${secs(f.ms).padStart(7)} ${f.claimId}/${f.faultId} · ${f.runs} runs${f.selectionSource ? ` · ${f.selectionSource}` : ''} · ${f.defenders.join(', ') || 'no defenders'}`);
|
|
132
|
+
if (report.faults.length > limit) out.push(` … ${report.faults.length - limit} more`);
|
|
133
|
+
}
|
|
126
134
|
out.push('');
|
|
127
135
|
out.push('defender files, by the time of the runs that included them');
|
|
128
|
-
out.push(' (a
|
|
136
|
+
out.push(' (a fault runs its selected defender set at once, so these overlap and do not sum to the total)');
|
|
129
137
|
for (const d of report.defenders.slice(0, limit)) {
|
|
130
138
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file.padEnd(44)} ${d.claims.length} claim${d.claims.length === 1 ? '' : 's'}`);
|
|
131
139
|
}
|
|
132
140
|
if (report.defenders.length > limit) out.push(` … ${report.defenders.length - limit} more`);
|
|
133
141
|
if (report.sharedDefenders.length) {
|
|
134
142
|
out.push('');
|
|
135
|
-
out.push('shared defenders — each
|
|
143
|
+
out.push('shared defenders — each fault selects the file; durations include recorded baseline sharing');
|
|
136
144
|
for (const d of report.sharedDefenders.slice(0, limit)) {
|
|
137
145
|
out.push(` ${secs(d.ms).padStart(7)} ${d.file}`);
|
|
138
|
-
out.push(` named by ${d.
|
|
146
|
+
out.push(` named by ${d.faults.join(', ')}`);
|
|
139
147
|
}
|
|
140
148
|
out.push('');
|
|
141
149
|
out.push(' To make a probe cheaper, split the behaviour a shared defender proves into a');
|
package/src/probe/probe.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import { selectRunner, discoverRunnerManifest, RUNNERS, OWNED_RUNNERS, partition
|
|
|
13
13
|
import { classify, shouldStopEarly } from './classify.mjs';
|
|
14
14
|
import { blastRadius, rank } from './rank.mjs';
|
|
15
15
|
import { classifyIndependence } from './independence.mjs';
|
|
16
|
-
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable } from './attribution.mjs';
|
|
16
|
+
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
|
|
17
17
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
18
18
|
import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
19
19
|
import { persistenceSignalsFor } from '../supply/persistence.mjs';
|
|
@@ -154,11 +154,14 @@ export async function probe({
|
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
for (const claim of scopedClaims) {
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
157
|
+
for (const fault of claim.faults) {
|
|
158
|
+
const { requested } = defenderSelection(claim, fault);
|
|
159
|
+
const discovered = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, configuredFiles);
|
|
160
|
+
const declared = requested.length ? resolveDefenders(projectDir, requested) : discovered.canDetect;
|
|
161
|
+
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
162
|
+
for (const dependency of discovered?.dependencies ?? []) watched.add(relative(root, join(projectDir, dependency)));
|
|
163
|
+
for (const g of requested) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
164
|
+
}
|
|
162
165
|
}
|
|
163
166
|
for (const config of [
|
|
164
167
|
'package.json', 'pnpm-workspace.yaml', 'pnpm-workspace.yml', 'lerna.json', 'nx.json', 'turbo.json', 'rush.json', 'workspace.json',
|
|
@@ -315,20 +318,21 @@ export async function probe({
|
|
|
315
318
|
|
|
316
319
|
for (const claim of scopedClaims) {
|
|
317
320
|
commandBudget?.assertOpen();
|
|
318
|
-
const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
319
|
-
if (declared && !commandTemplate) {
|
|
320
|
-
const excluded = declared.filter((file) => !ownerByFile.has(file));
|
|
321
|
-
for (const file of excluded) {
|
|
322
|
-
const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
|
|
323
|
-
const check = expectedOwner && ownedChecked.get(expectedOwner);
|
|
324
|
-
if (expectedOwner && check && !check.ok) {
|
|
325
|
-
throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
if (excluded.length) throw new PreconditionError(`${claim.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
|
|
329
|
-
}
|
|
330
321
|
for (const fault of claim.faults) {
|
|
331
322
|
commandBudget?.assertOpen();
|
|
323
|
+
const { requested } = defenderSelection(claim, fault);
|
|
324
|
+
const declared = requested.length ? resolveDefenders(iso.projectDir, requested) : null;
|
|
325
|
+
if (declared && !commandTemplate) {
|
|
326
|
+
const excluded = declared.filter((file) => !ownerByFile.has(file));
|
|
327
|
+
for (const file of excluded) {
|
|
328
|
+
const expectedOwner = owned.find((candidate) => candidate.owns?.(iso.projectDir, file));
|
|
329
|
+
const check = expectedOwner && ownedChecked.get(expectedOwner);
|
|
330
|
+
if (expectedOwner && check && !check.ok) {
|
|
331
|
+
throw new PreconditionError(`${file} is a ${expectedOwner.name} test (${expectedOwner.name === 'python' ? 'it is a .py file' : `it lives under ${expectedOwner.name}'s testDir`}) but ${expectedOwner.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}).`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (excluded.length) throw new PreconditionError(`${claim.id}/${fault.id}: defendedBy resolves to ${excluded.join(', ')}, but the configured runners do not collect ${excluded.length === 1 ? 'that file' : 'those files'}`);
|
|
335
|
+
}
|
|
332
336
|
// Mock-awareness: a discovered file that mocks the target is not a
|
|
333
337
|
// defender; a declared one that mocks it stays (the author named it)
|
|
334
338
|
// but is listed, because a declared defender that mocks the subject
|
|
@@ -413,7 +417,7 @@ export function errorRecord({ claim, fault, defenders, discovered, byRunner, moc
|
|
|
413
417
|
subject: subjectOf(fault, sha256),
|
|
414
418
|
verdict: 'unverifiable',
|
|
415
419
|
detail: { baselineRuns: [], probeRuns: [], reason: 'probe-error', message },
|
|
416
|
-
defenders: {
|
|
420
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
|
|
417
421
|
inputs,
|
|
418
422
|
};
|
|
419
423
|
}
|
|
@@ -428,7 +432,7 @@ export function discoveryIndeterminateRecord({ claim, fault, defenders, discover
|
|
|
428
432
|
subject: subjectOf(fault, sha256),
|
|
429
433
|
verdict: 'unverifiable',
|
|
430
434
|
detail: { baselineRuns: [], probeRuns: [], reason: 'defender-discovery-indeterminate', message },
|
|
431
|
-
defenders: {
|
|
435
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: false, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
|
|
432
436
|
inputs,
|
|
433
437
|
};
|
|
434
438
|
}
|
|
@@ -516,8 +520,8 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
516
520
|
const subject = subjectOf(fault, sha256);
|
|
517
521
|
|
|
518
522
|
// Same source, same defenders, same N, same fault: the verdict cannot have changed.
|
|
519
|
-
if (isReusable(prior, { inputs,
|
|
520
|
-
return { ...prior, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
523
|
+
if (isReusable(prior, { inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
524
|
+
return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
521
525
|
}
|
|
522
526
|
|
|
523
527
|
const detail = { baselineRuns: [], probeRuns: [] };
|
|
@@ -655,7 +659,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
655
659
|
subject,
|
|
656
660
|
verdict,
|
|
657
661
|
detail,
|
|
658
|
-
defenders: {
|
|
662
|
+
defenders: { ...defenderSelection(claim, fault), resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(allSignals.length ? { signals: allSignals } : {}) },
|
|
659
663
|
inputs,
|
|
660
664
|
rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast, independence: detail.independence?.class }),
|
|
661
665
|
};
|
package/src/status/status.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { gate } from '../baseline/baseline.mjs';
|
|
|
6
6
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
7
7
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
8
8
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
|
+
import { defenderSelection } from '../probe/attribution.mjs';
|
|
9
10
|
import { coAuthorshipWarning, sortForReport } from '../render.mjs';
|
|
10
11
|
import { computeChangedGate, defaultIgnorePath } from '../gate/changed.mjs';
|
|
11
12
|
import { headSha, isAncestor } from '../git.mjs';
|
|
@@ -171,7 +172,12 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
171
172
|
}
|
|
172
173
|
const target = join(projectDir, cur.fault.file);
|
|
173
174
|
if (existsSync(target) && hashFile(target) !== r.inputs.targetHash) doc.stale.push(`${cur.fault.file} changed since ${key} was probed`);
|
|
174
|
-
const
|
|
175
|
+
const selection = defenderSelection(cur.claim, cur.fault);
|
|
176
|
+
const defenders = selection.requested.length ? resolveDefenders(projectDir, selection.requested) : discoverDefenders(projectDir, cur.fault.file);
|
|
177
|
+
const priorOrigin = r.defenders.selectionSource ?? (r.defenders.requested.length ? 'claim' : 'discovery');
|
|
178
|
+
if (priorOrigin !== selection.selectionSource || JSON.stringify(r.defenders.requested) !== JSON.stringify(selection.requested) || JSON.stringify([...r.defenders.resolved].sort()) !== JSON.stringify([...defenders].sort())) {
|
|
179
|
+
doc.stale.push(`defender selection for ${key} changed since it was probed`);
|
|
180
|
+
}
|
|
175
181
|
for (const d of defenders) {
|
|
176
182
|
const h = r.inputs.defenderHashes[d];
|
|
177
183
|
if (!h) doc.stale.push(`${d} now defends ${key} but was not probed`);
|
|
@@ -224,7 +230,8 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
224
230
|
if (top) {
|
|
225
231
|
doc.state = 'unproven';
|
|
226
232
|
const cur = faultIndex.get(`${top.claimId}/${top.subjectId}`);
|
|
227
|
-
const
|
|
233
|
+
const { requested } = defenderSelection(cur.claim, cur.fault);
|
|
234
|
+
const defenders = requested.length ? resolveDefenders(projectDir, requested) : discoverDefenders(projectDir, cur.fault.file);
|
|
228
235
|
const where = defenders[0] ?? `a new test file that imports ${cur.fault.file}`;
|
|
229
236
|
const whyByVerdict = {
|
|
230
237
|
survived: `${top.claimId}/${top.subjectId} (${top.severity}) survived: "${cur.claim.statement}" can be false with the suite green.`,
|