testguard-cli 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +61 -11
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +10 -4
- package/spec/lib/validate.mjs +3 -0
- package/spec/schemas/baseline.schema.json +4 -0
- package/spec/schemas/claims.schema.json +1 -1
- package/spec/schemas/evidence.schema.json +26 -2
- package/src/baseline/baseline.mjs +1 -1
- package/src/brief/brief.mjs +1 -1
- package/src/cli.mjs +12 -2
- package/src/commands/baseline.mjs +1 -1
- package/src/commands/claims.mjs +4 -2
- package/src/commands/probe.mjs +11 -2
- package/src/commands/scaffold.mjs +34 -0
- package/src/git.mjs +28 -2
- package/src/probe/classify.mjs +2 -1
- package/src/probe/discover.mjs +12 -0
- package/src/probe/probe.mjs +47 -10
- package/src/probe/rank.mjs +14 -0
- package/src/render.mjs +13 -3
- package/src/scaffold/producers.mjs +85 -0
- package/src/scaffold/scaffold.mjs +118 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,68 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.2.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **`testguard scaffold <file>`** — mechanical fault producer (#6). Proposes
|
|
15
|
+
the five shapes both field reports found behind ~80% of hand-written
|
|
16
|
+
faults: guard forced false, single-line guard or state change removed,
|
|
17
|
+
`return <check>` → `return true`, security literal weakened, check call
|
|
18
|
+
removed. Every proposal is an exact-line anchor with `expectHits` and
|
|
19
|
+
`occurrence` computed from the file (verifiable by construction; anything
|
|
20
|
+
`locate()` would reject is never emitted), `producedBy: derived`,
|
|
21
|
+
`defendedBy` prefilled from the tests that import the module, grouped
|
|
22
|
+
under a preceding `@claim` annotation or by enclosing function.
|
|
23
|
+
Statements are `TODO:` placeholders; the output is a draft under
|
|
24
|
+
`.testguard/`, never the claims file. `--claim <ID>` puts everything under
|
|
25
|
+
one claim and copies it if it exists; `--json` prints instead.
|
|
26
|
+
- Self-claim `TG-SCAFFOLD-ANCHORS-HIT`; install smoke exercises `scaffold`.
|
|
27
|
+
|
|
28
|
+
## [0.1.3] - 2026-09-17
|
|
29
|
+
|
|
30
|
+
From a second field report on a real codebase (456 tests, 27 claims, 35
|
|
31
|
+
faults; 13 survived on the first run, two critical claims with zero coverage).
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- **Worktree mode probed HEAD while reading the claims file from the working
|
|
36
|
+
tree**, so uncommitted defender changes were silently ignored — the same
|
|
37
|
+
survivors came back with no hint why. `probe` now refuses (exit 2) when any
|
|
38
|
+
resolved defender or fault target has uncommitted changes, naming the files
|
|
39
|
+
and the commit it would have probed. Every summary names the commit probed.
|
|
40
|
+
- `killed-by-undeclared-tests` never said which tests killed the fault; the
|
|
41
|
+
author could not fix `defendedBy` without grepping the suite. Evidence now
|
|
42
|
+
carries `detail.undeclaredKillers` and the CLI names the files.
|
|
43
|
+
- `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
|
|
44
|
+
expected }` is recorded and printed.
|
|
45
|
+
- A replacement with an unbalanced paren was `suite-failed-to-load`, not
|
|
46
|
+
`replacement-does-not-compile`: esbuild/vitest wording is now matched.
|
|
47
|
+
- The claims schema promised defender discovery for an absent `defendedBy`;
|
|
48
|
+
the tool answered `nocover`. Discovery is implemented: the test files that
|
|
49
|
+
import the fault's target (relative or alias), recorded as
|
|
50
|
+
`defenders.discovered`. `nocover` now means exactly "no test file imports
|
|
51
|
+
this source".
|
|
52
|
+
- `baseline.json` records `dirty`, as evidence already did.
|
|
53
|
+
|
|
54
|
+
### Added
|
|
55
|
+
|
|
56
|
+
- `--include-dirty`: snapshot the working tree (tracked edits and untracked,
|
|
57
|
+
non-ignored files) into a throwaway commit and probe that. HEAD, index and
|
|
58
|
+
the user's tree are never touched; `run.repo.snapshot` records the commit.
|
|
59
|
+
- A one-line progress indicator on stderr (TTY only) so a minute of silence
|
|
60
|
+
is not mistaken for a hang.
|
|
61
|
+
- Fixture: a claim with no `defendedBy` whose defender is discovered.
|
|
62
|
+
|
|
63
|
+
### Changed
|
|
64
|
+
|
|
65
|
+
- **Default output shows only unproven faults plus a killed count.**
|
|
66
|
+
`--verbose` restores the full stream.
|
|
67
|
+
- The `survived` hint reminds the author to check that the fault is
|
|
68
|
+
observable at all before writing a test for it.
|
|
69
|
+
- README: `$schema` path for consumers, `min-release-age` note, the
|
|
70
|
+
worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
|
|
71
|
+
|
|
10
72
|
## [0.1.2] - 2026-09-17
|
|
11
73
|
|
|
12
74
|
From a field report on a real codebase (63 test files, 458 tests, 39 faults).
|
package/README.md
CHANGED
|
@@ -52,9 +52,13 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
|
+
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
59
|
+
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
60
|
+
`npm i -D testguard-cli --min-release-age=0`.
|
|
61
|
+
|
|
58
62
|
## How it works
|
|
59
63
|
|
|
60
64
|
```bash
|
|
@@ -62,11 +66,14 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
62
66
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
63
67
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
64
68
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
69
|
+
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
65
70
|
```
|
|
66
71
|
|
|
67
|
-
1. **Claims** live in `testguard.claims.json
|
|
68
|
-
|
|
69
|
-
|
|
72
|
+
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
73
|
+
`"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
|
|
74
|
+
a statement, where it comes from, which tests supposedly defend it, and
|
|
75
|
+
one or more *faults* — each a deterministic source change that would make
|
|
76
|
+
the statement false. Every
|
|
70
77
|
claim and every fault records who produced it. `testguard claims`
|
|
71
78
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
72
79
|
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
@@ -91,10 +98,23 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
91
98
|
written to `.testguard/evidence.json` — validated against the spec before
|
|
92
99
|
it is written.
|
|
93
100
|
|
|
101
|
+
Worktree mode probes a **commit**. If a defender or target file has
|
|
102
|
+
uncommitted changes, `probe` refuses and says so — otherwise your new
|
|
103
|
+
tests would be silently absent and the same survivors would come back
|
|
104
|
+
with no hint why. `--include-dirty` snapshots the working tree (tracked
|
|
105
|
+
edits and new files) into a throwaway commit and probes that; your tree,
|
|
106
|
+
HEAD and index are never touched. Every summary names the commit probed.
|
|
107
|
+
|
|
108
|
+
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
109
|
+
files that import the fault's target, by relative path or resolved alias.
|
|
110
|
+
`NOCOVER` then means exactly "no test file imports this source".
|
|
111
|
+
|
|
94
112
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
95
|
-
suite N times per survivor); iterate on one claim with
|
|
96
|
-
`--
|
|
97
|
-
|
|
113
|
+
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
114
|
+
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
115
|
+
clean there; test files may be dirty); final pass with defaults. By default
|
|
116
|
+
the stream shows only unproven faults plus a killed count — `--verbose`
|
|
117
|
+
shows every fault. A custom
|
|
98
118
|
runner (`pnpm --filter`, a specific config) goes in
|
|
99
119
|
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
100
120
|
see your `node_modules`, pass `--node-modules <dir>`.
|
|
@@ -103,7 +123,8 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
103
123
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
104
124
|
in CI costs only what changed.
|
|
105
125
|
4. **Brief** turns evidence plus baseline into a ranked, capped
|
|
106
|
-
`## TEST BLINDSPOT CONTEXT` block
|
|
126
|
+
`## TEST BLINDSPOT CONTEXT` block, printed and also written to
|
|
127
|
+
`.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
|
|
107
128
|
— for Claude Code, in `.claude/settings.json`:
|
|
108
129
|
|
|
109
130
|
```json
|
|
@@ -115,6 +136,34 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
115
136
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
116
137
|
so the hook can never break a session.
|
|
117
138
|
|
|
139
|
+
### Authoring faults mechanically
|
|
140
|
+
|
|
141
|
+
Writing faults by hand means reading the code to find exact anchors. Two
|
|
142
|
+
field reports found that ~80% of hand-written faults are one of five shapes,
|
|
143
|
+
so `scaffold` proposes them for you:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
npx testguard-cli scaffold src/auth.ts # → .testguard/scaffold-auth.json (a draft, never your claims file)
|
|
147
|
+
npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN # every proposal under one claim; copies it if it exists
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
| Shape | What it proposes |
|
|
151
|
+
|---|---|
|
|
152
|
+
| `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
|
|
153
|
+
| `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
|
|
154
|
+
| `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
|
|
155
|
+
| `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
|
|
156
|
+
| `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
|
|
157
|
+
|
|
158
|
+
Every proposal's `find` is the exact line with `expectHits`/`occurrence`
|
|
159
|
+
computed from the file, so it is verifiable by construction; provenance is
|
|
160
|
+
`producer: derived`; `defendedBy` is prefilled from the tests that import
|
|
161
|
+
the module; proposals are grouped under a preceding `@claim <ID>` annotation
|
|
162
|
+
or by enclosing function. Statements are `TODO:` placeholders — a proposal
|
|
163
|
+
becomes a claim only when a human states what it defends. Deterministic
|
|
164
|
+
heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
|
|
165
|
+
emitted.
|
|
166
|
+
|
|
118
167
|
**Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
|
|
119
168
|
The baseline is the frozen contract; the other two are regenerated per run.
|
|
120
169
|
|
|
@@ -140,13 +189,14 @@ through every verdict.
|
|
|
140
189
|
|
|
141
190
|
## Status
|
|
142
191
|
|
|
143
|
-
**v0.
|
|
192
|
+
**v0.2.** Five commands, vitest runner, hand-authored faults plus a
|
|
193
|
+
mechanical scaffold for the five common shapes. The contract
|
|
144
194
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
145
195
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
146
196
|
|
|
147
197
|
Not yet: test generation (the two-gate acceptance loop), other runners,
|
|
148
|
-
|
|
149
|
-
|
|
198
|
+
AST-aware producers, and calibration of fault classes against real escaped
|
|
199
|
+
bugs. Each is designed for; none is claimed.
|
|
150
200
|
|
|
151
201
|
## Licence
|
|
152
202
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -35,10 +35,16 @@ Rules that follow from the table:
|
|
|
35
35
|
reporting `survived` would be wrong the other way. It is `flaky-defender`.
|
|
36
36
|
5. **Escalation never upgrades a verdict.** A fault that survives its declared
|
|
37
37
|
defenders may be re-run against the whole suite. If the wider suite kills
|
|
38
|
-
it, the verdict stays `survived` with reason `killed-by-undeclared-tests
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
it, the verdict stays `survived` with reason `killed-by-undeclared-tests`
|
|
39
|
+
and `detail.undeclaredKillers` names the tests, so the author can fix
|
|
40
|
+
`defendedBy`. The claim's stated evidence chain is broken even though the
|
|
41
|
+
suite is not blind. It gates, and ranks below a true survivor.
|
|
42
|
+
6. **A verdict names the commit it is about.** Evidence records `repo.head`;
|
|
43
|
+
when the working tree was probed instead, `repo.snapshot` holds the
|
|
44
|
+
throwaway commit that captured it. A tool must refuse to probe a commit
|
|
45
|
+
while defenders or targets have uncommitted changes, unless told to
|
|
46
|
+
snapshot the working tree — otherwise the answer looks right and is not.
|
|
47
|
+
7. **Never a single global score.** Output is per claim, ranked. Blindness is
|
|
42
48
|
concentrated, and one number hides where.
|
|
43
49
|
|
|
44
50
|
## Baseline and delta
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -72,6 +72,9 @@ const semantic = {
|
|
|
72
72
|
if (r.verdict === 'nocover' && !r.defenders.nocover) {
|
|
73
73
|
errors.push({ path: `${p}/defenders/nocover`, message: 'nocover verdict requires defenders.nocover = true' });
|
|
74
74
|
}
|
|
75
|
+
if (r.detail.undeclaredKillers && r.detail.reason !== 'killed-by-undeclared-tests') {
|
|
76
|
+
errors.push({ path: `${p}/detail/undeclaredKillers`, message: 'undeclaredKillers is only meaningful with reason killed-by-undeclared-tests' });
|
|
77
|
+
}
|
|
75
78
|
if (r.verdict === 'unverifiable' && !r.detail.reason) {
|
|
76
79
|
errors.push({ path: `${p}/detail/reason`, message: 'unverifiable requires a reason (e.g. anchor-missing, anchor-ambiguous)' });
|
|
77
80
|
}
|
|
@@ -11,6 +11,10 @@
|
|
|
11
11
|
"tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
|
|
12
12
|
"createdAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
|
|
13
13
|
"head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
14
|
+
"dirty": {
|
|
15
|
+
"description": "Whether the working tree had uncommitted changes when the evidence behind this baseline was taken.",
|
|
16
|
+
"type": "boolean"
|
|
17
|
+
},
|
|
14
18
|
"fingerprints": {
|
|
15
19
|
"type": "object",
|
|
16
20
|
"propertyNames": { "pattern": "^[a-f0-9]{64}$" },
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
|
|
43
43
|
"producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
|
|
44
44
|
"defendedBy": {
|
|
45
|
-
"description": "Globs (
|
|
45
|
+
"description": "Globs (project-relative) for the test files that supposedly defend this claim. Empty or absent means: discover defenders as the test files that import each fault's target file (relative paths and resolved aliases). If nothing resolves, the verdict is `nocover`.",
|
|
46
46
|
"type": "array",
|
|
47
47
|
"items": { "type": "string", "minLength": 1 },
|
|
48
48
|
"uniqueItems": true
|
|
@@ -21,7 +21,11 @@
|
|
|
21
21
|
"required": ["head", "dirty"],
|
|
22
22
|
"properties": {
|
|
23
23
|
"head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
24
|
-
"dirty": { "type": "boolean" }
|
|
24
|
+
"dirty": { "type": "boolean" },
|
|
25
|
+
"snapshot": {
|
|
26
|
+
"description": "When the working tree was probed instead of `head`: the dangling commit that captured it. Not a ref; may be garbage-collected.",
|
|
27
|
+
"$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
|
|
28
|
+
}
|
|
25
29
|
},
|
|
26
30
|
"additionalProperties": false
|
|
27
31
|
},
|
|
@@ -149,6 +153,22 @@
|
|
|
149
153
|
"escalationRuns": {
|
|
150
154
|
"type": "array",
|
|
151
155
|
"items": { "$ref": "#/$defs/testRun" }
|
|
156
|
+
},
|
|
157
|
+
"undeclaredKillers": {
|
|
158
|
+
"description": "With reason `killed-by-undeclared-tests`: the tests (as `file::name`) that failed in every escalation run. Names what to add to `defendedBy`.",
|
|
159
|
+
"type": "array",
|
|
160
|
+
"minItems": 1,
|
|
161
|
+
"items": { "type": "string", "minLength": 1, "maxLength": 1024 }
|
|
162
|
+
},
|
|
163
|
+
"anchor": {
|
|
164
|
+
"description": "With an anchor-related reason: how many times `find` occurred versus how many were expected.",
|
|
165
|
+
"type": "object",
|
|
166
|
+
"required": ["hits", "expected"],
|
|
167
|
+
"properties": {
|
|
168
|
+
"hits": { "type": "integer", "minimum": 0 },
|
|
169
|
+
"expected": { "type": "integer", "minimum": 1 }
|
|
170
|
+
},
|
|
171
|
+
"additionalProperties": false
|
|
152
172
|
}
|
|
153
173
|
},
|
|
154
174
|
"additionalProperties": false
|
|
@@ -159,7 +179,11 @@
|
|
|
159
179
|
"properties": {
|
|
160
180
|
"requested": { "type": "array", "items": { "type": "string" } },
|
|
161
181
|
"resolved": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" } },
|
|
162
|
-
"nocover": { "type": "boolean" }
|
|
182
|
+
"nocover": { "type": "boolean" },
|
|
183
|
+
"discovered": {
|
|
184
|
+
"description": "True when no defenders were declared and `resolved` was found by static import of the subject's file.",
|
|
185
|
+
"type": "boolean"
|
|
186
|
+
}
|
|
163
187
|
},
|
|
164
188
|
"additionalProperties": false
|
|
165
189
|
},
|
|
@@ -7,7 +7,7 @@ export function buildBaseline(evidence, { createdAt = new Date().toISOString() }
|
|
|
7
7
|
if (r.verdict === 'killed') continue;
|
|
8
8
|
fingerprints[r.fingerprint] = (fingerprints[r.fingerprint] ?? 0) + 1;
|
|
9
9
|
}
|
|
10
|
-
return { schemaVersion: 1, tool: evidence.tool, createdAt, head: evidence.run.repo.head, fingerprints };
|
|
10
|
+
return { schemaVersion: 1, tool: evidence.tool, createdAt, head: evidence.run.repo.head, dirty: evidence.run.repo.dirty, fingerprints };
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
/**
|
package/src/brief/brief.mjs
CHANGED
|
@@ -11,7 +11,7 @@ export function hintFor(r) {
|
|
|
11
11
|
case 'survived':
|
|
12
12
|
return r.detail.reason === 'killed-by-undeclared-tests'
|
|
13
13
|
? `Only tests outside its declared defenders (${defenders}) catch this; fix the claim's defendedBy or move the assertion.`
|
|
14
|
-
: `${defenders} stayed green with this fault applied; add an assertion that fails on it and passes on HEAD.`;
|
|
14
|
+
: `${defenders} stayed green with this fault applied; add an assertion that fails on it and passes on HEAD. If no test's outcome can change, first check the fault is observable at all.`;
|
|
15
15
|
case 'nocover':
|
|
16
16
|
return `No test file matches ${r.defenders.requested.join(', ') || '(no defenders declared)'}; nothing defends this claim.`;
|
|
17
17
|
case 'unverifiable':
|
package/src/cli.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { probeCommand } from './commands/probe.mjs';
|
|
|
10
10
|
import { claimsCommand } from './commands/claims.mjs';
|
|
11
11
|
import { baselineCommand } from './commands/baseline.mjs';
|
|
12
12
|
import { briefCommand } from './commands/brief.mjs';
|
|
13
|
+
import { scaffoldCommand } from './commands/scaffold.mjs';
|
|
13
14
|
|
|
14
15
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
15
16
|
|
|
@@ -19,6 +20,7 @@ const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a
|
|
|
19
20
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
20
21
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
21
22
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
23
|
+
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
22
24
|
|
|
23
25
|
probe
|
|
24
26
|
--claims <path> claims file (default: <dir>/testguard.claims.json)
|
|
@@ -29,6 +31,8 @@ probe
|
|
|
29
31
|
--severity <level> gate only at or above (default: low)
|
|
30
32
|
--ref <commit> probe this commit in the scratch worktree (default: HEAD)
|
|
31
33
|
--claim <ID,ID> probe only these claims; writes .testguard/evidence-partial.json
|
|
34
|
+
--include-dirty probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
|
|
35
|
+
--verbose also print each killed fault (default: only unproven ones, plus a count)
|
|
32
36
|
--runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
|
|
33
37
|
--node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
|
|
34
38
|
--in-place mutate the working tree instead of a scratch worktree
|
|
@@ -36,6 +40,9 @@ probe
|
|
|
36
40
|
--no-reuse re-probe claims whose inputs have not changed
|
|
37
41
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
38
42
|
|
|
43
|
+
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
44
|
+
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
45
|
+
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
39
46
|
claims --json
|
|
40
47
|
baseline --evidence <path> --out <path>
|
|
41
48
|
brief --evidence <path> --baseline <path> --max <n> --text (print only; safe for hooks)
|
|
@@ -43,7 +50,7 @@ brief --evidence <path> --baseline <path> --max <n> --text (print only;
|
|
|
43
50
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
|
|
44
51
|
`;
|
|
45
52
|
|
|
46
|
-
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand };
|
|
53
|
+
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand };
|
|
47
54
|
|
|
48
55
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
49
56
|
let parsed;
|
|
@@ -62,6 +69,8 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
62
69
|
max: { type: 'string', default: '20' },
|
|
63
70
|
ref: { type: 'string', default: 'HEAD' },
|
|
64
71
|
claim: { type: 'string' },
|
|
72
|
+
'include-dirty': { type: 'boolean', default: false },
|
|
73
|
+
verbose: { type: 'boolean', default: false },
|
|
65
74
|
'runner-cmd': { type: 'string' },
|
|
66
75
|
'node-modules': { type: 'string' },
|
|
67
76
|
'in-place': { type: 'boolean', default: false },
|
|
@@ -100,7 +109,8 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
100
109
|
return 3;
|
|
101
110
|
}
|
|
102
111
|
try {
|
|
103
|
-
|
|
112
|
+
const projectDir = command === 'scaffold' ? resolve('.') : resolve(dirArg ?? '.');
|
|
113
|
+
return await handler({ projectDir, file: dirArg, values, version: VERSION }, io);
|
|
104
114
|
} catch (e) {
|
|
105
115
|
if (e instanceof ClaimsError || e instanceof PreconditionError || e instanceof GitError || e instanceof SpecDocError) {
|
|
106
116
|
io.err(`error: ${e.message}`);
|
|
@@ -15,7 +15,7 @@ export async function baselineCommand({ projectDir, values }, io) {
|
|
|
15
15
|
const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
|
|
16
16
|
writeSpecDoc('baseline', outPath, baseline);
|
|
17
17
|
const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
|
|
18
|
-
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)} → ${outPath}`);
|
|
18
|
+
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''} → ${outPath}`);
|
|
19
19
|
io.out('Commit this file; from now on only new findings gate. Ignore the regenerated ones — add to .gitignore:');
|
|
20
20
|
io.out(' .testguard/evidence.json');
|
|
21
21
|
io.out(' .testguard/brief.json');
|
package/src/commands/claims.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { resolve } from 'node:path';
|
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
3
|
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
4
4
|
import { resolveDefenders } from '../probe/runner-vitest.mjs';
|
|
5
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
5
6
|
|
|
6
7
|
export async function claimsCommand({ projectDir, values }, io) {
|
|
7
8
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
@@ -16,8 +17,9 @@ export async function claimsCommand({ projectDir, values }, io) {
|
|
|
16
17
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
17
18
|
io.out('');
|
|
18
19
|
for (const c of claims.claims) {
|
|
19
|
-
const
|
|
20
|
-
const
|
|
20
|
+
const declared = c.defendedBy?.length > 0;
|
|
21
|
+
const defenders = declared ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
|
|
22
|
+
const cover = defenders.length ? `${defenders.length} ${declared ? 'defender' : 'discovered'}${defenders.length === 1 ? '' : 's'}` : 'NO DEFENDER';
|
|
21
23
|
io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
|
|
22
24
|
}
|
|
23
25
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
package/src/commands/probe.mjs
CHANGED
|
@@ -41,7 +41,12 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
41
41
|
only,
|
|
42
42
|
escalate: !values['no-escalate'],
|
|
43
43
|
toolVersion: version,
|
|
44
|
-
|
|
44
|
+
includeDirty: values['include-dirty'],
|
|
45
|
+
onStage: !values.quiet && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
46
|
+
onProgress: values.quiet ? undefined : (r) => {
|
|
47
|
+
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
48
|
+
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r) + (r.reusedFrom ? ' (reused)' : ''));
|
|
49
|
+
},
|
|
45
50
|
});
|
|
46
51
|
writeSpecDoc('evidence', outPath, evidence);
|
|
47
52
|
|
|
@@ -52,7 +57,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
52
57
|
for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out(' ' + tag(r) + renderRecord(r));
|
|
53
58
|
}
|
|
54
59
|
io.out('');
|
|
55
|
-
|
|
60
|
+
if (!values.quiet && !values.verbose) {
|
|
61
|
+
const killed = evidence.records.filter((r) => r.verdict === 'killed').length;
|
|
62
|
+
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
63
|
+
}
|
|
64
|
+
io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
|
|
56
65
|
io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : ''}`);
|
|
57
66
|
return g.new.length > 0 ? 1 : 0;
|
|
58
67
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join, resolve, relative, basename, extname } from 'node:path';
|
|
3
|
+
import { scaffoldFile } from '../scaffold/scaffold.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { PreconditionError } from '../probe/worktree.mjs';
|
|
7
|
+
|
|
8
|
+
export async function scaffoldCommand({ projectDir, file, values, version }, io) {
|
|
9
|
+
if (!file) {
|
|
10
|
+
io.err('usage: testguard scaffold <source-file> [--claim <ID>] [--out <path>] [--json]');
|
|
11
|
+
return 3;
|
|
12
|
+
}
|
|
13
|
+
const abs = resolve(file);
|
|
14
|
+
if (!existsSync(abs)) throw new PreconditionError(`no such file: ${file}`);
|
|
15
|
+
const rel = relative(projectDir, abs);
|
|
16
|
+
if (rel.startsWith('..')) throw new PreconditionError(`${file} is outside the project directory ${projectDir}`);
|
|
17
|
+
|
|
18
|
+
const claimsPath = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
19
|
+
const existingClaims = existsSync(claimsPath) ? loadClaims(claimsPath) : undefined;
|
|
20
|
+
const { doc, stats } = scaffoldFile({ projectDir, file: rel, claimId: values.claim, existingClaims, toolVersion: version });
|
|
21
|
+
|
|
22
|
+
if (values.json) {
|
|
23
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
const outPath = values.out ? resolve(values.out) : join(projectDir, '.testguard', `scaffold-${basename(rel, extname(rel))}.json`);
|
|
27
|
+
writeSpecDoc('claims', outPath, doc);
|
|
28
|
+
const shapes = Object.entries(stats.byClass).map(([k, v]) => `${v} ${k}`).join(', ');
|
|
29
|
+
io.out(`${stats.proposals} proposed fault${stats.proposals === 1 ? '' : 's'} in ${stats.claims} draft claim${stats.claims === 1 ? '' : 's'} for ${rel}${shapes ? ` — ${shapes}` : ''}`);
|
|
30
|
+
io.out(stats.defendedBy.length ? `defendedBy prefilled from imports: ${stats.defendedBy.join(', ')}` : 'no test file imports this module; probing the draft as-is will report NOCOVER');
|
|
31
|
+
io.out(`draft: ${outPath}`);
|
|
32
|
+
io.out('Next: replace each TODO statement, drop proposals that are not claims, then move the claims into testguard.claims.json.');
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
package/src/git.mjs
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import { rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { randomBytes } from 'node:crypto';
|
|
3
6
|
|
|
4
7
|
export class GitError extends Error {}
|
|
5
8
|
|
|
6
|
-
export function git(args, cwd) {
|
|
7
|
-
const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
|
|
9
|
+
export function git(args, cwd, env) {
|
|
10
|
+
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: env ? { ...process.env, ...env } : process.env });
|
|
8
11
|
if (r.status !== 0) throw new GitError(`git ${args.join(' ')}: ${(r.stderr || r.stdout).trim()}`);
|
|
9
12
|
return r.stdout.trim();
|
|
10
13
|
}
|
|
@@ -25,6 +28,29 @@ export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--',
|
|
|
25
28
|
|
|
26
29
|
export const addWorktree = (repo, dest, ref = 'HEAD') => git(['worktree', 'add', '--detach', dest, ref], repo);
|
|
27
30
|
|
|
31
|
+
/**
|
|
32
|
+
* A dangling commit holding the working tree exactly as it is — tracked
|
|
33
|
+
* changes AND untracked (non-ignored) files — without touching HEAD, the
|
|
34
|
+
* index, or any ref. Built through a temporary index so the user's staging
|
|
35
|
+
* area is never read or written.
|
|
36
|
+
*/
|
|
37
|
+
export function snapshotWorkingTree(repo) {
|
|
38
|
+
const index = join(tmpdir(), `testguard-index-${randomBytes(6).toString('hex')}`);
|
|
39
|
+
const env = {
|
|
40
|
+
GIT_INDEX_FILE: index,
|
|
41
|
+
GIT_AUTHOR_NAME: 'testguard', GIT_AUTHOR_EMAIL: 'testguard@localhost',
|
|
42
|
+
GIT_COMMITTER_NAME: 'testguard', GIT_COMMITTER_EMAIL: 'testguard@localhost',
|
|
43
|
+
};
|
|
44
|
+
try {
|
|
45
|
+
git(['read-tree', 'HEAD'], repo, env);
|
|
46
|
+
git(['add', '-A', '--', '.'], repo, env);
|
|
47
|
+
const tree = git(['write-tree'], repo, env);
|
|
48
|
+
return git(['commit-tree', tree, '-p', 'HEAD', '-m', 'testguard: working-tree snapshot (not a ref)'], repo, env);
|
|
49
|
+
} finally {
|
|
50
|
+
rmSync(index, { force: true });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export function removeWorktree(repo, dest) {
|
|
29
55
|
spawnSync('git', ['worktree', 'remove', '--force', dest], { cwd: repo });
|
|
30
56
|
rmSync(dest, { recursive: true, force: true });
|
package/src/probe/classify.mjs
CHANGED
|
@@ -15,7 +15,8 @@ export function classify({ defenders, anchor, baselineRuns, probeRuns, confirmRu
|
|
|
15
15
|
}
|
|
16
16
|
const loadError = probeRuns.find((r) => r.outcome === 'error');
|
|
17
17
|
if (loadError) {
|
|
18
|
-
|
|
18
|
+
// esbuild/vitest wording: "Transform failed with 1 error", `Expected ")" but found ";"`, "Unexpected token"
|
|
19
|
+
const parseError = /syntax|parse|transform failed|expected .+ but found|unexpected token/i.test(loadError.loadMessage ?? '');
|
|
19
20
|
return { verdict: 'fault-invalid', reason: parseError ? 'replacement-does-not-compile' : 'suite-failed-to-load' };
|
|
20
21
|
}
|
|
21
22
|
if (probeRuns.some((r) => r.outcome === 'timeout' || r.timeouts > 0)) return { verdict: 'timeout', reason: 'test-timed-out' };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { listTestFiles } from './runner-vitest.mjs';
|
|
3
|
+
import { fileImports } from './rank.mjs';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* When a claim declares no defenders: the test files that import the fault's
|
|
7
|
+
* target file, directly, by relative path or through a resolved alias. This
|
|
8
|
+
* is what `nocover` measures against — no test file even imports the source.
|
|
9
|
+
*/
|
|
10
|
+
export function discoverDefenders(projectDir, targetRel) {
|
|
11
|
+
return listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
|
|
12
|
+
}
|
package/src/probe/probe.mjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
1
|
+
import { existsSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { git } from '../git.mjs';
|
|
2
3
|
import { createRequire } from 'node:module';
|
|
3
4
|
import { join, relative, resolve } from 'node:path';
|
|
4
|
-
import { repoRoot as gitRoot, headSha, isDirty } from '../git.mjs';
|
|
5
|
+
import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../git.mjs';
|
|
6
|
+
import { discoverDefenders } from './discover.mjs';
|
|
5
7
|
import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
|
|
6
8
|
import { applyFault, locate } from './inject.mjs';
|
|
7
9
|
import * as vitest from './runner-vitest.mjs';
|
|
@@ -34,13 +36,17 @@ export async function probe({
|
|
|
34
36
|
runnerCommand,
|
|
35
37
|
nodeModules,
|
|
36
38
|
only,
|
|
39
|
+
includeDirty = false,
|
|
40
|
+
onStage = () => {},
|
|
37
41
|
escalate = true,
|
|
38
42
|
scratchBase,
|
|
39
43
|
toolVersion = '0.0.0',
|
|
40
44
|
previous,
|
|
41
45
|
onProgress = () => {},
|
|
42
46
|
}) {
|
|
43
|
-
|
|
47
|
+
// realpath: git reports the repository root by its real path (/private/var
|
|
48
|
+
// on macOS, not /var); every relative() below must start from the same place.
|
|
49
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
44
50
|
const root = gitRoot(projectDir);
|
|
45
51
|
if (mode === 'in-place' && ref !== 'HEAD') throw new PreconditionError('--ref needs a scratch worktree; drop --in-place');
|
|
46
52
|
const head = headSha(root, ref);
|
|
@@ -50,6 +56,8 @@ export async function probe({
|
|
|
50
56
|
if (mode === 'in-place' && isDirty(root, targets)) {
|
|
51
57
|
throw new PreconditionError(`uncommitted changes in fault target files (${targets.join(', ')}); commit or stash them, or drop --in-place. Only the files faults are applied to must be clean — test files may be dirty, which is what makes --in-place usable while writing tests.`);
|
|
52
58
|
}
|
|
59
|
+
if (includeDirty && mode !== 'worktree') throw new PreconditionError('--include-dirty applies to worktree mode; drop --in-place');
|
|
60
|
+
if (includeDirty && ref !== 'HEAD') throw new PreconditionError('--include-dirty snapshots the working tree; it cannot be combined with --ref');
|
|
53
61
|
const selected = only ? new Set(only) : null;
|
|
54
62
|
if (selected) {
|
|
55
63
|
const known = new Set(claims.claims.map((c) => c.id));
|
|
@@ -57,8 +65,28 @@ export async function probe({
|
|
|
57
65
|
if (unknown.length) throw new PreconditionError(`--claim: unknown claim id(s) ${unknown.join(', ')}`);
|
|
58
66
|
}
|
|
59
67
|
|
|
68
|
+
// Worktree mode probes a commit, not the working tree. Uncommitted defender
|
|
69
|
+
// or target edits would be silently absent — the same survivors, no hint why.
|
|
70
|
+
let snapshot;
|
|
71
|
+
if (mode === 'worktree') {
|
|
72
|
+
if (includeDirty) {
|
|
73
|
+
if (isDirty(root)) snapshot = snapshotWorkingTree(root);
|
|
74
|
+
} else if (ref === 'HEAD') {
|
|
75
|
+
const watched = new Set(targets);
|
|
76
|
+
for (const claim of claims.claims) {
|
|
77
|
+
if (selected && !selected.has(claim.id)) continue;
|
|
78
|
+
const declared = claim.defendedBy?.length ? vitest.resolveDefenders(projectDir, claim.defendedBy) : claim.faults.flatMap((f) => discoverDefenders(projectDir, f.file));
|
|
79
|
+
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
80
|
+
for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
81
|
+
}
|
|
82
|
+
const dirty = git(['status', '--porcelain', '--', ...watched], root).split('\n').filter(Boolean).map((l) => l.replace(/^[ MADRCU?!]{1,2}\s+/, '').replace(/^.* -> /, ''));
|
|
83
|
+
if (dirty.length) {
|
|
84
|
+
throw new PreconditionError(`${dirty.length} defender/target file${dirty.length === 1 ? ' has' : 's have'} uncommitted changes (${dirty.join(', ')}); worktree mode probes HEAD (${head.slice(0, 7)}), so those changes would be silently ignored. Commit them, run with --include-dirty to probe the working tree, or use --in-place.`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
60
88
|
const startedAt = new Date().toISOString();
|
|
61
|
-
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
89
|
+
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
62
90
|
const records = [];
|
|
63
91
|
let runnerVersion;
|
|
64
92
|
try {
|
|
@@ -81,9 +109,11 @@ export async function probe({
|
|
|
81
109
|
|
|
82
110
|
for (const claim of claims.claims) {
|
|
83
111
|
if (selected && !selected.has(claim.id)) continue;
|
|
84
|
-
const
|
|
112
|
+
const declared = claim.defendedBy?.length ? vitest.resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
85
113
|
for (const fault of claim.faults) {
|
|
86
|
-
const
|
|
114
|
+
const defenders = declared ?? discoverDefenders(iso.projectDir, fault.file);
|
|
115
|
+
const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
|
|
116
|
+
const record = await probeOne({ claim, fault, defenders, discovered: declared === null, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id });
|
|
87
117
|
records.push(record);
|
|
88
118
|
onProgress(record);
|
|
89
119
|
}
|
|
@@ -99,7 +129,7 @@ export async function probe({
|
|
|
99
129
|
id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
|
|
100
130
|
startedAt,
|
|
101
131
|
finishedAt: new Date().toISOString(),
|
|
102
|
-
repo: { head, dirty: isDirty(root) },
|
|
132
|
+
repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
|
|
103
133
|
runner: { name: vitest.name, ...((runnerVersion ?? readRunnerVersion(projectDir)) ? { version: runnerVersion ?? readRunnerVersion(projectDir) } : {}) },
|
|
104
134
|
confirmRuns,
|
|
105
135
|
mode,
|
|
@@ -108,7 +138,7 @@ export async function probe({
|
|
|
108
138
|
};
|
|
109
139
|
}
|
|
110
140
|
|
|
111
|
-
async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, prior, priorRunId }) {
|
|
141
|
+
async function probeOne({ claim, fault, defenders, discovered, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior, priorRunId }) {
|
|
112
142
|
const targetPath = join(iso.projectDir, fault.file);
|
|
113
143
|
const targetExists = existsSync(targetPath);
|
|
114
144
|
const inputs = {
|
|
@@ -134,6 +164,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
134
164
|
const runs = [];
|
|
135
165
|
let loadMessage;
|
|
136
166
|
for (let i = 0; i < confirmRuns; i++) {
|
|
167
|
+
stage('baseline', i + 1, confirmRuns);
|
|
137
168
|
const res = await runDefenders(defenders);
|
|
138
169
|
runs.push(res.run);
|
|
139
170
|
if (res.run.outcome !== 'pass') {
|
|
@@ -156,6 +187,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
156
187
|
try {
|
|
157
188
|
const probeRuns = rawProbeRuns;
|
|
158
189
|
for (let i = 0; i < confirmRuns; i++) {
|
|
190
|
+
stage('probe', i + 1, confirmRuns);
|
|
159
191
|
const { run, timeouts, loadMessage } = await runDefenders(defenders);
|
|
160
192
|
probeRuns.push({ ...run, timeouts, loadMessage });
|
|
161
193
|
if (shouldStopEarly(probeRuns)) break;
|
|
@@ -171,6 +203,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
171
203
|
const runs = [];
|
|
172
204
|
let killers = null;
|
|
173
205
|
for (let i = 0; i < confirmRuns; i++) {
|
|
206
|
+
stage('escalation', i + 1, confirmRuns);
|
|
174
207
|
const { run, failedTests } = await runDefenders(allTests);
|
|
175
208
|
runs.push(run);
|
|
176
209
|
killers = killers === null ? new Set(failedTests) : new Set(failedTests.filter((t) => killers.has(t)));
|
|
@@ -178,7 +211,10 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
178
211
|
}
|
|
179
212
|
detail.escalated = true;
|
|
180
213
|
detail.escalationRuns = runs;
|
|
181
|
-
if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill))
|
|
214
|
+
if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill)) {
|
|
215
|
+
detail.reason = 'killed-by-undeclared-tests';
|
|
216
|
+
detail.undeclaredKillers = [...killers].sort();
|
|
217
|
+
}
|
|
182
218
|
}
|
|
183
219
|
} finally {
|
|
184
220
|
mutation.restore();
|
|
@@ -189,6 +225,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
189
225
|
|
|
190
226
|
const { verdict, reason } = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns: rawProbeRuns, confirmRuns });
|
|
191
227
|
if (reason && !detail.reason) detail.reason = reason;
|
|
228
|
+
if (anchor && anchor.status !== 'ok' && anchor.status !== 'file-missing' && anchor.status !== 'defenders-failed-to-load') detail.anchor = { hits: anchor.hits, expected: anchor.expected };
|
|
192
229
|
|
|
193
230
|
const blast = targetExists ? blastRadius(iso.projectDir, fault.file) : 0;
|
|
194
231
|
|
|
@@ -198,7 +235,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
198
235
|
subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy },
|
|
199
236
|
verdict,
|
|
200
237
|
detail,
|
|
201
|
-
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0 },
|
|
238
|
+
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}) },
|
|
202
239
|
inputs,
|
|
203
240
|
rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast }),
|
|
204
241
|
};
|
package/src/probe/rank.mjs
CHANGED
|
@@ -94,6 +94,20 @@ function resolvesTo(fromFile, specifier, targetAbs, aliases) {
|
|
|
94
94
|
return bases.some((b) => expandBase(b).includes(targetAbs));
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
+
/** Does the file at `absFile` import `targetRel` (relative or alias-resolved)? */
|
|
98
|
+
export function fileImports(projectDir, absFile, targetRel) {
|
|
99
|
+
const targetAbs = resolve(projectDir, targetRel);
|
|
100
|
+
const aliases = loadAliases(projectDir);
|
|
101
|
+
let src;
|
|
102
|
+
try {
|
|
103
|
+
src = readFileSync(absFile, 'utf8');
|
|
104
|
+
} catch {
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
for (const m of src.matchAll(IMPORT_RE)) if (resolvesTo(absFile, m[1], targetAbs, aliases)) return true;
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
|
|
97
111
|
/**
|
|
98
112
|
* Number of non-test source files that import `targetRel`: relative specifiers,
|
|
99
113
|
* tsconfig/jsconfig `paths` aliases and package.json `imports` are resolved;
|
package/src/render.mjs
CHANGED
|
@@ -5,7 +5,16 @@ export const formatVerdict = (v) => (v === 'killed' ? 'killed' : v.toUpperCase()
|
|
|
5
5
|
|
|
6
6
|
export function renderRecord(r) {
|
|
7
7
|
const head = `${formatVerdict(r.verdict).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
|
|
8
|
-
|
|
8
|
+
let why = '';
|
|
9
|
+
if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
|
|
10
|
+
const files = [...new Set(r.detail.undeclaredKillers.map((k) => k.split('::')[0]))];
|
|
11
|
+
why = ` [killed-by-undeclared-tests: ${files.join(', ')}]`;
|
|
12
|
+
} else if (r.detail.reason === 'anchor-ambiguous' && r.detail.anchor) {
|
|
13
|
+
why = ` [anchor-ambiguous: ${r.detail.anchor.hits} hits, expected ${r.detail.anchor.expected}]`;
|
|
14
|
+
} else if (r.detail.reason) {
|
|
15
|
+
why = ` [${r.detail.reason}]`;
|
|
16
|
+
}
|
|
17
|
+
if (r.defenders.discovered) why += ' (defenders discovered by import)';
|
|
9
18
|
return `${head} ${r.claim.severity.padEnd(8)} ${r.subject.file} ${r.subject.description}${why}`;
|
|
10
19
|
}
|
|
11
20
|
|
|
@@ -15,12 +24,13 @@ export function summarize(records) {
|
|
|
15
24
|
return byVerdict;
|
|
16
25
|
}
|
|
17
26
|
|
|
18
|
-
export function renderSummary(records) {
|
|
27
|
+
export function renderSummary(records, run) {
|
|
19
28
|
const byVerdict = summarize(records);
|
|
20
29
|
const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v)}`);
|
|
21
30
|
const unproven = records.filter((r) => r.verdict !== 'killed');
|
|
22
31
|
const claims = new Set(unproven.map((r) => r.claim.id)).size;
|
|
23
|
-
|
|
32
|
+
const where = run ? ` Probed ${run.repo.snapshot ? `working tree (snapshot ${run.repo.snapshot.slice(0, 7)} of ${run.repo.head.slice(0, 7)})` : run.mode === 'in-place' ? `in place at ${run.repo.head.slice(0, 7)}${run.repo.dirty ? ' (dirty)' : ''}` : run.repo.head.slice(0, 7)}.` : '';
|
|
33
|
+
return `${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
/** Survivors first, then by rank score; killed last. */
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mechanical fault producers. Deterministic, line-oriented, no AST, no LLM.
|
|
3
|
+
*
|
|
4
|
+
* Two independent field reports found that ~80% of hand-written faults are
|
|
5
|
+
* one of these shapes. Each producer looks at one line (plus a little
|
|
6
|
+
* context) and proposes a fault whose `find` is the exact line, so the anchor
|
|
7
|
+
* hits by construction. A human keeps or drops every proposal.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const COMMENT = /^\s*(\/\/|\*|\/\*)/;
|
|
11
|
+
const GUARD_BODY = /\b(return|throw)\b|\.status\(\s*4\d\d|\bredirect\(|\bfalse\b|\bnull\b/;
|
|
12
|
+
const IF_LINE = /^(\s*)(?:\}\s*)?(?:else\s+)?if\s*\((.+)\)\s*(\{\s*|(?:return|throw)\b.*;\s*)?$/;
|
|
13
|
+
const RETURN_CHECK = /^\s*return\s+(.+);\s*$/;
|
|
14
|
+
const CHECK_EXPR = /(===|!==|\.includes\(|\.has\(|\.some\(|\.every\(|\.test\(|\.startsWith\(|\.endsWith\(|\binstanceof\b|&&|\|\||^!)/;
|
|
15
|
+
const CHECK_CALL = /^\s*(?:await\s+)?(?:[\w$]+\.)*(verify|validate|assert|check|require|ensure|authoriz|authentic|rateLimit|throttle|enforce|guard)\w*\s*\(.*\)\s*;\s*$/i;
|
|
16
|
+
const MUTATION = /^\s*(?!(?:const|let|var|return|if|for|while|else|switch|case|import|export|throw)\b)[\w$]+(?:[.\[][\w$'"\]]+)*\s*(=|\+=|-=|\|\|=|&&=|\?\?=)\s*(?!=)[^;]*;\s*$/;
|
|
17
|
+
const FLAG_TRUE = /\b(httpOnly|secure|signed|requireTLS|rejectUnauthorized|strict)\s*:\s*true\b/;
|
|
18
|
+
const SAME_SITE = /\bsameSite\s*:\s*(['"])(strict|lax)\1/i;
|
|
19
|
+
const COST_NUM = /\b(\w*(?:cost|rounds|iterations)\w*)\s*([:=])\s*(\d+)\b/i;
|
|
20
|
+
const WINDOW_NUM = /\b(\w*(?:ttl|expir|tolerance|window|maxAge|max_age|timeout|limit|attempts|length|minLength|clockSkew|skew)\w*)\s*([:=])\s*(\d+)\b/i;
|
|
21
|
+
const FUNCTION_HEAD = [
|
|
22
|
+
/^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s*\*?\s*([\w$]+)\s*\(/,
|
|
23
|
+
/^\s*(?:export\s+)?(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[\w$]+)\s*=>/,
|
|
24
|
+
/^\s*(?:export\s+)?(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?function\b/,
|
|
25
|
+
/^\s*(?:public\s+|private\s+|protected\s+)?(?:static\s+)?(?:async\s+)?([\w$]+)\s*\([^)]*\)\s*(?::\s*[\w<>\[\]| ]+)?\s*\{\s*$/,
|
|
26
|
+
];
|
|
27
|
+
const NOT_A_FUNCTION = new Set(['if', 'for', 'while', 'switch', 'catch', 'else', 'return', 'function', 'constructor']);
|
|
28
|
+
|
|
29
|
+
/** Name of the function whose head is on this line, or null. */
|
|
30
|
+
export function functionHead(line) {
|
|
31
|
+
for (const re of FUNCTION_HEAD) {
|
|
32
|
+
const m = re.exec(line);
|
|
33
|
+
if (m && !NOT_A_FUNCTION.has(m[1])) return m[1];
|
|
34
|
+
}
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isGuard(cond, lines, i) {
|
|
39
|
+
if (/^\s*!/.test(cond)) return true;
|
|
40
|
+
const tail = lines.slice(i, i + 4).join('\n');
|
|
41
|
+
return GUARD_BODY.test(tail);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Proposals for one line. Each: { faultClass, description, replace } where find is the line itself. */
|
|
45
|
+
export function proposalsForLine(lines, i) {
|
|
46
|
+
const line = lines[i];
|
|
47
|
+
const out = [];
|
|
48
|
+
if (!line.trim() || COMMENT.test(line)) return out;
|
|
49
|
+
|
|
50
|
+
const ifm = IF_LINE.exec(line);
|
|
51
|
+
if (ifm && isGuard(ifm[2], lines, i)) {
|
|
52
|
+
const singleLine = ifm[3] && /^(return|throw)\b/.test(ifm[3].trim());
|
|
53
|
+
if (singleLine) {
|
|
54
|
+
out.push({ faultClass: 'statement-deleted', description: `Guard removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
55
|
+
} else {
|
|
56
|
+
out.push({ faultClass: 'condition-forced', description: `Guard never triggers: \`if (${ifm[2].trim()})\` becomes \`if (false)\`.`, replace: line.replace(`(${ifm[2]})`, '(false)') });
|
|
57
|
+
}
|
|
58
|
+
return out; // an `if` line is not also a mutation/return/call
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const rm = RETURN_CHECK.exec(line);
|
|
62
|
+
if (rm && CHECK_EXPR.test(rm[1]) && !/^(new|await)\b/.test(rm[1].trim())) {
|
|
63
|
+
out.push({ faultClass: 'return-altered', description: `Check always passes: \`return ${rm[1].trim()}\` becomes \`return true\`.`, replace: line.replace(/return\s+.+;/, 'return true;') });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (CHECK_CALL.test(line)) {
|
|
67
|
+
out.push({ faultClass: 'call-removed', description: `Check call removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
68
|
+
} else if (MUTATION.test(line)) {
|
|
69
|
+
out.push({ faultClass: 'statement-deleted', description: `State change removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let m;
|
|
73
|
+
if ((m = FLAG_TRUE.exec(line))) {
|
|
74
|
+
out.push({ faultClass: 'literal-changed', description: `Security flag flipped: \`${m[1]}: true\` becomes \`${m[1]}: false\`.`, replace: line.replace(m[0], `${m[1]}: false`) });
|
|
75
|
+
}
|
|
76
|
+
if ((m = SAME_SITE.exec(line))) {
|
|
77
|
+
out.push({ faultClass: 'literal-changed', description: `sameSite weakened: \`${m[2]}\` becomes \`none\`.`, replace: line.replace(m[0], `sameSite: ${m[1]}none${m[1]}`) });
|
|
78
|
+
}
|
|
79
|
+
if ((m = COST_NUM.exec(line))) {
|
|
80
|
+
out.push({ faultClass: 'literal-changed', description: `Work factor collapsed: \`${m[1]}\` ${m[3]} becomes 1.`, replace: line.replace(m[0], `${m[1]}${m[2] === ':' ? ': ' : ' = '}1`) });
|
|
81
|
+
} else if ((m = WINDOW_NUM.exec(line))) {
|
|
82
|
+
out.push({ faultClass: 'literal-changed', description: `Window widened ×1000: \`${m[1]}\` ${m[3]} becomes ${Number(m[3]) * 1000}.`, replace: line.replace(m[0], `${m[1]}${m[2] === ':' ? ': ' : ' = '}${Number(m[3]) * 1000}`) });
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { basename, extname, join } from 'node:path';
|
|
3
|
+
import { proposalsForLine, functionHead } from './producers.mjs';
|
|
4
|
+
import { locate } from '../probe/inject.mjs';
|
|
5
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
6
|
+
import { validate } from '../../spec/lib/validate.mjs';
|
|
7
|
+
|
|
8
|
+
const ANNOTATION = /@claim\s+([A-Za-z0-9]+(?:[._]?[A-Za-z0-9]+)*-[A-Za-z0-9._-]*[A-Za-z0-9])\b/;
|
|
9
|
+
|
|
10
|
+
const idPart = (s) => s.replace(/[^A-Za-z0-9]+/g, '-').replace(/^-+|-+$/g, '').toUpperCase();
|
|
11
|
+
|
|
12
|
+
/** How many times `find` occurs, and which occurrence the line at `offset` is. */
|
|
13
|
+
function anchorFor(source, find, offset) {
|
|
14
|
+
let hits = 0;
|
|
15
|
+
let occurrence = 0;
|
|
16
|
+
let i = -1;
|
|
17
|
+
while ((i = source.indexOf(find, i + 1)) !== -1) {
|
|
18
|
+
hits++;
|
|
19
|
+
if (i === offset) occurrence = hits;
|
|
20
|
+
}
|
|
21
|
+
return { hits, occurrence };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Scan one source file and produce a DRAFT claims document: every proposal is
|
|
26
|
+
* an exact-line anchor that `locate()` accepts, grouped by enclosing function
|
|
27
|
+
* (or by a preceding `@claim <ID>` annotation, or entirely under `claimId`),
|
|
28
|
+
* with `producedBy: { producer: "derived" }` and TODO statements a human must
|
|
29
|
+
* replace. Never touches the real claims file.
|
|
30
|
+
*/
|
|
31
|
+
export function scaffoldFile({ projectDir, file, claimId, existingClaims, toolVersion = '0.0.0' }) {
|
|
32
|
+
const source = readFileSync(join(projectDir, file), 'utf8');
|
|
33
|
+
const lines = source.split('\n');
|
|
34
|
+
const stem = idPart(basename(file, extname(file)));
|
|
35
|
+
const producedBy = { producer: 'derived', by: `testguard scaffold ${toolVersion}` };
|
|
36
|
+
|
|
37
|
+
const groups = new Map(); // key → { id, proposals[] , annotated }
|
|
38
|
+
const groupFor = (key, id) => {
|
|
39
|
+
if (!groups.has(key)) groups.set(key, { id, proposals: [] });
|
|
40
|
+
return groups.get(key);
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
let offset = 0;
|
|
44
|
+
let fn = null;
|
|
45
|
+
let fnDepth = 0;
|
|
46
|
+
let depth = 0;
|
|
47
|
+
let pendingAnnotation = null;
|
|
48
|
+
const proposals = [];
|
|
49
|
+
lines.forEach((line, i) => {
|
|
50
|
+
const ann = ANNOTATION.exec(line);
|
|
51
|
+
if (ann && /^\s*(\/\/|\*|\/\*)/.test(line)) pendingAnnotation = ann[1];
|
|
52
|
+
|
|
53
|
+
const head = functionHead(line);
|
|
54
|
+
if (head) {
|
|
55
|
+
fn = head;
|
|
56
|
+
fnDepth = depth;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for (const p of proposalsForLine(lines, i)) {
|
|
60
|
+
const find = line;
|
|
61
|
+
const { hits, occurrence } = anchorFor(source, find, offset);
|
|
62
|
+
const fault = { ...p, find, replace: p.replace, expectHits: hits, occurrence, line: i + 1, fn, annotation: pendingAnnotation };
|
|
63
|
+
if (locate(source, fault).status !== 'ok') continue; // never propose an anchor that would be unverifiable
|
|
64
|
+
proposals.push(fault);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
68
|
+
if (fn && depth <= fnDepth && !head) fn = null;
|
|
69
|
+
if (pendingAnnotation && !ann && !head && line.trim() && !/^\s*(\/\/|\*|\/\*)/.test(line)) pendingAnnotation = null;
|
|
70
|
+
offset += line.length + 1;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const existing = new Map((existingClaims?.claims ?? []).map((c) => [c.id, c]));
|
|
74
|
+
const usedIds = new Set();
|
|
75
|
+
for (const p of proposals) {
|
|
76
|
+
const key = claimId ? '__all__' : p.annotation ?? p.fn ?? '__file__';
|
|
77
|
+
let id = claimId ?? p.annotation ?? `${stem}-${p.fn ? idPart(p.fn) : 'FILE'}`;
|
|
78
|
+
groupFor(key, id).proposals.push(p);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const defendedBy = discoverDefenders(projectDir, file);
|
|
82
|
+
const claims = [];
|
|
83
|
+
for (const g of groups.values()) {
|
|
84
|
+
let id = g.id;
|
|
85
|
+
while (usedIds.has(id)) id += '-2';
|
|
86
|
+
usedIds.add(id);
|
|
87
|
+
const base = existing.get(id);
|
|
88
|
+
const fnLabel = g.proposals[0].fn ? `\`${g.proposals[0].fn}\`` : 'this file';
|
|
89
|
+
const claim = {
|
|
90
|
+
id,
|
|
91
|
+
statement: base?.statement ?? `TODO: state what ${fnLabel} in ${file} guarantees (${g.proposals.length} proposed fault${g.proposals.length === 1 ? '' : 's'}; keep or drop each)`,
|
|
92
|
+
source: base?.source ?? { kind: 'manual', ref: `testguard scaffold ${file}` },
|
|
93
|
+
severity: base?.severity ?? 'medium',
|
|
94
|
+
producedBy: base?.producedBy ?? producedBy,
|
|
95
|
+
...(base?.defendedBy?.length ? { defendedBy: base.defendedBy } : defendedBy.length ? { defendedBy } : {}),
|
|
96
|
+
faults: g.proposals.map((p, n) => ({
|
|
97
|
+
id: `S${n + 1}`,
|
|
98
|
+
description: `[line ${p.line}] ${p.description}`,
|
|
99
|
+
faultClass: p.faultClass,
|
|
100
|
+
file,
|
|
101
|
+
find: p.find,
|
|
102
|
+
replace: p.replace,
|
|
103
|
+
...(p.expectHits > 1 ? { expectHits: p.expectHits, occurrence: p.occurrence } : {}),
|
|
104
|
+
producedBy,
|
|
105
|
+
})),
|
|
106
|
+
...(base?.tags ? { tags: base.tags } : {}),
|
|
107
|
+
};
|
|
108
|
+
claims.push(claim);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const doc = { $schema: './node_modules/testguard-cli/spec/schemas/claims.schema.json', schemaVersion: 1, claims };
|
|
112
|
+
const result = validate('claims', doc);
|
|
113
|
+
if (!result.ok) throw new Error(`scaffold produced a non-conforming draft:\n${result.errors.map((e) => ` ${e.path}: ${e.message}`).join('\n')}`);
|
|
114
|
+
|
|
115
|
+
const byClass = {};
|
|
116
|
+
for (const p of proposals) byClass[p.faultClass] = (byClass[p.faultClass] ?? 0) + 1;
|
|
117
|
+
return { doc, stats: { proposals: proposals.length, claims: claims.length, byClass, defendedBy } };
|
|
118
|
+
}
|