testguard-cli 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +47 -8
- package/cli/testguard.mjs +8 -0
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +12 -6
- package/spec/lib/validate.mjs +3 -0
- package/spec/schemas/baseline.schema.json +4 -0
- package/spec/schemas/claims.schema.json +1 -1
- package/spec/schemas/common.schema.json +1 -1
- package/spec/schemas/evidence.schema.json +26 -2
- package/src/baseline/baseline.mjs +1 -1
- package/src/brief/brief.mjs +1 -1
- package/src/cli.mjs +11 -1
- package/src/commands/baseline.mjs +4 -2
- package/src/commands/claims.mjs +7 -4
- package/src/commands/probe.mjs +19 -5
- package/src/git.mjs +28 -2
- package/src/probe/classify.mjs +2 -1
- package/src/probe/discover.mjs +12 -0
- package/src/probe/probe.mjs +87 -19
- package/src/probe/rank.mjs +100 -7
- package/src/probe/runner-vitest.mjs +61 -2
- package/src/probe/worktree.mjs +28 -11
- package/src/render.mjs +15 -4
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,97 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.1.3] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
From a second field report on a real codebase (456 tests, 27 claims, 35
|
|
13
|
+
faults; 13 survived on the first run, two critical claims with zero coverage).
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- **Worktree mode probed HEAD while reading the claims file from the working
|
|
18
|
+
tree**, so uncommitted defender changes were silently ignored — the same
|
|
19
|
+
survivors came back with no hint why. `probe` now refuses (exit 2) when any
|
|
20
|
+
resolved defender or fault target has uncommitted changes, naming the files
|
|
21
|
+
and the commit it would have probed. Every summary names the commit probed.
|
|
22
|
+
- `killed-by-undeclared-tests` never said which tests killed the fault; the
|
|
23
|
+
author could not fix `defendedBy` without grepping the suite. Evidence now
|
|
24
|
+
carries `detail.undeclaredKillers` and the CLI names the files.
|
|
25
|
+
- `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
|
|
26
|
+
expected }` is recorded and printed.
|
|
27
|
+
- A replacement with an unbalanced paren was `suite-failed-to-load`, not
|
|
28
|
+
`replacement-does-not-compile`: esbuild/vitest wording is now matched.
|
|
29
|
+
- The claims schema promised defender discovery for an absent `defendedBy`;
|
|
30
|
+
the tool answered `nocover`. Discovery is implemented: the test files that
|
|
31
|
+
import the fault's target (relative or alias), recorded as
|
|
32
|
+
`defenders.discovered`. `nocover` now means exactly "no test file imports
|
|
33
|
+
this source".
|
|
34
|
+
- `baseline.json` records `dirty`, as evidence already did.
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- `--include-dirty`: snapshot the working tree (tracked edits and untracked,
|
|
39
|
+
non-ignored files) into a throwaway commit and probe that. HEAD, index and
|
|
40
|
+
the user's tree are never touched; `run.repo.snapshot` records the commit.
|
|
41
|
+
- A one-line progress indicator on stderr (TTY only) so a minute of silence
|
|
42
|
+
is not mistaken for a hang.
|
|
43
|
+
- Fixture: a claim with no `defendedBy` whose defender is discovered.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- **Default output shows only unproven faults plus a killed count.**
|
|
48
|
+
`--verbose` restores the full stream.
|
|
49
|
+
- The `survived` hint reminds the author to check that the fault is
|
|
50
|
+
observable at all before writing a test for it.
|
|
51
|
+
- README: `$schema` path for consumers, `min-release-age` note, the
|
|
52
|
+
worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
|
|
53
|
+
|
|
54
|
+
## [0.1.2] - 2026-09-17
|
|
55
|
+
|
|
56
|
+
From a field report on a real codebase (63 test files, 458 tests, 39 faults).
|
|
57
|
+
|
|
58
|
+
### Fixed
|
|
59
|
+
|
|
60
|
+
- **A runner that cannot be resolved was reported as `FLAKY-DEFENDER`.** In
|
|
61
|
+
the scratch worktree, a symlinked `node_modules` (the sibling/auto-worktree
|
|
62
|
+
layout) was invisible, vitest failed to load, and the load error was
|
|
63
|
+
classified as flaky tests — blaming the wrong party. Symlinked
|
|
64
|
+
`node_modules` are now linked to their resolved target; the runner is
|
|
65
|
+
checked before any verdict and an unresolvable one is a precondition
|
|
66
|
+
failure (exit 2) with the fix in the message; defenders that fail to
|
|
67
|
+
*load* during a baseline are `UNVERIFIABLE` (`defenders-failed-to-load`),
|
|
68
|
+
never flaky.
|
|
69
|
+
- **Blast radius ignored path aliases.** `tsconfig`/`jsconfig` `paths` (with
|
|
70
|
+
`baseUrl` and relative `extends`) and `package.json#imports` are resolved,
|
|
71
|
+
so a module imported 83 times via `@/…` no longer ranks as if nothing
|
|
72
|
+
imported it. Bare package specifiers remain ignored, as documented.
|
|
73
|
+
- Summary said "N unproven claims" when N counted faults; it now reports
|
|
74
|
+
unproven faults *and* the distinct claims they belong to.
|
|
75
|
+
- `testguard claims` never showed which claims carry a `@claim` annotation;
|
|
76
|
+
it now reports the count and marks each annotated row.
|
|
77
|
+
- Piping output to a closed reader (`testguard claims | head`) no longer
|
|
78
|
+
crashes with an `EPIPE` stack trace.
|
|
79
|
+
|
|
80
|
+
### Added
|
|
81
|
+
|
|
82
|
+
- `--claim <ID,ID>` probes only the named claims and writes
|
|
83
|
+
`.testguard/evidence-partial.json`, keeping the canonical evidence intact —
|
|
84
|
+
turns a fix-loop iteration from minutes into seconds.
|
|
85
|
+
- `--runner-cmd "<cmd>"` with `{files}` and `{out}` placeholders for
|
|
86
|
+
monorepos, custom configs and other package managers.
|
|
87
|
+
- `--node-modules <dir>` (or `TESTGUARD_NODE_MODULES`) to link a specific
|
|
88
|
+
`node_modules` into the scratch worktree.
|
|
89
|
+
|
|
90
|
+
### Changed
|
|
91
|
+
|
|
92
|
+
- The `--in-place` precondition message says what it means: only fault
|
|
93
|
+
target files must be clean; test files may be dirty.
|
|
94
|
+
- `testguard baseline` prints the two `.gitignore` lines for the regenerated
|
|
95
|
+
files instead of leaving it to the README.
|
|
96
|
+
- With no baseline, the ranked block is not printed a second time under the
|
|
97
|
+
per-fault stream.
|
|
98
|
+
- `--quiet` is documented precisely.
|
|
99
|
+
- Staged Homebrew formula carries the sha256 of the published 0.1.1 tarball.
|
|
100
|
+
|
|
10
101
|
## [0.1.1] - 2026-09-17
|
|
11
102
|
|
|
12
103
|
### Fixed
|
package/README.md
CHANGED
|
@@ -35,6 +35,15 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
35
35
|
100% coverage, where the one assertion that mattered used
|
|
36
36
|
`expect.objectContaining({...})` and omitted the field carrying the data.
|
|
37
37
|
|
|
38
|
+
A second, independent run on a different AI-authored codebase (63 test
|
|
39
|
+
files, 458 tests, 24 hand-written security claims, 39 faults): **21 of 39
|
|
40
|
+
faults survived a fully green suite — 9 of them critical.** Super-admin
|
|
41
|
+
gating, membership checks, cookie flags and the whole authorization callback
|
|
42
|
+
could be disabled without a single test noticing. One test file had
|
|
43
|
+
re-implemented the authorization logic *inside the test* and asserted
|
|
44
|
+
against the copy: fifteen green tests, zero detection. After wrapper-level
|
|
45
|
+
tests were written against the survivors, 39/39 were killed.
|
|
46
|
+
|
|
38
47
|
## Install
|
|
39
48
|
|
|
40
49
|
| How | Command |
|
|
@@ -43,9 +52,13 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
43
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
44
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
45
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
46
|
-
| GitHub Action | `uses: raccioly/testguard@v0.1.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
|
|
47
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
48
57
|
|
|
58
|
+
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
59
|
+
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
60
|
+
`npm i -D testguard-cli --min-release-age=0`.
|
|
61
|
+
|
|
49
62
|
## How it works
|
|
50
63
|
|
|
51
64
|
```bash
|
|
@@ -55,12 +68,14 @@ npx testguard-cli baseline # freeze today's unproven findings; from now on on
|
|
|
55
68
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
56
69
|
```
|
|
57
70
|
|
|
58
|
-
1. **Claims** live in `testguard.claims.json
|
|
59
|
-
|
|
60
|
-
|
|
71
|
+
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
72
|
+
`"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
|
|
73
|
+
a statement, where it comes from, which tests supposedly defend it, and
|
|
74
|
+
one or more *faults* — each a deterministic source change that would make
|
|
75
|
+
the statement false. Every
|
|
61
76
|
claim and every fault records who produced it. `testguard claims`
|
|
62
77
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
63
|
-
source.
|
|
78
|
+
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
64
79
|
2. **Probe** confirms the defenders are green N times unmodified, applies
|
|
65
80
|
each fault in a scratch git worktree (your tree is never touched), runs
|
|
66
81
|
the defenders N times, re-runs survivors against the whole suite with
|
|
@@ -77,14 +92,38 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
77
92
|
| `FLAKY-DEFENDER` | the defenders are not reliably green, or disagreed across runs |
|
|
78
93
|
|
|
79
94
|
Never a single score. Findings are ranked by severity, claim provenance
|
|
80
|
-
and blast radius
|
|
81
|
-
|
|
95
|
+
and blast radius (relative imports, `tsconfig` path aliases and
|
|
96
|
+
`package.json#imports` are resolved; bare package names are not), and
|
|
97
|
+
written to `.testguard/evidence.json` — validated against the spec before
|
|
98
|
+
it is written.
|
|
99
|
+
|
|
100
|
+
Worktree mode probes a **commit**. If a defender or target file has
|
|
101
|
+
uncommitted changes, `probe` refuses and says so — otherwise your new
|
|
102
|
+
tests would be silently absent and the same survivors would come back
|
|
103
|
+
with no hint why. `--include-dirty` snapshots the working tree (tracked
|
|
104
|
+
edits and new files) into a throwaway commit and probes that; your tree,
|
|
105
|
+
HEAD and index are never touched. Every summary names the commit probed.
|
|
106
|
+
|
|
107
|
+
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
108
|
+
files that import the fault's target, by relative path or resolved alias.
|
|
109
|
+
`NOCOVER` then means exactly "no test file imports this source".
|
|
110
|
+
|
|
111
|
+
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
112
|
+
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
113
|
+
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
114
|
+
clean there; test files may be dirty); final pass with defaults. By default
|
|
115
|
+
the stream shows only unproven faults plus a killed count — `--verbose`
|
|
116
|
+
shows every fault. A custom
|
|
117
|
+
runner (`pnpm --filter`, a specific config) goes in
|
|
118
|
+
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
119
|
+
see your `node_modules`, pass `--node-modules <dir>`.
|
|
82
120
|
3. **Baseline** freezes every non-passing fingerprint. Later probes suppress
|
|
83
121
|
what was already known and exit non-zero only on what is new. Claims whose
|
|
84
122
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
85
123
|
in CI costs only what changed.
|
|
86
124
|
4. **Brief** turns evidence plus baseline into a ranked, capped
|
|
87
|
-
`## TEST BLINDSPOT CONTEXT` block
|
|
125
|
+
`## TEST BLINDSPOT CONTEXT` block, printed and also written to
|
|
126
|
+
`.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
|
|
88
127
|
— for Claude Code, in `.claude/settings.json`:
|
|
89
128
|
|
|
90
129
|
```json
|
package/cli/testguard.mjs
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { main } from '../src/cli.mjs';
|
|
3
3
|
|
|
4
|
+
// `testguard … | head` closes stdout early; that is not an error worth a stack trace.
|
|
5
|
+
for (const stream of [process.stdout, process.stderr]) {
|
|
6
|
+
stream.on('error', (err) => {
|
|
7
|
+
if (err.code === 'EPIPE') process.exit(0);
|
|
8
|
+
throw err;
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
4
12
|
main(process.argv.slice(2)).then((code) => {
|
|
5
13
|
process.exitCode = code;
|
|
6
14
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -12,7 +12,7 @@ The verdict set is closed. Only one value is a pass.
|
|
|
12
12
|
| `killed` | Fault applied; every one of N probe runs failed with a genuine assertion failure, on a defender set that was green N/N unmodified. | no |
|
|
13
13
|
| `survived` | Fault applied; every one of N probe runs passed. The claim is **unproven**. | **yes** |
|
|
14
14
|
| `nocover` | No defending test exists: the declared globs resolve to nothing, or no test imports the subject. Worse than `survived` — nothing was even tried. | **yes** |
|
|
15
|
-
| `unverifiable` | The
|
|
15
|
+
| `unverifiable` | The claim could not be probed: the fault's anchor is missing or ambiguous, or its defenders failed to load (`defenders-failed-to-load`). Carries a `reason`. A loud, gating verdict — a claim that cannot be probed is not "skipped", it is undefended until someone fixes the fault or the defenders. Never confused with `flaky-defender`, which requires tests that *ran*. | **yes** |
|
|
16
16
|
| `timeout` | Probe run exceeded its budget. Not counted as a kill; the pessimistic reading is the safe one because flakiness biases the metric optimistically. | **yes** |
|
|
17
17
|
| `fault-invalid` | The replacement does not load or compile. A bad fault, not a detection. | **yes** |
|
|
18
18
|
| `flaky-defender` | Defenders were not green N/N on unmodified source (`defenders-not-green`), or the N probe runs disagreed with each other (`inconsistent-probe`). Either way no verdict about the fault can be trusted; fix the defenders first. | **yes** |
|
|
@@ -35,10 +35,16 @@ Rules that follow from the table:
|
|
|
35
35
|
reporting `survived` would be wrong the other way. It is `flaky-defender`.
|
|
36
36
|
5. **Escalation never upgrades a verdict.** A fault that survives its declared
|
|
37
37
|
defenders may be re-run against the whole suite. If the wider suite kills
|
|
38
|
-
it, the verdict stays `survived` with reason `killed-by-undeclared-tests
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
it, the verdict stays `survived` with reason `killed-by-undeclared-tests`
|
|
39
|
+
and `detail.undeclaredKillers` names the tests, so the author can fix
|
|
40
|
+
`defendedBy`. The claim's stated evidence chain is broken even though the
|
|
41
|
+
suite is not blind. It gates, and ranks below a true survivor.
|
|
42
|
+
6. **A verdict names the commit it is about.** Evidence records `repo.head`;
|
|
43
|
+
when the working tree was probed instead, `repo.snapshot` holds the
|
|
44
|
+
throwaway commit that captured it. A tool must refuse to probe a commit
|
|
45
|
+
while defenders or targets have uncommitted changes, unless told to
|
|
46
|
+
snapshot the working tree — otherwise the answer looks right and is not.
|
|
47
|
+
7. **Never a single global score.** Output is per claim, ranked. Blindness is
|
|
42
48
|
concentrated, and one number hides where.
|
|
43
49
|
|
|
44
50
|
## Baseline and delta
|
|
@@ -81,7 +87,7 @@ evidence; they simply do not turn CI red.
|
|
|
81
87
|
|---|---|
|
|
82
88
|
| `0` | No new gating findings at or above the severity floor. |
|
|
83
89
|
| `1` | At least one new gating finding. |
|
|
84
|
-
| `2` | Precondition failed:
|
|
90
|
+
| `2` | Precondition failed: test runner not resolvable (e.g. no `node_modules` linked into the scratch worktree), working tree dirty for a fault target file, claims file invalid, no commits. Nothing was probed. |
|
|
85
91
|
| `3` | Usage or configuration error. |
|
|
86
92
|
|
|
87
93
|
A tool must never exit `0` because it had nothing to check. If the claims
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -72,6 +72,9 @@ const semantic = {
|
|
|
72
72
|
if (r.verdict === 'nocover' && !r.defenders.nocover) {
|
|
73
73
|
errors.push({ path: `${p}/defenders/nocover`, message: 'nocover verdict requires defenders.nocover = true' });
|
|
74
74
|
}
|
|
75
|
+
if (r.detail.undeclaredKillers && r.detail.reason !== 'killed-by-undeclared-tests') {
|
|
76
|
+
errors.push({ path: `${p}/detail/undeclaredKillers`, message: 'undeclaredKillers is only meaningful with reason killed-by-undeclared-tests' });
|
|
77
|
+
}
|
|
75
78
|
if (r.verdict === 'unverifiable' && !r.detail.reason) {
|
|
76
79
|
errors.push({ path: `${p}/detail/reason`, message: 'unverifiable requires a reason (e.g. anchor-missing, anchor-ambiguous)' });
|
|
77
80
|
}
|
|
@@ -11,6 +11,10 @@
|
|
|
11
11
|
"tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
|
|
12
12
|
"createdAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
|
|
13
13
|
"head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
14
|
+
"dirty": {
|
|
15
|
+
"description": "Whether the working tree had uncommitted changes when the evidence behind this baseline was taken.",
|
|
16
|
+
"type": "boolean"
|
|
17
|
+
},
|
|
14
18
|
"fingerprints": {
|
|
15
19
|
"type": "object",
|
|
16
20
|
"propertyNames": { "pattern": "^[a-f0-9]{64}$" },
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
|
|
43
43
|
"producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
|
|
44
44
|
"defendedBy": {
|
|
45
|
-
"description": "Globs (
|
|
45
|
+
"description": "Globs (project-relative) for the test files that supposedly defend this claim. Empty or absent means: discover defenders as the test files that import each fault's target file (relative paths and resolved aliases). If nothing resolves, the verdict is `nocover`.",
|
|
46
46
|
"type": "array",
|
|
47
47
|
"items": { "type": "string", "minLength": 1 },
|
|
48
48
|
"uniqueItems": true
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"enum": ["critical", "high", "medium", "low"]
|
|
32
32
|
},
|
|
33
33
|
"verdict": {
|
|
34
|
-
"description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md).",
|
|
34
|
+
"description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md). `unverifiable` covers a fault that could not be applied (anchor missing/ambiguous) and a claim whose defenders failed to load (`defenders-failed-to-load`).",
|
|
35
35
|
"enum": [
|
|
36
36
|
"killed",
|
|
37
37
|
"survived",
|
|
@@ -21,7 +21,11 @@
|
|
|
21
21
|
"required": ["head", "dirty"],
|
|
22
22
|
"properties": {
|
|
23
23
|
"head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
|
|
24
|
-
"dirty": { "type": "boolean" }
|
|
24
|
+
"dirty": { "type": "boolean" },
|
|
25
|
+
"snapshot": {
|
|
26
|
+
"description": "When the working tree was probed instead of `head`: the dangling commit that captured it. Not a ref; may be garbage-collected.",
|
|
27
|
+
"$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
|
|
28
|
+
}
|
|
25
29
|
},
|
|
26
30
|
"additionalProperties": false
|
|
27
31
|
},
|
|
@@ -149,6 +153,22 @@
|
|
|
149
153
|
"escalationRuns": {
|
|
150
154
|
"type": "array",
|
|
151
155
|
"items": { "$ref": "#/$defs/testRun" }
|
|
156
|
+
},
|
|
157
|
+
"undeclaredKillers": {
|
|
158
|
+
"description": "With reason `killed-by-undeclared-tests`: the tests (as `file::name`) that failed in every escalation run. Names what to add to `defendedBy`.",
|
|
159
|
+
"type": "array",
|
|
160
|
+
"minItems": 1,
|
|
161
|
+
"items": { "type": "string", "minLength": 1, "maxLength": 1024 }
|
|
162
|
+
},
|
|
163
|
+
"anchor": {
|
|
164
|
+
"description": "With an anchor-related reason: how many times `find` occurred versus how many were expected.",
|
|
165
|
+
"type": "object",
|
|
166
|
+
"required": ["hits", "expected"],
|
|
167
|
+
"properties": {
|
|
168
|
+
"hits": { "type": "integer", "minimum": 0 },
|
|
169
|
+
"expected": { "type": "integer", "minimum": 1 }
|
|
170
|
+
},
|
|
171
|
+
"additionalProperties": false
|
|
152
172
|
}
|
|
153
173
|
},
|
|
154
174
|
"additionalProperties": false
|
|
@@ -159,7 +179,11 @@
|
|
|
159
179
|
"properties": {
|
|
160
180
|
"requested": { "type": "array", "items": { "type": "string" } },
|
|
161
181
|
"resolved": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" } },
|
|
162
|
-
"nocover": { "type": "boolean" }
|
|
182
|
+
"nocover": { "type": "boolean" },
|
|
183
|
+
"discovered": {
|
|
184
|
+
"description": "True when no defenders were declared and `resolved` was found by static import of the subject's file.",
|
|
185
|
+
"type": "boolean"
|
|
186
|
+
}
|
|
163
187
|
},
|
|
164
188
|
"additionalProperties": false
|
|
165
189
|
},
|
|
@@ -7,7 +7,7 @@ export function buildBaseline(evidence, { createdAt = new Date().toISOString() }
|
|
|
7
7
|
if (r.verdict === 'killed') continue;
|
|
8
8
|
fingerprints[r.fingerprint] = (fingerprints[r.fingerprint] ?? 0) + 1;
|
|
9
9
|
}
|
|
10
|
-
return { schemaVersion: 1, tool: evidence.tool, createdAt, head: evidence.run.repo.head, fingerprints };
|
|
10
|
+
return { schemaVersion: 1, tool: evidence.tool, createdAt, head: evidence.run.repo.head, dirty: evidence.run.repo.dirty, fingerprints };
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
/**
|
package/src/brief/brief.mjs
CHANGED
|
@@ -11,7 +11,7 @@ export function hintFor(r) {
|
|
|
11
11
|
case 'survived':
|
|
12
12
|
return r.detail.reason === 'killed-by-undeclared-tests'
|
|
13
13
|
? `Only tests outside its declared defenders (${defenders}) catch this; fix the claim's defendedBy or move the assertion.`
|
|
14
|
-
: `${defenders} stayed green with this fault applied; add an assertion that fails on it and passes on HEAD.`;
|
|
14
|
+
: `${defenders} stayed green with this fault applied; add an assertion that fails on it and passes on HEAD. If no test's outcome can change, first check the fault is observable at all.`;
|
|
15
15
|
case 'nocover':
|
|
16
16
|
return `No test file matches ${r.defenders.requested.join(', ') || '(no defenders declared)'}; nothing defends this claim.`;
|
|
17
17
|
case 'unverifiable':
|
package/src/cli.mjs
CHANGED
|
@@ -28,10 +28,15 @@ probe
|
|
|
28
28
|
--baseline <path> baseline to gate against (default: <dir>/.testguard/baseline.json if present)
|
|
29
29
|
--severity <level> gate only at or above (default: low)
|
|
30
30
|
--ref <commit> probe this commit in the scratch worktree (default: HEAD)
|
|
31
|
+
--claim <ID,ID> probe only these claims; writes .testguard/evidence-partial.json
|
|
32
|
+
--include-dirty probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
|
|
33
|
+
--verbose also print each killed fault (default: only unproven ones, plus a count)
|
|
34
|
+
--runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
|
|
35
|
+
--node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
|
|
31
36
|
--in-place mutate the working tree instead of a scratch worktree
|
|
32
37
|
--no-escalate do not re-run survivors against the whole suite
|
|
33
38
|
--no-reuse re-probe claims whose inputs have not changed
|
|
34
|
-
--quiet summary
|
|
39
|
+
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
35
40
|
|
|
36
41
|
claims --json
|
|
37
42
|
baseline --evidence <path> --out <path>
|
|
@@ -58,6 +63,11 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
58
63
|
severity: { type: 'string', default: 'low' },
|
|
59
64
|
max: { type: 'string', default: '20' },
|
|
60
65
|
ref: { type: 'string', default: 'HEAD' },
|
|
66
|
+
claim: { type: 'string' },
|
|
67
|
+
'include-dirty': { type: 'boolean', default: false },
|
|
68
|
+
verbose: { type: 'boolean', default: false },
|
|
69
|
+
'runner-cmd': { type: 'string' },
|
|
70
|
+
'node-modules': { type: 'string' },
|
|
61
71
|
'in-place': { type: 'boolean', default: false },
|
|
62
72
|
'no-escalate': { type: 'boolean', default: false },
|
|
63
73
|
'no-reuse': { type: 'boolean', default: false },
|
|
@@ -15,7 +15,9 @@ export async function baselineCommand({ projectDir, values }, io) {
|
|
|
15
15
|
const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
|
|
16
16
|
writeSpecDoc('baseline', outPath, baseline);
|
|
17
17
|
const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
|
|
18
|
-
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)} → ${outPath}`);
|
|
19
|
-
io.out('Commit this file
|
|
18
|
+
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''} → ${outPath}`);
|
|
19
|
+
io.out('Commit this file; from now on only new findings gate. Ignore the regenerated ones — add to .gitignore:');
|
|
20
|
+
io.out(' .testguard/evidence.json');
|
|
21
|
+
io.out(' .testguard/brief.json');
|
|
20
22
|
return 0;
|
|
21
23
|
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { resolve } from 'node:path';
|
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
3
|
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
4
4
|
import { resolveDefenders } from '../probe/runner-vitest.mjs';
|
|
5
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
5
6
|
|
|
6
7
|
export async function claimsCommand({ projectDir, values }, io) {
|
|
7
8
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
@@ -12,12 +13,14 @@ export async function claimsCommand({ projectDir, values }, io) {
|
|
|
12
13
|
if (values.json) {
|
|
13
14
|
io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
|
|
14
15
|
} else {
|
|
15
|
-
|
|
16
|
+
const annotated = new Set(drift.annotated);
|
|
17
|
+
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
16
18
|
io.out('');
|
|
17
19
|
for (const c of claims.claims) {
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
|
|
20
|
+
const declared = c.defendedBy?.length > 0;
|
|
21
|
+
const defenders = declared ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
|
|
22
|
+
const cover = defenders.length ? `${defenders.length} ${declared ? 'defender' : 'discovered'}${defenders.length === 1 ? '' : 's'}` : 'NO DEFENDER';
|
|
23
|
+
io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
|
|
21
24
|
}
|
|
22
25
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
23
26
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
package/src/commands/probe.mjs
CHANGED
|
@@ -21,7 +21,9 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
21
21
|
io.err('claims file declares no claims; nothing to verify');
|
|
22
22
|
return 2;
|
|
23
23
|
}
|
|
24
|
-
const
|
|
24
|
+
const only = values.claim ? values.claim.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
|
|
25
|
+
// A --claim run is partial evidence; keep it away from the canonical file unless --out says otherwise.
|
|
26
|
+
const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : evidencePath(projectDir);
|
|
25
27
|
const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
|
|
26
28
|
const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
|
|
27
29
|
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
@@ -34,20 +36,32 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
34
36
|
budgetMs,
|
|
35
37
|
mode: values['in-place'] ? 'in-place' : 'worktree',
|
|
36
38
|
ref: values.ref,
|
|
39
|
+
runnerCommand: values['runner-cmd'],
|
|
40
|
+
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
41
|
+
only,
|
|
37
42
|
escalate: !values['no-escalate'],
|
|
38
43
|
toolVersion: version,
|
|
39
|
-
|
|
44
|
+
includeDirty: values['include-dirty'],
|
|
45
|
+
onStage: !values.quiet && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
46
|
+
onProgress: values.quiet ? undefined : (r) => {
|
|
47
|
+
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
48
|
+
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r) + (r.reusedFrom ? ' (reused)' : ''));
|
|
49
|
+
},
|
|
40
50
|
});
|
|
41
51
|
writeSpecDoc('evidence', outPath, evidence);
|
|
42
52
|
|
|
43
53
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
44
|
-
if (!values.quiet) {
|
|
54
|
+
if (!values.quiet && baseline) {
|
|
45
55
|
io.out('');
|
|
46
56
|
const tag = (r) => (g.new.includes(r) ? '[NEW] ' : g.baselined.includes(r) ? '[baseline] ' : '[below floor] ');
|
|
47
57
|
for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out(' ' + tag(r) + renderRecord(r));
|
|
48
58
|
}
|
|
49
59
|
io.out('');
|
|
50
|
-
|
|
51
|
-
|
|
60
|
+
if (!values.quiet && !values.verbose) {
|
|
61
|
+
const killed = evidence.records.filter((r) => r.verdict === 'killed').length;
|
|
62
|
+
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
63
|
+
}
|
|
64
|
+
io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
|
|
65
|
+
io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : ''}`);
|
|
52
66
|
return g.new.length > 0 ? 1 : 0;
|
|
53
67
|
}
|
package/src/git.mjs
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import { rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { randomBytes } from 'node:crypto';
|
|
3
6
|
|
|
4
7
|
export class GitError extends Error {}
|
|
5
8
|
|
|
6
|
-
export function git(args, cwd) {
|
|
7
|
-
const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
|
|
9
|
+
export function git(args, cwd, env) {
|
|
10
|
+
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: env ? { ...process.env, ...env } : process.env });
|
|
8
11
|
if (r.status !== 0) throw new GitError(`git ${args.join(' ')}: ${(r.stderr || r.stdout).trim()}`);
|
|
9
12
|
return r.stdout.trim();
|
|
10
13
|
}
|
|
@@ -25,6 +28,29 @@ export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--',
|
|
|
25
28
|
|
|
26
29
|
export const addWorktree = (repo, dest, ref = 'HEAD') => git(['worktree', 'add', '--detach', dest, ref], repo);
|
|
27
30
|
|
|
31
|
+
/**
|
|
32
|
+
* A dangling commit holding the working tree exactly as it is — tracked
|
|
33
|
+
* changes AND untracked (non-ignored) files — without touching HEAD, the
|
|
34
|
+
* index, or any ref. Built through a temporary index so the user's staging
|
|
35
|
+
* area is never read or written.
|
|
36
|
+
*/
|
|
37
|
+
export function snapshotWorkingTree(repo) {
|
|
38
|
+
const index = join(tmpdir(), `testguard-index-${randomBytes(6).toString('hex')}`);
|
|
39
|
+
const env = {
|
|
40
|
+
GIT_INDEX_FILE: index,
|
|
41
|
+
GIT_AUTHOR_NAME: 'testguard', GIT_AUTHOR_EMAIL: 'testguard@localhost',
|
|
42
|
+
GIT_COMMITTER_NAME: 'testguard', GIT_COMMITTER_EMAIL: 'testguard@localhost',
|
|
43
|
+
};
|
|
44
|
+
try {
|
|
45
|
+
git(['read-tree', 'HEAD'], repo, env);
|
|
46
|
+
git(['add', '-A', '--', '.'], repo, env);
|
|
47
|
+
const tree = git(['write-tree'], repo, env);
|
|
48
|
+
return git(['commit-tree', tree, '-p', 'HEAD', '-m', 'testguard: working-tree snapshot (not a ref)'], repo, env);
|
|
49
|
+
} finally {
|
|
50
|
+
rmSync(index, { force: true });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export function removeWorktree(repo, dest) {
|
|
29
55
|
spawnSync('git', ['worktree', 'remove', '--force', dest], { cwd: repo });
|
|
30
56
|
rmSync(dest, { recursive: true, force: true });
|
package/src/probe/classify.mjs
CHANGED
|
@@ -15,7 +15,8 @@ export function classify({ defenders, anchor, baselineRuns, probeRuns, confirmRu
|
|
|
15
15
|
}
|
|
16
16
|
const loadError = probeRuns.find((r) => r.outcome === 'error');
|
|
17
17
|
if (loadError) {
|
|
18
|
-
|
|
18
|
+
// esbuild/vitest wording: "Transform failed with 1 error", `Expected ")" but found ";"`, "Unexpected token"
|
|
19
|
+
const parseError = /syntax|parse|transform failed|expected .+ but found|unexpected token/i.test(loadError.loadMessage ?? '');
|
|
19
20
|
return { verdict: 'fault-invalid', reason: parseError ? 'replacement-does-not-compile' : 'suite-failed-to-load' };
|
|
20
21
|
}
|
|
21
22
|
if (probeRuns.some((r) => r.outcome === 'timeout' || r.timeouts > 0)) return { verdict: 'timeout', reason: 'test-timed-out' };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { listTestFiles } from './runner-vitest.mjs';
|
|
3
|
+
import { fileImports } from './rank.mjs';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* When a claim declares no defenders: the test files that import the fault's
|
|
7
|
+
* target file, directly, by relative path or through a resolved alias. This
|
|
8
|
+
* is what `nocover` measures against — no test file even imports the source.
|
|
9
|
+
*/
|
|
10
|
+
export function discoverDefenders(projectDir, targetRel) {
|
|
11
|
+
return listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
|
|
12
|
+
}
|
package/src/probe/probe.mjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
1
|
+
import { existsSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { git } from '../git.mjs';
|
|
2
3
|
import { createRequire } from 'node:module';
|
|
3
4
|
import { join, relative, resolve } from 'node:path';
|
|
4
|
-
import { repoRoot as gitRoot, headSha, isDirty } from '../git.mjs';
|
|
5
|
+
import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../git.mjs';
|
|
6
|
+
import { discoverDefenders } from './discover.mjs';
|
|
5
7
|
import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
|
|
6
8
|
import { applyFault, locate } from './inject.mjs';
|
|
7
9
|
import * as vitest from './runner-vitest.mjs';
|
|
@@ -12,7 +14,7 @@ import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
|
12
14
|
|
|
13
15
|
const isKill = (r) => r.outcome === 'fail' && r.assertionFailures > 0;
|
|
14
16
|
|
|
15
|
-
function
|
|
17
|
+
function readRunnerVersion(projectDir) {
|
|
16
18
|
try {
|
|
17
19
|
return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve('vitest/package.json'), 'utf8')).version;
|
|
18
20
|
} catch {
|
|
@@ -31,13 +33,20 @@ export async function probe({
|
|
|
31
33
|
mode = 'worktree',
|
|
32
34
|
ref = 'HEAD',
|
|
33
35
|
budgetMs = 120_000,
|
|
36
|
+
runnerCommand,
|
|
37
|
+
nodeModules,
|
|
38
|
+
only,
|
|
39
|
+
includeDirty = false,
|
|
40
|
+
onStage = () => {},
|
|
34
41
|
escalate = true,
|
|
35
42
|
scratchBase,
|
|
36
43
|
toolVersion = '0.0.0',
|
|
37
44
|
previous,
|
|
38
45
|
onProgress = () => {},
|
|
39
46
|
}) {
|
|
40
|
-
|
|
47
|
+
// realpath: git reports the repository root by its real path (/private/var
|
|
48
|
+
// on macOS, not /var); every relative() below must start from the same place.
|
|
49
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
41
50
|
const root = gitRoot(projectDir);
|
|
42
51
|
if (mode === 'in-place' && ref !== 'HEAD') throw new PreconditionError('--ref needs a scratch worktree; drop --in-place');
|
|
43
52
|
const head = headSha(root, ref);
|
|
@@ -45,24 +54,66 @@ export async function probe({
|
|
|
45
54
|
|
|
46
55
|
const targets = [...new Set(claims.claims.flatMap((c) => c.faults.map((f) => relative(root, join(projectDir, f.file)))))];
|
|
47
56
|
if (mode === 'in-place' && isDirty(root, targets)) {
|
|
48
|
-
throw new PreconditionError(`uncommitted changes in target files (${targets.join(', ')}); commit or stash
|
|
57
|
+
throw new PreconditionError(`uncommitted changes in fault target files (${targets.join(', ')}); commit or stash them, or drop --in-place. Only the files faults are applied to must be clean — test files may be dirty, which is what makes --in-place usable while writing tests.`);
|
|
58
|
+
}
|
|
59
|
+
if (includeDirty && mode !== 'worktree') throw new PreconditionError('--include-dirty applies to worktree mode; drop --in-place');
|
|
60
|
+
if (includeDirty && ref !== 'HEAD') throw new PreconditionError('--include-dirty snapshots the working tree; it cannot be combined with --ref');
|
|
61
|
+
const selected = only ? new Set(only) : null;
|
|
62
|
+
if (selected) {
|
|
63
|
+
const known = new Set(claims.claims.map((c) => c.id));
|
|
64
|
+
const unknown = [...selected].filter((id) => !known.has(id));
|
|
65
|
+
if (unknown.length) throw new PreconditionError(`--claim: unknown claim id(s) ${unknown.join(', ')}`);
|
|
49
66
|
}
|
|
50
67
|
|
|
68
|
+
// Worktree mode probes a commit, not the working tree. Uncommitted defender
|
|
69
|
+
// or target edits would be silently absent — the same survivors, no hint why.
|
|
70
|
+
let snapshot;
|
|
71
|
+
if (mode === 'worktree') {
|
|
72
|
+
if (includeDirty) {
|
|
73
|
+
if (isDirty(root)) snapshot = snapshotWorkingTree(root);
|
|
74
|
+
} else if (ref === 'HEAD') {
|
|
75
|
+
const watched = new Set(targets);
|
|
76
|
+
for (const claim of claims.claims) {
|
|
77
|
+
if (selected && !selected.has(claim.id)) continue;
|
|
78
|
+
const declared = claim.defendedBy?.length ? vitest.resolveDefenders(projectDir, claim.defendedBy) : claim.faults.flatMap((f) => discoverDefenders(projectDir, f.file));
|
|
79
|
+
for (const d of declared) watched.add(relative(root, join(projectDir, d)));
|
|
80
|
+
for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
|
|
81
|
+
}
|
|
82
|
+
const dirty = git(['status', '--porcelain', '--', ...watched], root).split('\n').filter(Boolean).map((l) => l.replace(/^[ MADRCU?!]{1,2}\s+/, '').replace(/^.* -> /, ''));
|
|
83
|
+
if (dirty.length) {
|
|
84
|
+
throw new PreconditionError(`${dirty.length} defender/target file${dirty.length === 1 ? ' has' : 's have'} uncommitted changes (${dirty.join(', ')}); worktree mode probes HEAD (${head.slice(0, 7)}), so those changes would be silently ignored. Commit them, run with --include-dirty to probe the working tree, or use --in-place.`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
51
88
|
const startedAt = new Date().toISOString();
|
|
52
|
-
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref, scratchBase }) : inPlace({ repoRoot: root, projectDir });
|
|
89
|
+
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
53
90
|
const records = [];
|
|
91
|
+
let runnerVersion;
|
|
54
92
|
try {
|
|
93
|
+
const commandTemplate = runnerCommand ? vitest.parseCommandTemplate(runnerCommand) : undefined;
|
|
94
|
+
if (!commandTemplate) {
|
|
95
|
+
// An unresolvable runner is a precondition failure, not a flaky defender.
|
|
96
|
+
const check = await vitest.checkRunner({ projectDir: iso.projectDir });
|
|
97
|
+
if (!check.ok) {
|
|
98
|
+
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}). ` +
|
|
99
|
+
(mode === 'worktree' ? 'No usable node_modules was linked: pass --node-modules <path>, or run with --in-place.' : 'Install dependencies first.'));
|
|
100
|
+
}
|
|
101
|
+
runnerVersion = check.version;
|
|
102
|
+
}
|
|
55
103
|
const allTests = vitest.listTestFiles(iso.projectDir);
|
|
56
104
|
const baselineCache = new Map();
|
|
57
105
|
const prior = previous && previous.run.confirmRuns === confirmRuns
|
|
58
106
|
? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
|
|
59
107
|
: new Map();
|
|
60
|
-
const runDefenders = (files) => vitest.runVitest({ projectDir: iso.projectDir, files, budgetMs });
|
|
108
|
+
const runDefenders = (files) => vitest.runVitest({ projectDir: iso.projectDir, files, budgetMs, commandTemplate });
|
|
61
109
|
|
|
62
110
|
for (const claim of claims.claims) {
|
|
63
|
-
|
|
111
|
+
if (selected && !selected.has(claim.id)) continue;
|
|
112
|
+
const declared = claim.defendedBy?.length ? vitest.resolveDefenders(iso.projectDir, claim.defendedBy) : null;
|
|
64
113
|
for (const fault of claim.faults) {
|
|
65
|
-
const
|
|
114
|
+
const defenders = declared ?? discoverDefenders(iso.projectDir, fault.file);
|
|
115
|
+
const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
|
|
116
|
+
const record = await probeOne({ claim, fault, defenders, discovered: declared === null, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id });
|
|
66
117
|
records.push(record);
|
|
67
118
|
onProgress(record);
|
|
68
119
|
}
|
|
@@ -78,8 +129,8 @@ export async function probe({
|
|
|
78
129
|
id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
|
|
79
130
|
startedAt,
|
|
80
131
|
finishedAt: new Date().toISOString(),
|
|
81
|
-
repo: { head, dirty: isDirty(root) },
|
|
82
|
-
runner: { name: vitest.name, ...(runnerVersion(projectDir) ? { version: runnerVersion(projectDir) } : {}) },
|
|
132
|
+
repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
|
|
133
|
+
runner: { name: vitest.name, ...((runnerVersion ?? readRunnerVersion(projectDir)) ? { version: runnerVersion ?? readRunnerVersion(projectDir) } : {}) },
|
|
83
134
|
confirmRuns,
|
|
84
135
|
mode,
|
|
85
136
|
},
|
|
@@ -87,7 +138,7 @@ export async function probe({
|
|
|
87
138
|
};
|
|
88
139
|
}
|
|
89
140
|
|
|
90
|
-
async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, prior, priorRunId }) {
|
|
141
|
+
async function probeOne({ claim, fault, defenders, discovered, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior, priorRunId }) {
|
|
91
142
|
const targetPath = join(iso.projectDir, fault.file);
|
|
92
143
|
const targetExists = existsSync(targetPath);
|
|
93
144
|
const inputs = {
|
|
@@ -111,20 +162,32 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
111
162
|
const key = defenders.join('\n');
|
|
112
163
|
if (!baselineCache.has(key)) {
|
|
113
164
|
const runs = [];
|
|
165
|
+
let loadMessage;
|
|
114
166
|
for (let i = 0; i < confirmRuns; i++) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
167
|
+
stage('baseline', i + 1, confirmRuns);
|
|
168
|
+
const res = await runDefenders(defenders);
|
|
169
|
+
runs.push(res.run);
|
|
170
|
+
if (res.run.outcome !== 'pass') {
|
|
171
|
+
loadMessage = res.loadMessage;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
118
174
|
}
|
|
119
|
-
baselineCache.set(key, runs);
|
|
175
|
+
baselineCache.set(key, { runs, loadMessage });
|
|
176
|
+
}
|
|
177
|
+
const baseline = baselineCache.get(key);
|
|
178
|
+
detail.baselineRuns = baseline.runs;
|
|
179
|
+
if (baseline.runs.some((r) => r.outcome === 'error')) {
|
|
180
|
+
// The defenders did not load. That is not flakiness and not a verdict
|
|
181
|
+
// about the fault; the claim cannot be probed until they do.
|
|
182
|
+
anchor = { status: 'defenders-failed-to-load', hits: anchor.hits, expected: anchor.expected, message: baseline.loadMessage };
|
|
120
183
|
}
|
|
121
|
-
detail.baselineRuns = baselineCache.get(key);
|
|
122
184
|
|
|
123
185
|
if (detail.baselineRuns.every((r) => r.outcome === 'pass') && detail.baselineRuns.length === confirmRuns) {
|
|
124
186
|
const mutation = applyFault(iso.projectDir, fault);
|
|
125
187
|
try {
|
|
126
188
|
const probeRuns = rawProbeRuns;
|
|
127
189
|
for (let i = 0; i < confirmRuns; i++) {
|
|
190
|
+
stage('probe', i + 1, confirmRuns);
|
|
128
191
|
const { run, timeouts, loadMessage } = await runDefenders(defenders);
|
|
129
192
|
probeRuns.push({ ...run, timeouts, loadMessage });
|
|
130
193
|
if (shouldStopEarly(probeRuns)) break;
|
|
@@ -140,6 +203,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
140
203
|
const runs = [];
|
|
141
204
|
let killers = null;
|
|
142
205
|
for (let i = 0; i < confirmRuns; i++) {
|
|
206
|
+
stage('escalation', i + 1, confirmRuns);
|
|
143
207
|
const { run, failedTests } = await runDefenders(allTests);
|
|
144
208
|
runs.push(run);
|
|
145
209
|
killers = killers === null ? new Set(failedTests) : new Set(failedTests.filter((t) => killers.has(t)));
|
|
@@ -147,7 +211,10 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
147
211
|
}
|
|
148
212
|
detail.escalated = true;
|
|
149
213
|
detail.escalationRuns = runs;
|
|
150
|
-
if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill))
|
|
214
|
+
if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill)) {
|
|
215
|
+
detail.reason = 'killed-by-undeclared-tests';
|
|
216
|
+
detail.undeclaredKillers = [...killers].sort();
|
|
217
|
+
}
|
|
151
218
|
}
|
|
152
219
|
} finally {
|
|
153
220
|
mutation.restore();
|
|
@@ -158,6 +225,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
158
225
|
|
|
159
226
|
const { verdict, reason } = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns: rawProbeRuns, confirmRuns });
|
|
160
227
|
if (reason && !detail.reason) detail.reason = reason;
|
|
228
|
+
if (anchor && anchor.status !== 'ok' && anchor.status !== 'file-missing' && anchor.status !== 'defenders-failed-to-load') detail.anchor = { hits: anchor.hits, expected: anchor.expected };
|
|
161
229
|
|
|
162
230
|
const blast = targetExists ? blastRadius(iso.projectDir, fault.file) : 0;
|
|
163
231
|
|
|
@@ -167,7 +235,7 @@ async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, e
|
|
|
167
235
|
subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy },
|
|
168
236
|
verdict,
|
|
169
237
|
detail,
|
|
170
|
-
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0 },
|
|
238
|
+
defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}) },
|
|
171
239
|
inputs,
|
|
172
240
|
rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast }),
|
|
173
241
|
};
|
package/src/probe/rank.mjs
CHANGED
|
@@ -1,7 +1,78 @@
|
|
|
1
|
-
import { readFileSync } from 'node:fs';
|
|
1
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
2
2
|
import { dirname, join, resolve, extname } from 'node:path';
|
|
3
3
|
import { walk } from '../util/glob.mjs';
|
|
4
4
|
|
|
5
|
+
const aliasCache = new Map();
|
|
6
|
+
|
|
7
|
+
/** Minimal JSONC: strip comments and trailing commas, as tsconfig allows. */
|
|
8
|
+
function parseJsonc(text) {
|
|
9
|
+
return JSON.parse(text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '').replace(/,(\s*[}\]])/g, '$1'));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Alias rules for a project: tsconfig/jsconfig `compilerOptions.paths` (with
|
|
14
|
+
* `baseUrl`, following a relative `extends` a few levels) and package.json
|
|
15
|
+
* `imports`. Bare specifiers that match no rule are ignored, as documented.
|
|
16
|
+
*/
|
|
17
|
+
export function loadAliases(projectDir) {
|
|
18
|
+
if (aliasCache.has(projectDir)) return aliasCache.get(projectDir);
|
|
19
|
+
const rules = [];
|
|
20
|
+
const toRule = (pattern, targets, base) => {
|
|
21
|
+
const [prefix, suffix = ''] = pattern.split('*');
|
|
22
|
+
rules.push({ prefix, suffix, wildcard: pattern.includes('*'), targets: targets.map((t) => resolve(base, t)) });
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
let cfgPath = ['tsconfig.json', 'jsconfig.json'].map((f) => join(projectDir, f)).find(existsSync);
|
|
26
|
+
let baseUrl;
|
|
27
|
+
let paths;
|
|
28
|
+
for (let hop = 0; cfgPath && hop < 5; hop++) {
|
|
29
|
+
let cfg;
|
|
30
|
+
try {
|
|
31
|
+
cfg = parseJsonc(readFileSync(cfgPath, 'utf8'));
|
|
32
|
+
} catch {
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
const co = cfg.compilerOptions ?? {};
|
|
36
|
+
if (baseUrl === undefined && co.baseUrl !== undefined) baseUrl = resolve(dirname(cfgPath), co.baseUrl);
|
|
37
|
+
if (paths === undefined && co.paths !== undefined) paths = { dir: dirname(cfgPath), map: co.paths };
|
|
38
|
+
const ext = typeof cfg.extends === 'string' && cfg.extends.startsWith('.') ? cfg.extends : null;
|
|
39
|
+
cfgPath = ext ? resolve(dirname(cfgPath), ext.endsWith('.json') ? ext : ext + '.json') : null;
|
|
40
|
+
}
|
|
41
|
+
if (paths) {
|
|
42
|
+
const base = baseUrl ?? paths.dir;
|
|
43
|
+
for (const [pattern, targets] of Object.entries(paths.map)) if (Array.isArray(targets)) toRule(pattern, targets, base);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const pkgPath = join(projectDir, 'package.json');
|
|
47
|
+
if (existsSync(pkgPath)) {
|
|
48
|
+
try {
|
|
49
|
+
const imports = JSON.parse(readFileSync(pkgPath, 'utf8')).imports ?? {};
|
|
50
|
+
for (const [pattern, target] of Object.entries(imports)) {
|
|
51
|
+
const t = typeof target === 'string' ? target : target?.default ?? target?.import ?? target?.node;
|
|
52
|
+
if (typeof t === 'string' && t.startsWith('.')) toRule(pattern, [t], projectDir);
|
|
53
|
+
}
|
|
54
|
+
} catch {}
|
|
55
|
+
}
|
|
56
|
+
aliasCache.set(projectDir, rules);
|
|
57
|
+
return rules;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Absolute candidate bases an aliased specifier could mean, or [] if it matches no rule. */
|
|
61
|
+
function aliasCandidates(specifier, rules) {
|
|
62
|
+
const out = [];
|
|
63
|
+
for (const r of rules) {
|
|
64
|
+
if (r.wildcard) {
|
|
65
|
+
if (specifier.startsWith(r.prefix) && specifier.endsWith(r.suffix) && specifier.length >= r.prefix.length + r.suffix.length) {
|
|
66
|
+
const star = specifier.slice(r.prefix.length, specifier.length - r.suffix.length);
|
|
67
|
+
for (const t of r.targets) out.push(t.replace('*', star));
|
|
68
|
+
}
|
|
69
|
+
} else if (specifier === r.prefix) {
|
|
70
|
+
out.push(...r.targets);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
|
|
5
76
|
const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
|
|
6
77
|
const IMPORT_RE = /(?:from\s*|import\s*\(?\s*|require\s*\(\s*)['"]([^'"]+)['"]/g;
|
|
7
78
|
const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
@@ -11,18 +82,40 @@ const SEVERITY_WEIGHT = { critical: 8, high: 4, medium: 2, low: 1 };
|
|
|
11
82
|
// independent evidence than one from a human-governed spec.
|
|
12
83
|
const SOURCE_WEIGHT = { spec: 1, adr: 1, manual: 0.9, annotation: 0.75, comment: 0.75 };
|
|
13
84
|
|
|
14
|
-
function
|
|
15
|
-
if (!specifier.startsWith('.')) return false;
|
|
16
|
-
const base = resolve(dirname(fromFile), specifier);
|
|
85
|
+
function expandBase(base) {
|
|
17
86
|
const candidates = [base, ...[...SOURCE_EXT].map((e) => base + e), ...[...SOURCE_EXT].map((e) => join(base, 'index' + e))];
|
|
18
87
|
const stripped = base.replace(/\.(js|mjs|cjs)$/, '');
|
|
19
88
|
candidates.push(stripped + '.ts', stripped + '.mts', stripped + '.tsx');
|
|
20
|
-
return candidates
|
|
89
|
+
return candidates;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function resolvesTo(fromFile, specifier, targetAbs, aliases) {
|
|
93
|
+
const bases = specifier.startsWith('.') ? [resolve(dirname(fromFile), specifier)] : aliasCandidates(specifier, aliases);
|
|
94
|
+
return bases.some((b) => expandBase(b).includes(targetAbs));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Does the file at `absFile` import `targetRel` (relative or alias-resolved)? */
|
|
98
|
+
export function fileImports(projectDir, absFile, targetRel) {
|
|
99
|
+
const targetAbs = resolve(projectDir, targetRel);
|
|
100
|
+
const aliases = loadAliases(projectDir);
|
|
101
|
+
let src;
|
|
102
|
+
try {
|
|
103
|
+
src = readFileSync(absFile, 'utf8');
|
|
104
|
+
} catch {
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
for (const m of src.matchAll(IMPORT_RE)) if (resolvesTo(absFile, m[1], targetAbs, aliases)) return true;
|
|
108
|
+
return false;
|
|
21
109
|
}
|
|
22
110
|
|
|
23
|
-
/**
|
|
111
|
+
/**
|
|
112
|
+
* Number of non-test source files that import `targetRel`: relative specifiers,
|
|
113
|
+
* tsconfig/jsconfig `paths` aliases and package.json `imports` are resolved;
|
|
114
|
+
* bare package specifiers are not. Direct imports only.
|
|
115
|
+
*/
|
|
24
116
|
export function blastRadius(projectDir, targetRel) {
|
|
25
117
|
const targetAbs = resolve(projectDir, targetRel);
|
|
118
|
+
const aliases = loadAliases(projectDir);
|
|
26
119
|
let count = 0;
|
|
27
120
|
for (const rel of walk(projectDir)) {
|
|
28
121
|
if (!SOURCE_EXT.has(extname(rel)) || TEST_RE.test(rel) || rel === targetRel) continue;
|
|
@@ -34,7 +127,7 @@ export function blastRadius(projectDir, targetRel) {
|
|
|
34
127
|
continue;
|
|
35
128
|
}
|
|
36
129
|
for (const m of src.matchAll(IMPORT_RE)) {
|
|
37
|
-
if (resolvesTo(abs, m[1], targetAbs)) {
|
|
130
|
+
if (resolvesTo(abs, m[1], targetAbs, aliases)) {
|
|
38
131
|
count++;
|
|
39
132
|
break;
|
|
40
133
|
}
|
|
@@ -10,6 +10,63 @@ const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test
|
|
|
10
10
|
|
|
11
11
|
export const name = 'vitest';
|
|
12
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Split a runner command template into argv. Supports double and single
|
|
15
|
+
* quotes; `{files}` expands to the test files (one argv entry each) and
|
|
16
|
+
* `{out}` to the JSON report path. Everything else is passed through.
|
|
17
|
+
*/
|
|
18
|
+
export function parseCommandTemplate(template) {
|
|
19
|
+
const words = [];
|
|
20
|
+
let cur = '';
|
|
21
|
+
let quote = null;
|
|
22
|
+
let has = false;
|
|
23
|
+
for (const ch of template) {
|
|
24
|
+
if (quote) {
|
|
25
|
+
if (ch === quote) quote = null;
|
|
26
|
+
else cur += ch;
|
|
27
|
+
} else if (ch === '"' || ch === "'") {
|
|
28
|
+
quote = ch;
|
|
29
|
+
has = true;
|
|
30
|
+
} else if (/\s/.test(ch)) {
|
|
31
|
+
if (has || cur) words.push(cur);
|
|
32
|
+
cur = '';
|
|
33
|
+
has = false;
|
|
34
|
+
} else {
|
|
35
|
+
cur += ch;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (has || cur) words.push(cur);
|
|
39
|
+
if (quote) throw new RangeError('unterminated quote in --runner-cmd');
|
|
40
|
+
if (!words.includes('{files}') || !words.some((w) => w.includes('{out}'))) throw new RangeError('--runner-cmd must contain {files} (as its own word) and {out}');
|
|
41
|
+
return words;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** `{files}` must be a word of its own (one argv entry per file); `{out}` may be embedded, e.g. `--outputFile={out}`. */
|
|
45
|
+
export const expandCommand = (words, files, outFile) => words.flatMap((w) => (w === '{files}' ? files : [w.replaceAll('{out}', outFile)]));
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Can the runner be resolved at all from this directory? Run before any
|
|
49
|
+
* verdict is attempted: an unresolvable runner is a precondition failure,
|
|
50
|
+
* never a statement about the tests.
|
|
51
|
+
*/
|
|
52
|
+
export function checkRunner({ projectDir, budgetMs = 60_000, env = process.env }) {
|
|
53
|
+
const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
|
|
54
|
+
return new Promise((resolve) => {
|
|
55
|
+
const child = spawn(npx, ['--no-install', 'vitest', '--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
|
|
56
|
+
let out = '';
|
|
57
|
+
let err = '';
|
|
58
|
+
child.on('error', (e) => resolve({ ok: false, message: e.message }));
|
|
59
|
+
child.stdout.on('data', (d) => (out += d));
|
|
60
|
+
child.stderr.on('data', (d) => (err += d));
|
|
61
|
+
const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
|
|
62
|
+
child.on('close', (code) => {
|
|
63
|
+
clearTimeout(timer);
|
|
64
|
+
const version = out.trim().replace(/^vitest\//, '');
|
|
65
|
+
resolve(code === 0 && version ? { ok: true, version } : { ok: false, message: (err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}` });
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
13
70
|
/** Defender globs → existing files. Empty result is the `nocover` signal. */
|
|
14
71
|
export const resolveDefenders = (projectDir, globs) => matchGlobs(projectDir, globs ?? []);
|
|
15
72
|
|
|
@@ -47,16 +104,18 @@ export function parseReport(report, durationMs) {
|
|
|
47
104
|
* The budget matters because a synchronous infinite loop is immune to
|
|
48
105
|
* vitest's own test timeout; only killing the process ends it.
|
|
49
106
|
*/
|
|
50
|
-
export function runVitest({ projectDir, files, budgetMs = 120_000, command }) {
|
|
107
|
+
export function runVitest({ projectDir, files, budgetMs = 120_000, command, commandTemplate }) {
|
|
51
108
|
const outFile = join(tmpdir(), `testguard-vitest-${randomBytes(6).toString('hex')}.json`);
|
|
52
109
|
const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
|
|
53
|
-
const [cmd, ...args] = command
|
|
110
|
+
const [cmd, ...args] = command
|
|
111
|
+
?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`]);
|
|
54
112
|
const started = Date.now();
|
|
55
113
|
|
|
56
114
|
return new Promise((resolve) => {
|
|
57
115
|
const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0' } });
|
|
58
116
|
let stderr = '';
|
|
59
117
|
child.stderr.on('data', (d) => (stderr += d));
|
|
118
|
+
child.on('error', (e) => (stderr += e.message));
|
|
60
119
|
let killed = false;
|
|
61
120
|
const timer = setTimeout(() => {
|
|
62
121
|
killed = true;
|
package/src/probe/worktree.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync, mkdtempSync, readdirSync, symlinkSync, mkdirSync,
|
|
1
|
+
import { existsSync, mkdtempSync, readdirSync, symlinkSync, mkdirSync, statSync, realpathSync } from 'node:fs';
|
|
2
2
|
import { tmpdir } from 'node:os';
|
|
3
3
|
import { join, relative, dirname } from 'node:path';
|
|
4
4
|
import { addWorktree, removeWorktree, headSha } from '../git.mjs';
|
|
@@ -7,10 +7,21 @@ export class PreconditionError extends Error {}
|
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Find every node_modules directory in the main tree (to a shallow depth) so
|
|
10
|
-
* the scratch worktree can borrow them instead of reinstalling.
|
|
10
|
+
* the scratch worktree can borrow them instead of reinstalling. A symlinked
|
|
11
|
+
* node_modules — the layout a sibling or auto-created worktree produces — is
|
|
12
|
+
* a directory for this purpose, and is linked to its resolved target.
|
|
11
13
|
*/
|
|
12
|
-
function findNodeModules(root, depth = 3) {
|
|
14
|
+
export function findNodeModules(root, depth = 3) {
|
|
13
15
|
const found = [];
|
|
16
|
+
const isDir = (dir, e) => {
|
|
17
|
+
if (e.isDirectory()) return true;
|
|
18
|
+
if (!e.isSymbolicLink()) return false;
|
|
19
|
+
try {
|
|
20
|
+
return statSync(join(dir, e.name)).isDirectory();
|
|
21
|
+
} catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
};
|
|
14
25
|
const visit = (dir, rel, d) => {
|
|
15
26
|
let entries;
|
|
16
27
|
try {
|
|
@@ -19,10 +30,10 @@ function findNodeModules(root, depth = 3) {
|
|
|
19
30
|
return;
|
|
20
31
|
}
|
|
21
32
|
for (const e of entries) {
|
|
22
|
-
if (!e
|
|
33
|
+
if (!isDir(dir, e)) continue;
|
|
23
34
|
const relPath = rel ? `${rel}/${e.name}` : e.name;
|
|
24
35
|
if (e.name === 'node_modules') {
|
|
25
|
-
found.push(relPath);
|
|
36
|
+
found.push({ rel: relPath, target: realpathSync(join(dir, e.name)) });
|
|
26
37
|
} else if (d < depth && e.name !== '.git' && !e.name.startsWith('.')) {
|
|
27
38
|
visit(join(dir, e.name), relPath, d + 1);
|
|
28
39
|
}
|
|
@@ -36,18 +47,24 @@ function findNodeModules(root, depth = 3) {
|
|
|
36
47
|
* A scratch git worktree at HEAD, with the main tree's node_modules linked in.
|
|
37
48
|
* Faults are applied here; the user's tree is never touched.
|
|
38
49
|
*/
|
|
39
|
-
export function createScratch({ repoRoot, projectDir, ref = 'HEAD', scratchBase = tmpdir() }) {
|
|
50
|
+
export function createScratch({ repoRoot, projectDir, ref = 'HEAD', scratchBase = tmpdir(), nodeModules }) {
|
|
40
51
|
const sha = headSha(repoRoot, ref);
|
|
41
52
|
if (!sha) {
|
|
42
53
|
throw new PreconditionError(ref === 'HEAD' ? 'repository has no commits; commit first, or run with --in-place' : `ref ${ref} does not resolve to a commit`);
|
|
43
54
|
}
|
|
44
55
|
const dest = mkdtempSync(join(scratchBase, 'testguard-'));
|
|
45
56
|
addWorktree(repoRoot, dest, sha);
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
57
|
+
const links = findNodeModules(repoRoot);
|
|
58
|
+
if (nodeModules) {
|
|
59
|
+
// Explicit override: link it where the probed project expects it.
|
|
60
|
+
const rel = join(relative(repoRoot, projectDir), 'node_modules');
|
|
61
|
+
links.unshift({ rel, target: realpathSync(nodeModules) });
|
|
62
|
+
}
|
|
63
|
+
for (const { rel, target } of links) {
|
|
64
|
+
const at = join(dest, rel);
|
|
65
|
+
if (existsSync(at)) continue;
|
|
66
|
+
mkdirSync(dirname(at), { recursive: true });
|
|
67
|
+
symlinkSync(target, at, 'dir');
|
|
51
68
|
}
|
|
52
69
|
return {
|
|
53
70
|
mode: 'worktree',
|
package/src/render.mjs
CHANGED
|
@@ -5,7 +5,16 @@ export const formatVerdict = (v) => (v === 'killed' ? 'killed' : v.toUpperCase()
|
|
|
5
5
|
|
|
6
6
|
export function renderRecord(r) {
|
|
7
7
|
const head = `${formatVerdict(r.verdict).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
|
|
8
|
-
|
|
8
|
+
let why = '';
|
|
9
|
+
if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
|
|
10
|
+
const files = [...new Set(r.detail.undeclaredKillers.map((k) => k.split('::')[0]))];
|
|
11
|
+
why = ` [killed-by-undeclared-tests: ${files.join(', ')}]`;
|
|
12
|
+
} else if (r.detail.reason === 'anchor-ambiguous' && r.detail.anchor) {
|
|
13
|
+
why = ` [anchor-ambiguous: ${r.detail.anchor.hits} hits, expected ${r.detail.anchor.expected}]`;
|
|
14
|
+
} else if (r.detail.reason) {
|
|
15
|
+
why = ` [${r.detail.reason}]`;
|
|
16
|
+
}
|
|
17
|
+
if (r.defenders.discovered) why += ' (defenders discovered by import)';
|
|
9
18
|
return `${head} ${r.claim.severity.padEnd(8)} ${r.subject.file} ${r.subject.description}${why}`;
|
|
10
19
|
}
|
|
11
20
|
|
|
@@ -15,11 +24,13 @@ export function summarize(records) {
|
|
|
15
24
|
return byVerdict;
|
|
16
25
|
}
|
|
17
26
|
|
|
18
|
-
export function renderSummary(records) {
|
|
27
|
+
export function renderSummary(records, run) {
|
|
19
28
|
const byVerdict = summarize(records);
|
|
20
29
|
const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v)}`);
|
|
21
|
-
const
|
|
22
|
-
|
|
30
|
+
const unproven = records.filter((r) => r.verdict !== 'killed');
|
|
31
|
+
const claims = new Set(unproven.map((r) => r.claim.id)).size;
|
|
32
|
+
const where = run ? ` Probed ${run.repo.snapshot ? `working tree (snapshot ${run.repo.snapshot.slice(0, 7)} of ${run.repo.head.slice(0, 7)})` : run.mode === 'in-place' ? `in place at ${run.repo.head.slice(0, 7)}${run.repo.dirty ? ' (dirty)' : ''}` : run.repo.head.slice(0, 7)}.` : '';
|
|
33
|
+
return `${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
|
|
23
34
|
}
|
|
24
35
|
|
|
25
36
|
/** Survivors first, then by rank score; killed last. */
|