testguard-cli 0.15.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +18 -4
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +34 -5
- package/spec/lib/validate.mjs +26 -1
- package/spec/schemas/common.schema.json +10 -0
- package/spec/schemas/gate.schema.json +1 -0
- package/spec/schemas/status.schema.json +2 -1
- package/src/cli.mjs +30 -3
- package/src/commands/probe.mjs +13 -0
- package/src/gate/changed.mjs +40 -3
- package/src/init/templates/SKILL.md +1 -1
- package/src/probe/probe.mjs +1 -1
- package/src/probe/runners/discovery.mjs +4 -2
- package/src/probe/runners/shared.mjs +16 -1
- package/src/status/status.mjs +6 -0
- package/src/status/surface.mjs +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.15.1] - 2026-09-30
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.15.0`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- fix: runner timeout identity and reported adoption regressions (#180)
|
|
17
|
+
- chore(homebrew): sha256 for v0.15.0 (#177)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- Assertion messages quoting "timed out" no longer become runner timeouts.
|
|
23
|
+
Native timeout headers and structured timeout failures remain non-detections
|
|
24
|
+
(#179).
|
|
25
|
+
- Configuration dependency hashing no longer rejects harmless absolute-looking
|
|
26
|
+
strings such as Vite URL bases, middleware routes, or `split('/')`. Existing
|
|
27
|
+
outside-project file dependencies and symlinks still fail closed (#178).
|
|
28
|
+
- Discovery failures no longer assert that dependencies are missing when the
|
|
29
|
+
runner resolved successfully but its configuration could not be discovered.
|
|
30
|
+
- Each subcommand's `--help` now shows its own options and an example; global
|
|
31
|
+
`--help` exits successfully and points to command-specific help (#120).
|
|
32
|
+
- Parent gates, status change reports, and claimed-surface counts respect valid
|
|
33
|
+
nested TestGuard project boundaries. Delegated files are explicitly reported
|
|
34
|
+
and require a separate child gate; invalid or symlinked markers fail (#150).
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- Explicit `probe --allow-empty` adoption mode, also available as an opt-in
|
|
39
|
+
GitHub Action input. A valid empty claims file skips verification without
|
|
40
|
+
writing evidence; JSON keeps the `no-claims` state. Default empty probes,
|
|
41
|
+
invalid claims, and explicit claim selections still fail (#176).
|
|
42
|
+
|
|
10
43
|
## [0.15.0] - 2026-09-29
|
|
11
44
|
|
|
12
45
|
Automated weekly release — everything merged since `v0.14.2`.
|
package/README.md
CHANGED
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.15.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.15.1` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.15.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.15.1/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -407,6 +407,20 @@ is missing. TestGuard itself still makes no network calls.
|
|
|
407
407
|
|
|
408
408
|
### Starting from zero claims
|
|
409
409
|
|
|
410
|
+
During gradual adoption, use `testguard probe --allow-empty` to skip a valid
|
|
411
|
+
empty claims file without blocking a push. It explicitly reports that no
|
|
412
|
+
verification was performed and writes no evidence; `--json` retains the
|
|
413
|
+
`no-claims` state. Missing or invalid claims files still fail. Run `gate`
|
|
414
|
+
separately to enforce changed-file coverage. For the pre-commit probe hook,
|
|
415
|
+
set `args: [--allow-empty]`; for the GitHub Action, set `allow-empty: 'true'`.
|
|
416
|
+
The default probe still exits 2 when no claims exist.
|
|
417
|
+
|
|
418
|
+
In a monorepo, a valid nested `testguard.claims.json` marks a separate project.
|
|
419
|
+
The parent gate reports its changed files as delegated and does not count them
|
|
420
|
+
as parent coverage. Run `gate` in each child project as well; a passing parent
|
|
421
|
+
gate does not establish that child coverage passed. Invalid child claims fail
|
|
422
|
+
evaluation instead of hiding files.
|
|
423
|
+
|
|
410
424
|
`gate` names the changed files that carry no claim, and stops there —
|
|
411
425
|
correctly, because stating a claim is a human act. But a project adopting this
|
|
412
426
|
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
@@ -663,12 +677,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
663
677
|
# GitHub Actions
|
|
664
678
|
- uses: actions/checkout@v4
|
|
665
679
|
with: { fetch-depth: 0 }
|
|
666
|
-
- uses: raccioly/testguard@v0.15.
|
|
680
|
+
- uses: raccioly/testguard@v0.15.1
|
|
667
681
|
with: { command: gate }
|
|
668
682
|
|
|
669
683
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
670
684
|
include:
|
|
671
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.15.
|
|
685
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.15.1/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
672
686
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
673
687
|
```
|
|
674
688
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.1",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -36,7 +36,11 @@ Rules that follow from the table:
|
|
|
36
36
|
that failed and then passed on retry (Playwright's `flaky`) is a
|
|
37
37
|
**non-green run** even when the runner exits 0, and a test the runner
|
|
38
38
|
reports as timed out (`timedOut`) is a timeout, never an assertion
|
|
39
|
-
failure.
|
|
39
|
+
failure. Jest-compatible timeout messages are recognized by native runner
|
|
40
|
+
error headers or structured timeout identity, not by words quoted in an
|
|
41
|
+
assertion, stack frame, or source excerpt. Structured assertion identity
|
|
42
|
+
does not hide a separate timeout in the same failed test.
|
|
43
|
+
Python is read the same way: an `AssertionError` and any other
|
|
40
44
|
exception raised by the test body both count, because both are the suite
|
|
41
45
|
rejecting the behaviour; a collection error — a module that will not
|
|
42
46
|
import, which is what a replacement that does not parse looks like from
|
|
@@ -172,6 +176,11 @@ Rules that follow from the table:
|
|
|
172
176
|
competing owned runners or a declared defender absent from every manifest
|
|
173
177
|
are precondition failures, never heuristic routing decisions.
|
|
174
178
|
|
|
179
|
+
Static string literals are conservative dependency candidates, not proof
|
|
180
|
+
of a filesystem dependency. Missing candidates and ordinary directories
|
|
181
|
+
(including `/` in URL bases and string separators) are ignored. An existing
|
|
182
|
+
candidate file outside the project, or a candidate symlink, is rejected.
|
|
183
|
+
|
|
175
184
|
14. **Barrel resolution follows symbols, not module reachability.** Automatic
|
|
176
185
|
discovery may traverse a re-export chain only for the binding the test
|
|
177
186
|
actually imports. Importing an unrelated symbol from a broad barrel does
|
|
@@ -494,10 +503,22 @@ Rules:
|
|
|
494
503
|
`unclaimed-changes` and its next action is to write the claim, before any
|
|
495
504
|
`unproven` finding is surfaced. The brief renders them first. The claim is
|
|
496
505
|
written before more code.
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
506
|
+
7. **A descendant claims file defines a separate project.** The first
|
|
507
|
+
descendant directory on a changed file's path with a valid, regular,
|
|
508
|
+
non-symlinked `testguard.claims.json` owns that subtree. Invalid or unreadable
|
|
509
|
+
markers fail evaluation; they cannot hide unclaimed files. Parent gate and
|
|
510
|
+
status report the delegation as optional `nested` entries with project and
|
|
511
|
+
file paths relative to the parent, and instruct the operator to run that
|
|
512
|
+
project's gate. Delegation is not coverage, an ignore reliance, or proof
|
|
513
|
+
that the child passed. Parent source-surface counts omit these subtrees.
|
|
514
|
+
Removing the marker restores parent evaluation. The accounting identity is
|
|
515
|
+
`changed = evaluated + excluded.length + sum(nested[].files.length)`;
|
|
516
|
+
`--strict` still fails a non-empty change with zero parent evaluation.
|
|
517
|
+
|
|
518
|
+
Exit codes: `0` every changed parent-owned source file is claimed or excused
|
|
519
|
+
(or nothing changed); delegated projects still require separate child gates.
|
|
520
|
+
`1` at least one uncovered file (or strict mode over an
|
|
521
|
+
all-excluded or all-delegated change); `2` the change cannot be evaluated (unresolvable
|
|
501
522
|
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
502
523
|
|
|
503
524
|
## Anchor preflight
|
|
@@ -548,3 +569,11 @@ evidence; they simply do not turn CI red.
|
|
|
548
569
|
A tool must never exit `0` because it had nothing to check. If the claims
|
|
549
570
|
file is empty or every claim is out of scope, that is reported explicitly and
|
|
550
571
|
the exit code is `2`.
|
|
572
|
+
|
|
573
|
+
`probe --allow-empty` is an explicit adoption exception for a valid claims
|
|
574
|
+
file containing zero claims, with no `--claim` selection. It exits `0`, says
|
|
575
|
+
that verification was skipped, and writes no evidence or baseline. Its JSON
|
|
576
|
+
output is the existing `no-claims` status document, without run metadata;
|
|
577
|
+
it never reports a clean measurement. Missing or invalid claims still fail,
|
|
578
|
+
and selected claims cannot be silently skipped. Run `gate` separately to
|
|
579
|
+
enforce coverage of changed files during adoption. Integrations must opt in.
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -28,6 +28,24 @@ const schemaFor = (kind) => {
|
|
|
28
28
|
return v;
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
+
const nestedErrors = (nested = [], prefix = '/nested', occupied = []) => {
|
|
32
|
+
const errors = [];
|
|
33
|
+
const projects = new Set();
|
|
34
|
+
const files = new Set(occupied);
|
|
35
|
+
for (const [i, entry] of nested.entries()) {
|
|
36
|
+
const p = `${prefix}/${i}`;
|
|
37
|
+
if (entry.project.split('/').some((part) => !part || part === '.' || part === '..') || entry.project.includes('\\')) errors.push({ path: `${p}/project`, message: 'nested project must be a canonical descendant directory' });
|
|
38
|
+
if (projects.has(entry.project)) errors.push({ path: `${p}/project`, message: 'nested project is duplicated' });
|
|
39
|
+
projects.add(entry.project);
|
|
40
|
+
for (const file of entry.files) {
|
|
41
|
+
if (!file.startsWith(`${entry.project}/`) || file.split('/').some((part) => !part || part === '.' || part === '..') || file.includes('\\')) errors.push({ path: `${p}/files`, message: 'nested file must remain beneath its canonical project directory' });
|
|
42
|
+
if (files.has(file)) errors.push({ path: `${p}/files`, message: 'nested file is counted in more than one bucket' });
|
|
43
|
+
files.add(file);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return errors;
|
|
47
|
+
};
|
|
48
|
+
|
|
31
49
|
// Rules JSON Schema cannot express. Each returns an array of {path, message}.
|
|
32
50
|
const semantic = {
|
|
33
51
|
claims(doc) {
|
|
@@ -243,6 +261,11 @@ const semantic = {
|
|
|
243
261
|
|
|
244
262
|
status(doc) {
|
|
245
263
|
const errors = [];
|
|
264
|
+
if (doc.changes?.nested) {
|
|
265
|
+
errors.push(...nestedErrors(doc.changes.nested, '/changes/nested', doc.changes.uncovered.map((file) => file.file)));
|
|
266
|
+
const nestedCount = doc.changes.nested.reduce((count, entry) => count + entry.files.length, 0);
|
|
267
|
+
if (doc.changes.changed !== doc.changes.evaluated + doc.changes.excluded + nestedCount) errors.push({ path: '/changes/changed', message: 'changed must account for evaluated, excluded, and nested files' });
|
|
268
|
+
}
|
|
246
269
|
if (['write-test', 'review-fault-change', 'repair-fault'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
|
|
247
270
|
if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
|
|
248
271
|
if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
|
|
@@ -311,8 +334,10 @@ const semantic = {
|
|
|
311
334
|
|
|
312
335
|
gate(doc) {
|
|
313
336
|
const errors = [];
|
|
337
|
+
const nestedCount = (doc.nested ?? []).reduce((count, entry) => count + entry.files.length, 0);
|
|
338
|
+
errors.push(...nestedErrors(doc.nested, '/nested', [...doc.covered, ...doc.uncovered, ...doc.excluded].map((entry) => entry.file)));
|
|
314
339
|
if (doc.evaluated !== doc.covered.length + doc.uncovered.length) errors.push({ path: '/evaluated', message: `evaluated (${doc.evaluated}) must equal covered (${doc.covered.length}) + uncovered (${doc.uncovered.length})` });
|
|
315
|
-
if (doc.changed !== doc.evaluated + doc.excluded.length) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length})` });
|
|
340
|
+
if (doc.changed !== doc.evaluated + doc.excluded.length + nestedCount) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length}) + nested (${nestedCount})` });
|
|
316
341
|
if (doc.uncovered.length > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'an uncovered changed file must exit 1; the gate never passes over unclaimed code' });
|
|
317
342
|
if (doc.uncovered.length === 0 && doc.exitCode === 1 && !(doc.strict && doc.changed > 0 && doc.evaluated === 0)) errors.push({ path: '/exitCode', message: 'exit 1 without uncovered files is only valid under --strict when a non-empty change evaluated nothing' });
|
|
318
343
|
doc.covered.forEach((c, i) => {
|
|
@@ -4,6 +4,16 @@
|
|
|
4
4
|
"title": "claimspec v1 \u2014 shared definitions",
|
|
5
5
|
"description": "Definitions shared by every Guard-spec format. Referenced as urn:claimspec:v1:common#/$defs/<name>.",
|
|
6
6
|
"$defs": {
|
|
7
|
+
"nestedProject": {
|
|
8
|
+
"description": "Changed files delegated to a separately validated descendant TestGuard project. This is not coverage or evidence; run that project's gate separately.",
|
|
9
|
+
"type": "object",
|
|
10
|
+
"required": ["project", "files"],
|
|
11
|
+
"properties": {
|
|
12
|
+
"project": { "$ref": "#/$defs/repoPath" },
|
|
13
|
+
"files": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "#/$defs/repoPath" } }
|
|
14
|
+
},
|
|
15
|
+
"additionalProperties": false
|
|
16
|
+
},
|
|
7
17
|
"sha256": {
|
|
8
18
|
"type": "string",
|
|
9
19
|
"pattern": "^[a-f0-9]{64}$"
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
18
|
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
19
19
|
"evaluated": { "description": "covered + uncovered: the changed files that are source and not excluded.", "type": "integer", "minimum": 0 },
|
|
20
|
+
"nested": { "description": "Optional descendant-project delegations; absence means none. These files are neither covered nor evaluated by the parent.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
20
21
|
"excluded": {
|
|
21
22
|
"type": "array",
|
|
22
23
|
"items": {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
27
27
|
},
|
|
28
28
|
"state": {
|
|
29
|
-
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
29
|
+
"description": "no-claims: no claims file or a valid empty claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). invalid-anchors: one or more exact fault anchors are missing or ambiguous, so probing would be unable to apply them. unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
30
30
|
"enum": [
|
|
31
31
|
"no-claims",
|
|
32
32
|
"unclaimed-changes",
|
|
@@ -392,6 +392,7 @@
|
|
|
392
392
|
"expired"
|
|
393
393
|
],
|
|
394
394
|
"properties": {
|
|
395
|
+
"nested": { "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/nestedProject" } },
|
|
395
396
|
"ref": {
|
|
396
397
|
"type": "string",
|
|
397
398
|
"minLength": 1,
|
package/src/cli.mjs
CHANGED
|
@@ -75,6 +75,7 @@ probe
|
|
|
75
75
|
--in-place mutate the working tree instead of a scratch worktree
|
|
76
76
|
--no-escalate do not re-run survivors against the whole suite
|
|
77
77
|
--no-reuse re-probe claims whose inputs have not changed
|
|
78
|
+
--allow-empty adoption only: skip a valid empty claims file; writes no evidence; run gate separately
|
|
78
79
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
79
80
|
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
80
81
|
|
|
@@ -134,6 +135,27 @@ exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or uncl
|
|
|
134
135
|
|
|
135
136
|
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand, concerns: concernsCommand };
|
|
136
137
|
|
|
138
|
+
/** Derive command help from the same option descriptions as the overview. */
|
|
139
|
+
export function commandUsage(command) {
|
|
140
|
+
const lines = USAGE.split('\n');
|
|
141
|
+
const overview = lines.find((line) => line.startsWith(` testguard ${command} `)) ?? ` testguard ${command}`;
|
|
142
|
+
const sections = [];
|
|
143
|
+
let selected = false;
|
|
144
|
+
for (const line of lines) {
|
|
145
|
+
const heading = /^([a-z]+)(?:\s|$)/.exec(line)?.[1];
|
|
146
|
+
if (heading && Object.hasOwn(COMMANDS, heading)) selected = heading === command;
|
|
147
|
+
else if (line.startsWith('every command ') || line.startsWith('exit codes:')) selected = false;
|
|
148
|
+
if (selected) sections.push(line);
|
|
149
|
+
}
|
|
150
|
+
if (command === 'concerns') sections.push('concerns --concerns <path> read a custom concerns file; --json prints the validated inventory');
|
|
151
|
+
const examples = {
|
|
152
|
+
probe: 'probe . --confirm 3', gate: 'gate . --changed origin/main',
|
|
153
|
+
scaffold: 'scaffold src/example.mjs --json', replay: 'replay . --since HEAD~10..HEAD --max 3',
|
|
154
|
+
admit: 'admit test/example.test.mjs --claim EXAMPLE-001', sweep: 'sweep . --changed origin/main --cap 10',
|
|
155
|
+
};
|
|
156
|
+
return `${lines[0]}\n\n${overview}\n\n${sections.join('\n').trim()}\n\nCommon: --json, --help, --version\nExample: testguard ${examples[command] ?? `${command}${command === 'mcp' ? '' : ' .'}`}\n`;
|
|
157
|
+
}
|
|
158
|
+
|
|
137
159
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
138
160
|
let parsed;
|
|
139
161
|
try {
|
|
@@ -180,6 +202,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
180
202
|
'in-place': { type: 'boolean', default: false },
|
|
181
203
|
'no-escalate': { type: 'boolean', default: false },
|
|
182
204
|
'no-reuse': { type: 'boolean', default: false },
|
|
205
|
+
'allow-empty': { type: 'boolean', default: false },
|
|
183
206
|
quiet: { type: 'boolean', default: false },
|
|
184
207
|
json: { type: 'boolean', default: false },
|
|
185
208
|
text: { type: 'boolean', default: false },
|
|
@@ -202,9 +225,9 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
202
225
|
return 0;
|
|
203
226
|
}
|
|
204
227
|
const [command, dirArg] = positionals;
|
|
205
|
-
if (
|
|
206
|
-
io.out(USAGE);
|
|
207
|
-
return
|
|
228
|
+
if (!command) {
|
|
229
|
+
io.out(USAGE + "\nRun 'testguard <command> --help' for that command's options.");
|
|
230
|
+
return values.help ? 0 : 3;
|
|
208
231
|
}
|
|
209
232
|
const handler = COMMANDS[command];
|
|
210
233
|
if (!handler) {
|
|
@@ -212,6 +235,10 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
212
235
|
io.err(USAGE);
|
|
213
236
|
return 3;
|
|
214
237
|
}
|
|
238
|
+
if (values.help) {
|
|
239
|
+
io.out(commandUsage(command));
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
215
242
|
if (![...RUNNER_NAMES, 'auto'].includes(values.runner)) {
|
|
216
243
|
io.err(`--runner must be one of ${RUNNER_NAMES.join(', ')} or auto`);
|
|
217
244
|
return 3;
|
package/src/commands/probe.mjs
CHANGED
|
@@ -57,6 +57,19 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
57
57
|
const stageOut = stageReporter(progress, (s) => progressTo.write(s));
|
|
58
58
|
const claims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
|
|
59
59
|
if (claims.claims.length === 0) {
|
|
60
|
+
if (values['allow-empty'] && values.claim === undefined) {
|
|
61
|
+
io.err('claims file declares no claims; verification skipped (--allow-empty); run gate to check changed-file coverage');
|
|
62
|
+
if (values.json) {
|
|
63
|
+
const doc = computeStatus({ projectDir, toolVersion: version, paths: {
|
|
64
|
+
claims: values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir),
|
|
65
|
+
evidence: evidencePath(projectDir), provisional: provisionalEvidencePath(projectDir), baseline: baselinePath(projectDir),
|
|
66
|
+
} });
|
|
67
|
+
const result = validate('status', doc);
|
|
68
|
+
if (!result.ok) throw new Error(`empty probe JSON document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
69
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
70
|
+
}
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
60
73
|
io.err('claims file declares no claims; nothing to verify');
|
|
61
74
|
return 2;
|
|
62
75
|
}
|
package/src/gate/changed.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { existsSync, realpathSync } from 'node:fs';
|
|
1
|
+
import { existsSync, realpathSync, lstatSync } from 'node:fs';
|
|
2
2
|
import { extname, join, relative, resolve, dirname } from 'node:path';
|
|
3
3
|
import { git, repoRoot, headSha, GitError } from '../git.mjs';
|
|
4
|
-
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath, ClaimsError } from '../claims/load.mjs';
|
|
5
5
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
@@ -49,6 +49,33 @@ export const isDefaultExcluded = (file) => DEFAULT_EXCLUDE_RES.some((re) => re.t
|
|
|
49
49
|
|
|
50
50
|
export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
|
|
51
51
|
|
|
52
|
+
/** Cache validated descendant boundaries for one measurement, never across runs. */
|
|
53
|
+
export function nestedProjectResolver(projectDir) {
|
|
54
|
+
const root = realpathSync(resolve(projectDir));
|
|
55
|
+
const markers = new Map();
|
|
56
|
+
return (file) => {
|
|
57
|
+
const parts = file.split('/').slice(0, -1);
|
|
58
|
+
for (let i = 1; i <= parts.length; i++) {
|
|
59
|
+
const project = parts.slice(0, i).join('/');
|
|
60
|
+
if (!markers.has(project)) {
|
|
61
|
+
const marker = join(root, project, 'testguard.claims.json');
|
|
62
|
+
let stat;
|
|
63
|
+
try { stat = lstatSync(marker); }
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') throw new ClaimsError(`cannot inspect nested claims file ${marker}: ${error.message}`);
|
|
66
|
+
}
|
|
67
|
+
if (stat) {
|
|
68
|
+
if (!stat.isFile() || stat.isSymbolicLink() || realpathSync(marker) !== marker) throw new ClaimsError(`nested claims file must be a regular file without symlink traversal: ${marker}`);
|
|
69
|
+
loadClaims(marker);
|
|
70
|
+
}
|
|
71
|
+
markers.set(project, Boolean(stat));
|
|
72
|
+
}
|
|
73
|
+
if (markers.get(project)) return project;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
52
79
|
/**
|
|
53
80
|
* The reference the change is measured against. Environment overrides and CI
|
|
54
81
|
* bases are handled here; local Git discovery is separate because it needs a
|
|
@@ -212,7 +239,15 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
212
239
|
const excluded = [];
|
|
213
240
|
const covered = [];
|
|
214
241
|
const uncovered = [];
|
|
242
|
+
const nested = new Map();
|
|
243
|
+
const ownerOf = nestedProjectResolver(projectDir);
|
|
215
244
|
for (const file of changed) {
|
|
245
|
+
const owner = ownerOf(file);
|
|
246
|
+
if (owner) {
|
|
247
|
+
if (!nested.has(owner)) nested.set(owner, []);
|
|
248
|
+
nested.get(owner).push(file);
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
216
251
|
if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
|
|
217
252
|
const ux = userExcludes.find((x) => x.re.test(file));
|
|
218
253
|
if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
|
|
@@ -256,6 +291,7 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
256
291
|
changed: changed.length,
|
|
257
292
|
evaluated,
|
|
258
293
|
excluded,
|
|
294
|
+
...(nested.size ? { nested: [...nested].map(([project, files]) => ({ project, files })) } : {}),
|
|
259
295
|
covered,
|
|
260
296
|
uncovered,
|
|
261
297
|
reliedOn,
|
|
@@ -271,7 +307,8 @@ export function renderGate(doc) {
|
|
|
271
307
|
for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
|
|
272
308
|
for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
273
309
|
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
|
|
274
|
-
|
|
310
|
+
for (const n of doc.nested ?? []) lines.push(`delegated ${n.files.length} file(s) belong to the TestGuard project at ${n.project}/; run testguard gate ${n.project} --changed ${doc.ref}${doc.includeDirty ? ' --include-dirty' : ''}; child coverage has not been evaluated here`);
|
|
311
|
+
if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded or delegated (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
|
|
275
312
|
if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
|
|
276
313
|
if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
|
|
277
314
|
return lines.join('\n');
|
|
@@ -20,7 +20,7 @@ which files exist. `state` is one of:
|
|
|
20
20
|
|
|
21
21
|
| state | meaning | you do |
|
|
22
22
|
|---|---|---|
|
|
23
|
-
| `no-claims` |
|
|
23
|
+
| `no-claims` | missing or empty `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with the intended guarantee from a spec, ticket, or past bug; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
24
|
| `unclaimed-changes` | files you changed carry no claim (when a reference is explicit, supplied by CI, or safely inferred from local Git's remote default/upstream metadata) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
25
25
|
| `invalid-anchors` | an exact fault anchor moved or became ambiguous | run `testguard claims --check-anchors`; repair `next.target` in the claims file without changing what the fault means, then re-probe. Never guess or auto-fix an anchor. |
|
|
26
26
|
| `unprobed` | claims never probed | `testguard probe` |
|
package/src/probe/probe.mjs
CHANGED
|
@@ -202,7 +202,7 @@ export async function probe({
|
|
|
202
202
|
commandBudget?.assertOpen();
|
|
203
203
|
if (sel.error) {
|
|
204
204
|
throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
|
|
205
|
-
(mode === 'worktree' ? '
|
|
205
|
+
(mode === 'worktree' ? 'If dependencies are missing, pass --node-modules <path>, or run with --in-place. Otherwise fix the discovery error above.' : 'If dependencies are missing, install them first. Otherwise fix the discovery error above.'));
|
|
206
206
|
}
|
|
207
207
|
runner = sel.runner;
|
|
208
208
|
runnerVersion = sel.version;
|
|
@@ -275,7 +275,6 @@ const inside = (root, path) => {
|
|
|
275
275
|
const relativeConfigPath = (root, path) => relative(root, path).split(sep).join('/');
|
|
276
276
|
|
|
277
277
|
const existingConfigCandidate = (root, path) => {
|
|
278
|
-
if (!inside(root, path)) throw new DiscoveryError(`discovery config dependency resolves outside the project: ${path}`);
|
|
279
278
|
let stat;
|
|
280
279
|
try {
|
|
281
280
|
stat = lstatSync(path);
|
|
@@ -285,6 +284,7 @@ const existingConfigCandidate = (root, path) => {
|
|
|
285
284
|
}
|
|
286
285
|
if (stat.isSymbolicLink()) throw new DiscoveryError(`discovery config dependency is not readable: symbolic links are not allowed`);
|
|
287
286
|
if (!stat.isFile()) return null;
|
|
287
|
+
if (!inside(root, path)) throw new DiscoveryError(`discovery config dependency resolves outside the project: ${path}`);
|
|
288
288
|
return path;
|
|
289
289
|
};
|
|
290
290
|
|
|
@@ -305,7 +305,9 @@ const resolveConfigReference = (root, fromPath, rawReference) => {
|
|
|
305
305
|
|
|
306
306
|
const resolved = new Set();
|
|
307
307
|
for (const base of bases) {
|
|
308
|
-
|
|
308
|
+
// Strings in executable configs also describe URLs and separators. Only
|
|
309
|
+
// existing file candidates are dependencies; the boundary check follows
|
|
310
|
+
// that distinction, while symlinks remain refused even when dangling.
|
|
309
311
|
const candidates = extname(base)
|
|
310
312
|
? [base]
|
|
311
313
|
: [base, ...RESOLVABLE_CONFIG_EXTENSIONS.map((extension) => `${base}${extension}`),
|
|
@@ -184,6 +184,21 @@ export function firstInformativeLine(message) {
|
|
|
184
184
|
return lines.find((l) => /error|syntax|unexpected|expected .+ but found|cannot find|failed to parse|transform failed/i.test(l) && !/^●/.test(l)) ?? lines[0] ?? '';
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
+
function isRunnerTimeout(test) {
|
|
188
|
+
if (test.failureType === 'testTimeoutFailure') return true;
|
|
189
|
+
const details = test.failureDetails ?? [];
|
|
190
|
+
if (details.some((error) => error?.failureType === 'testTimeoutFailure')) return true;
|
|
191
|
+
return (test.failureMessages ?? []).some((message, index) => {
|
|
192
|
+
const error = details[index];
|
|
193
|
+
// Assertion text may quote a timeout header verbatim. Inspect only the
|
|
194
|
+
// error header, never stack frames, source excerpts, or expected values.
|
|
195
|
+
if (error?.name === 'AssertionError' || error?.code === 'ERR_ASSERTION' || error?.operator !== undefined) return false;
|
|
196
|
+
const header = String(message).replace(/\x1b\[[0-9;]*m/g, '').trimStart().split('\n')[0].trimEnd();
|
|
197
|
+
return /^(?:Error:\s*)?(?:thrown:\s*")?(?:Exceeded timeout of \d+(?:\.\d+)?\s*ms for a (?:test|hook)\b|(?:Test|Hook) timed out in \d+(?:\.\d+)?ms\b|test timed out after \d+(?:\.\d+)?ms\b)/i.test(header)
|
|
198
|
+
|| /^(?:(?:Error|AroundHookSetupError|AroundHookTeardownError):\s*)?The (?:setup|teardown) phase of "around(?:Each|All)" hook timed out after \d+(?:\.\d+)?ms\b/.test(header);
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
187
202
|
export function parseReport(report, durationMs) {
|
|
188
203
|
// A report that parsed as JSON but carries no `testResults` is not a suite
|
|
189
204
|
// that ran zero tests — it is a report of a shape this parser does not
|
|
@@ -195,7 +210,7 @@ export function parseReport(report, durationMs) {
|
|
|
195
210
|
const files = report.testResults ?? [];
|
|
196
211
|
const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
|
|
197
212
|
const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
|
|
198
|
-
const timeouts = results.filter(
|
|
213
|
+
const timeouts = results.filter(isRunnerTimeout).length;
|
|
199
214
|
const failedTests = results.map((t) => t.id);
|
|
200
215
|
const assertionFailures = results.length - timeouts;
|
|
201
216
|
const tests = { total: report.numTotalTests ?? 0, passed: report.numPassedTests ?? 0, failed: report.numFailedTests ?? 0 };
|
package/src/status/status.mjs
CHANGED
|
@@ -64,6 +64,7 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
64
64
|
if (ref) {
|
|
65
65
|
const g = computeChangedGate({ projectDir, ref, includeDirty, toolVersion });
|
|
66
66
|
changes = { ref: g.ref, base: g.base, includeDirty: g.includeDirty, changed: g.changed, evaluated: g.evaluated, excluded: g.excluded.length, uncovered: g.uncovered, reliedOn: g.reliedOn, expired: g.expired };
|
|
67
|
+
if (g.nested?.length) changes.nested = g.nested;
|
|
67
68
|
doc.changes = changes;
|
|
68
69
|
if (existsSync(defaultIgnorePath(projectDir))) doc.paths.ignore = rel(defaultIgnorePath(projectDir));
|
|
69
70
|
}
|
|
@@ -90,6 +91,10 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
90
91
|
doc.counts.claims = claims.claims.length;
|
|
91
92
|
doc.counts.faults = claims.claims.reduce((n, c) => n + c.faults.length, 0);
|
|
92
93
|
doc.surface = computeClaimedSurface({ projectDir, claims });
|
|
94
|
+
if (claims.claims.length === 0 && !changes?.uncovered.length) {
|
|
95
|
+
doc.next = { action: 'scaffold', command: 'testguard scaffold <source-file>', why: 'No claims have been declared; no verification was performed. State a claim before probing.' };
|
|
96
|
+
return doc;
|
|
97
|
+
}
|
|
93
98
|
const faultIndex = new Map();
|
|
94
99
|
for (const c of claims.claims) for (const f of c.faults) faultIndex.set(`${c.id}/${f.id}`, { claim: c, fault: f });
|
|
95
100
|
|
|
@@ -281,6 +286,7 @@ export function renderStatus(doc) {
|
|
|
281
286
|
for (const u of c.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
282
287
|
for (const r of c.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}`);
|
|
283
288
|
for (const e of c.expired) lines.push(`EXPIRED ignore "${e.pattern}" no longer excuses ${e.files.join(', ')}`);
|
|
289
|
+
for (const n of c.nested ?? []) lines.push(`delegated ${n.files.length} file(s) belong to ${n.project}/; run testguard gate ${n.project} --changed ${c.ref}${c.includeDirty ? ' --include-dirty' : ''}; child coverage has not been evaluated here`);
|
|
284
290
|
}
|
|
285
291
|
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
286
292
|
for (const f of doc.invalidFaults ?? []) lines.push(`INVALID ${f.claimId}/${f.subjectId} ${f.reason}: ${f.hits} hits, expected ${f.expected} (${f.file})`);
|
package/src/status/surface.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { extname, join } from 'node:path';
|
|
3
3
|
import { git } from '../git.mjs';
|
|
4
|
-
import { SOURCE_EXT, isDefaultExcluded, isTestFile } from '../gate/changed.mjs';
|
|
4
|
+
import { SOURCE_EXT, isDefaultExcluded, isTestFile, nestedProjectResolver } from '../gate/changed.mjs';
|
|
5
5
|
import { walk } from '../util/glob.mjs';
|
|
6
6
|
|
|
7
7
|
export const HISTORY_COMMITS = 200;
|
|
@@ -24,11 +24,12 @@ function listedFiles(projectDir) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
function claimableModules(projectDir) {
|
|
27
|
+
const ownerOf = nestedProjectResolver(projectDir);
|
|
27
28
|
return listedFiles(projectDir).filter((file) =>
|
|
28
29
|
existsSync(join(projectDir, file)) &&
|
|
29
30
|
SOURCE_EXT.has(extname(file)) &&
|
|
30
31
|
!isTestFile(file) &&
|
|
31
|
-
!isDefaultExcluded(file));
|
|
32
|
+
!isDefaultExcluded(file) && !ownerOf(file));
|
|
32
33
|
}
|
|
33
34
|
|
|
34
35
|
function recentChanges(projectDir, sourceModules) {
|