testguard-cli 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/scripts/install-smoke.mjs +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CHANGELOG.md +44 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/PKG-INFO +29 -9
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/README.md +28 -8
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/action.yml +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/README.md +1 -0
- testguard_cli-0.1.3/fixtures/known-answer/expected.json +53 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/testguard.claims.json +155 -34
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/package-lock.json +2 -2
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/package.json +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/packaging/homebrew/testguard.rb +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/pyproject.toml +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/GATE-SEMANTICS.md +10 -4
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/baseline.json +2 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/evidence.json +9 -3
- testguard_cli-0.1.3/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +537 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/lib/validate.mjs +3 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/baseline.schema.json +4 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/claims.schema.json +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/evidence.schema.json +26 -2
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/baseline/baseline.mjs +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/brief/brief.mjs +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/cli.mjs +4 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/baseline.mjs +1 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/claims.mjs +4 -2
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/probe.mjs +11 -2
- testguard_cli-0.1.3/src/git.mjs +58 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/classify.mjs +2 -1
- testguard_cli-0.1.3/src/probe/discover.mjs +12 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/probe.mjs +47 -10
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/rank.mjs +14 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/render.mjs +13 -3
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/brief.test.mjs +12 -1
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/classify.test.mjs +2 -0
- testguard_cli-0.1.3/test/discover.test.mjs +22 -0
- testguard_cli-0.1.3/test/probe-preconditions.test.mjs +37 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/probe.fixture.test.mjs +56 -1
- testguard_cli-0.1.3/test/snapshot.test.mjs +38 -0
- testguard_cli-0.1.3/testguard.claims.json +316 -0
- testguard_cli-0.1.2/fixtures/known-answer/expected.json +0 -15
- testguard_cli-0.1.2/src/git.mjs +0 -32
- testguard_cli-0.1.2/testguard.claims.json +0 -120
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.gitattributes +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/dependabot.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/scripts/sync-release-version.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/ci.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.gitignore +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.npmignore +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.npmrc +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.pre-commit-hooks.yaml +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/AGENTS.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CLAUDE.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CONTRIBUTING.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/LICENSE +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/PRIVACY.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/SECURITY.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/SUPPORT.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/bench/README.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/cli/testguard.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/README.md +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/brief.schema.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/common.schema.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/claims/load.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/brief.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/runner-vitest.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/util/glob.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/util/hash.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/glob.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/inject.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/runner-command.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/runner-vitest.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/writer.test.mjs +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.1.2 → testguard_cli-0.1.3}/vitest.config.mjs +0 -0
|
@@ -31,7 +31,7 @@ try {
|
|
|
31
31
|
const version = run(bin, ['--version'], consumer).trim();
|
|
32
32
|
const claims = run(bin, ['claims', '.'], consumer);
|
|
33
33
|
if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
|
|
34
|
-
if (
|
|
34
|
+
if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
|
|
35
35
|
const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
|
|
36
36
|
console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
|
|
37
37
|
} finally {
|
|
@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.1.3] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
From a second field report on a real codebase (456 tests, 27 claims, 35
|
|
13
|
+
faults; 13 survived on the first run, two critical claims with zero coverage).
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- **Worktree mode probed HEAD while reading the claims file from the working
|
|
18
|
+
tree**, so uncommitted defender changes were silently ignored — the same
|
|
19
|
+
survivors came back with no hint why. `probe` now refuses (exit 2) when any
|
|
20
|
+
resolved defender or fault target has uncommitted changes, naming the files
|
|
21
|
+
and the commit it would have probed. Every summary names the commit probed.
|
|
22
|
+
- `killed-by-undeclared-tests` never said which tests killed the fault; the
|
|
23
|
+
author could not fix `defendedBy` without grepping the suite. Evidence now
|
|
24
|
+
carries `detail.undeclaredKillers` and the CLI names the files.
|
|
25
|
+
- `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
|
|
26
|
+
expected }` is recorded and printed.
|
|
27
|
+
- A replacement with an unbalanced paren was `suite-failed-to-load`, not
|
|
28
|
+
`replacement-does-not-compile`: esbuild/vitest wording is now matched.
|
|
29
|
+
- The claims schema promised defender discovery for an absent `defendedBy`;
|
|
30
|
+
the tool answered `nocover`. Discovery is implemented: the test files that
|
|
31
|
+
import the fault's target (relative or alias), recorded as
|
|
32
|
+
`defenders.discovered`. `nocover` now means exactly "no test file imports
|
|
33
|
+
this source".
|
|
34
|
+
- `baseline.json` records `dirty`, as evidence already did.
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- `--include-dirty`: snapshot the working tree (tracked edits and untracked,
|
|
39
|
+
non-ignored files) into a throwaway commit and probe that. HEAD, index and
|
|
40
|
+
the user's tree are never touched; `run.repo.snapshot` records the commit.
|
|
41
|
+
- A one-line progress indicator on stderr (TTY only) so a minute of silence
|
|
42
|
+
is not mistaken for a hang.
|
|
43
|
+
- Fixture: a claim with no `defendedBy` whose defender is discovered.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- **Default output shows only unproven faults plus a killed count.**
|
|
48
|
+
`--verbose` restores the full stream.
|
|
49
|
+
- The `survived` hint reminds the author to check that the fault is
|
|
50
|
+
observable at all before writing a test for it.
|
|
51
|
+
- README: `$schema` path for consumers, `min-release-age` note, the
|
|
52
|
+
worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
|
|
53
|
+
|
|
10
54
|
## [0.1.2] - 2026-09-17
|
|
11
55
|
|
|
12
56
|
From a field report on a real codebase (63 test files, 458 tests, 39 faults).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -76,9 +76,13 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
76
76
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
77
77
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
78
78
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
79
|
-
| GitHub Action | `uses: raccioly/testguard@v0.1.
|
|
79
|
+
| GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
|
|
80
80
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
81
81
|
|
|
82
|
+
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
83
|
+
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
84
|
+
`npm i -D testguard-cli --min-release-age=0`.
|
|
85
|
+
|
|
82
86
|
## How it works
|
|
83
87
|
|
|
84
88
|
```bash
|
|
@@ -88,9 +92,11 @@ npx testguard-cli baseline # freeze today's unproven findings; from now on on
|
|
|
88
92
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
89
93
|
```
|
|
90
94
|
|
|
91
|
-
1. **Claims** live in `testguard.claims.json
|
|
92
|
-
|
|
93
|
-
|
|
95
|
+
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
96
|
+
`"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
|
|
97
|
+
a statement, where it comes from, which tests supposedly defend it, and
|
|
98
|
+
one or more *faults* — each a deterministic source change that would make
|
|
99
|
+
the statement false. Every
|
|
94
100
|
claim and every fault records who produced it. `testguard claims`
|
|
95
101
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
96
102
|
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
@@ -115,10 +121,23 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
115
121
|
written to `.testguard/evidence.json` — validated against the spec before
|
|
116
122
|
it is written.
|
|
117
123
|
|
|
124
|
+
Worktree mode probes a **commit**. If a defender or target file has
|
|
125
|
+
uncommitted changes, `probe` refuses and says so — otherwise your new
|
|
126
|
+
tests would be silently absent and the same survivors would come back
|
|
127
|
+
with no hint why. `--include-dirty` snapshots the working tree (tracked
|
|
128
|
+
edits and new files) into a throwaway commit and probes that; your tree,
|
|
129
|
+
HEAD and index are never touched. Every summary names the commit probed.
|
|
130
|
+
|
|
131
|
+
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
132
|
+
files that import the fault's target, by relative path or resolved alias.
|
|
133
|
+
`NOCOVER` then means exactly "no test file imports this source".
|
|
134
|
+
|
|
118
135
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
119
|
-
suite N times per survivor); iterate on one claim with
|
|
120
|
-
`--
|
|
121
|
-
|
|
136
|
+
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
137
|
+
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
138
|
+
clean there; test files may be dirty); final pass with defaults. By default
|
|
139
|
+
the stream shows only unproven faults plus a killed count — `--verbose`
|
|
140
|
+
shows every fault. A custom
|
|
122
141
|
runner (`pnpm --filter`, a specific config) goes in
|
|
123
142
|
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
124
143
|
see your `node_modules`, pass `--node-modules <dir>`.
|
|
@@ -127,7 +146,8 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
127
146
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
128
147
|
in CI costs only what changed.
|
|
129
148
|
4. **Brief** turns evidence plus baseline into a ranked, capped
|
|
130
|
-
`## TEST BLINDSPOT CONTEXT` block
|
|
149
|
+
`## TEST BLINDSPOT CONTEXT` block, printed and also written to
|
|
150
|
+
`.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
|
|
131
151
|
— for Claude Code, in `.claude/settings.json`:
|
|
132
152
|
|
|
133
153
|
```json
|
|
@@ -52,9 +52,13 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.1.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
|
+
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
59
|
+
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
60
|
+
`npm i -D testguard-cli --min-release-age=0`.
|
|
61
|
+
|
|
58
62
|
## How it works
|
|
59
63
|
|
|
60
64
|
```bash
|
|
@@ -64,9 +68,11 @@ npx testguard-cli baseline # freeze today's unproven findings; from now on on
|
|
|
64
68
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
65
69
|
```
|
|
66
70
|
|
|
67
|
-
1. **Claims** live in `testguard.claims.json
|
|
68
|
-
|
|
69
|
-
|
|
71
|
+
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
72
|
+
`"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
|
|
73
|
+
a statement, where it comes from, which tests supposedly defend it, and
|
|
74
|
+
one or more *faults* — each a deterministic source change that would make
|
|
75
|
+
the statement false. Every
|
|
70
76
|
claim and every fault records who produced it. `testguard claims`
|
|
71
77
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
72
78
|
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
@@ -91,10 +97,23 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
91
97
|
written to `.testguard/evidence.json` — validated against the spec before
|
|
92
98
|
it is written.
|
|
93
99
|
|
|
100
|
+
Worktree mode probes a **commit**. If a defender or target file has
|
|
101
|
+
uncommitted changes, `probe` refuses and says so — otherwise your new
|
|
102
|
+
tests would be silently absent and the same survivors would come back
|
|
103
|
+
with no hint why. `--include-dirty` snapshots the working tree (tracked
|
|
104
|
+
edits and new files) into a throwaway commit and probes that; your tree,
|
|
105
|
+
HEAD and index are never touched. Every summary names the commit probed.
|
|
106
|
+
|
|
107
|
+
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
108
|
+
files that import the fault's target, by relative path or resolved alias.
|
|
109
|
+
`NOCOVER` then means exactly "no test file imports this source".
|
|
110
|
+
|
|
94
111
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
95
|
-
suite N times per survivor); iterate on one claim with
|
|
96
|
-
`--
|
|
97
|
-
|
|
112
|
+
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
113
|
+
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
114
|
+
clean there; test files may be dirty); final pass with defaults. By default
|
|
115
|
+
the stream shows only unproven faults plus a killed count — `--verbose`
|
|
116
|
+
shows every fault. A custom
|
|
98
117
|
runner (`pnpm --filter`, a specific config) goes in
|
|
99
118
|
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
100
119
|
see your `node_modules`, pass `--node-modules <dir>`.
|
|
@@ -103,7 +122,8 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
103
122
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
104
123
|
in CI costs only what changed.
|
|
105
124
|
4. **Brief** turns evidence plus baseline into a ranked, capped
|
|
106
|
-
`## TEST BLINDSPOT CONTEXT` block
|
|
125
|
+
`## TEST BLINDSPOT CONTEXT` block, printed and also written to
|
|
126
|
+
`.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
|
|
107
127
|
— for Claude Code, in `.claude/settings.json`:
|
|
108
128
|
|
|
109
129
|
```json
|
|
@@ -24,6 +24,7 @@ evidence comes from running against a real codebase with real history.
|
|
|
24
24
|
| REDACT-006 | `mask()` replaces with equal-length asterisks | F1 syntax error | fault-invalid |
|
|
25
25
|
| EXPORT-001 | Exported rows never include `content` | F1 keeps content | nocover |
|
|
26
26
|
| FLAKY-001 | (defender is flaky) | F1 anything | flaky-defender |
|
|
27
|
+
| DISCOVER-001 | Every match is replaced (no `defendedBy`) | F1 loop body removed | killed — defenders **discovered** by import |
|
|
27
28
|
|
|
28
29
|
## The exhibit — REDACT-001/F1
|
|
29
30
|
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"confirmRuns": 3,
|
|
3
|
+
"expected": {
|
|
4
|
+
"REDACT-001/F1": {
|
|
5
|
+
"verdict": "survived",
|
|
6
|
+
"why": "The audit-row test asserts with objectContaining and omits `content`; a row carrying the raw input passes."
|
|
7
|
+
},
|
|
8
|
+
"REDACT-001/F2": {
|
|
9
|
+
"verdict": "killed",
|
|
10
|
+
"why": "The masked-output test asserts the secret is absent."
|
|
11
|
+
},
|
|
12
|
+
"REDACT-002/F1": {
|
|
13
|
+
"verdict": "killed",
|
|
14
|
+
"why": "The invalid-rule test asserts compileRules does not throw."
|
|
15
|
+
},
|
|
16
|
+
"REDACT-003/F1": {
|
|
17
|
+
"verdict": "survived",
|
|
18
|
+
"why": "No test exercises a missing scope; the fail-closed guard is undefended."
|
|
19
|
+
},
|
|
20
|
+
"REDACT-004/F1": {
|
|
21
|
+
"verdict": "timeout",
|
|
22
|
+
"why": "Tests awaiting redact() hit the 1s test timeout; a timeout is not an assertion failure and is not a kill."
|
|
23
|
+
},
|
|
24
|
+
"REDACT-005/F1": {
|
|
25
|
+
"verdict": "unverifiable",
|
|
26
|
+
"reason": "anchor-missing",
|
|
27
|
+
"why": "The find string is not present in the source."
|
|
28
|
+
},
|
|
29
|
+
"REDACT-005/F2": {
|
|
30
|
+
"verdict": "unverifiable",
|
|
31
|
+
"reason": "anchor-ambiguous",
|
|
32
|
+
"why": "`return null;` occurs twice; expectHits is 1."
|
|
33
|
+
},
|
|
34
|
+
"REDACT-006/F1": {
|
|
35
|
+
"verdict": "fault-invalid",
|
|
36
|
+
"reason": "replacement-does-not-compile",
|
|
37
|
+
"why": "The replacement introduces a syntax error; the suite cannot load."
|
|
38
|
+
},
|
|
39
|
+
"EXPORT-001/F1": {
|
|
40
|
+
"verdict": "nocover",
|
|
41
|
+
"why": "test/export.test.mjs does not exist; no defender resolves."
|
|
42
|
+
},
|
|
43
|
+
"FLAKY-001/F1": {
|
|
44
|
+
"verdict": "flaky-defender",
|
|
45
|
+
"reason": "defenders-not-green",
|
|
46
|
+
"why": "test/flaky.test.mjs fails every other run; the baseline is not green N/N."
|
|
47
|
+
},
|
|
48
|
+
"DISCOVER-001/F1": {
|
|
49
|
+
"verdict": "killed",
|
|
50
|
+
"why": "No defendedBy; test/redact.test.mjs is discovered because it imports src/redact.mjs, and its masked-output test fails."
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -4,10 +4,19 @@
|
|
|
4
4
|
{
|
|
5
5
|
"id": "REDACT-001",
|
|
6
6
|
"statement": "The audit row written for a masked message never contains the original input.",
|
|
7
|
-
"source": {
|
|
7
|
+
"source": {
|
|
8
|
+
"kind": "spec",
|
|
9
|
+
"ref": "README.md#claims"
|
|
10
|
+
},
|
|
8
11
|
"severity": "critical",
|
|
9
|
-
"producedBy": {
|
|
10
|
-
|
|
12
|
+
"producedBy": {
|
|
13
|
+
"producer": "human",
|
|
14
|
+
"by": "fixture author",
|
|
15
|
+
"at": "2026-09-17T00:00:00Z"
|
|
16
|
+
},
|
|
17
|
+
"defendedBy": [
|
|
18
|
+
"test/redact.test.mjs"
|
|
19
|
+
],
|
|
11
20
|
"faults": [
|
|
12
21
|
{
|
|
13
22
|
"id": "F1",
|
|
@@ -16,7 +25,10 @@
|
|
|
16
25
|
"file": "src/redact.mjs",
|
|
17
26
|
"find": "content: redacted,",
|
|
18
27
|
"replace": "content: input,",
|
|
19
|
-
"producedBy": {
|
|
28
|
+
"producedBy": {
|
|
29
|
+
"producer": "human",
|
|
30
|
+
"by": "fixture author"
|
|
31
|
+
}
|
|
20
32
|
},
|
|
21
33
|
{
|
|
22
34
|
"id": "F2",
|
|
@@ -25,17 +37,28 @@
|
|
|
25
37
|
"file": "src/redact.mjs",
|
|
26
38
|
"find": "const redacted = mask(input, rules);",
|
|
27
39
|
"replace": "const redacted = input;",
|
|
28
|
-
"producedBy": {
|
|
40
|
+
"producedBy": {
|
|
41
|
+
"producer": "human",
|
|
42
|
+
"by": "fixture author"
|
|
43
|
+
}
|
|
29
44
|
}
|
|
30
45
|
]
|
|
31
46
|
},
|
|
32
47
|
{
|
|
33
48
|
"id": "REDACT-002",
|
|
34
49
|
"statement": "A rule whose pattern is invalid is skipped; it never aborts the scan.",
|
|
35
|
-
"source": {
|
|
50
|
+
"source": {
|
|
51
|
+
"kind": "spec",
|
|
52
|
+
"ref": "README.md#claims"
|
|
53
|
+
},
|
|
36
54
|
"severity": "high",
|
|
37
|
-
"producedBy": {
|
|
38
|
-
|
|
55
|
+
"producedBy": {
|
|
56
|
+
"producer": "human",
|
|
57
|
+
"by": "fixture author"
|
|
58
|
+
},
|
|
59
|
+
"defendedBy": [
|
|
60
|
+
"test/redact.test.mjs"
|
|
61
|
+
],
|
|
39
62
|
"faults": [
|
|
40
63
|
{
|
|
41
64
|
"id": "F1",
|
|
@@ -44,17 +67,28 @@
|
|
|
44
67
|
"file": "src/redact.mjs",
|
|
45
68
|
"find": " } catch {\n return null;\n }",
|
|
46
69
|
"replace": " } catch (e) {\n throw e;\n }",
|
|
47
|
-
"producedBy": {
|
|
70
|
+
"producedBy": {
|
|
71
|
+
"producer": "human",
|
|
72
|
+
"by": "fixture author"
|
|
73
|
+
}
|
|
48
74
|
}
|
|
49
75
|
]
|
|
50
76
|
},
|
|
51
77
|
{
|
|
52
78
|
"id": "REDACT-003",
|
|
53
79
|
"statement": "A missing scope is an error; the engine never performs an unscoped scan.",
|
|
54
|
-
"source": {
|
|
80
|
+
"source": {
|
|
81
|
+
"kind": "annotation",
|
|
82
|
+
"ref": "src/redact.mjs @claim REDACT-003"
|
|
83
|
+
},
|
|
55
84
|
"severity": "high",
|
|
56
|
-
"producedBy": {
|
|
57
|
-
|
|
85
|
+
"producedBy": {
|
|
86
|
+
"producer": "human",
|
|
87
|
+
"by": "fixture author"
|
|
88
|
+
},
|
|
89
|
+
"defendedBy": [
|
|
90
|
+
"test/redact.test.mjs"
|
|
91
|
+
],
|
|
58
92
|
"faults": [
|
|
59
93
|
{
|
|
60
94
|
"id": "F1",
|
|
@@ -63,17 +97,28 @@
|
|
|
63
97
|
"file": "src/redact.mjs",
|
|
64
98
|
"find": "if (!ctx || !ctx.scope) {",
|
|
65
99
|
"replace": "if (false) {",
|
|
66
|
-
"producedBy": {
|
|
100
|
+
"producedBy": {
|
|
101
|
+
"producer": "human",
|
|
102
|
+
"by": "fixture author"
|
|
103
|
+
}
|
|
67
104
|
}
|
|
68
105
|
]
|
|
69
106
|
},
|
|
70
107
|
{
|
|
71
108
|
"id": "REDACT-004",
|
|
72
109
|
"statement": "redact() settles for every input.",
|
|
73
|
-
"source": {
|
|
110
|
+
"source": {
|
|
111
|
+
"kind": "spec",
|
|
112
|
+
"ref": "README.md#claims"
|
|
113
|
+
},
|
|
74
114
|
"severity": "medium",
|
|
75
|
-
"producedBy": {
|
|
76
|
-
|
|
115
|
+
"producedBy": {
|
|
116
|
+
"producer": "human",
|
|
117
|
+
"by": "fixture author"
|
|
118
|
+
},
|
|
119
|
+
"defendedBy": [
|
|
120
|
+
"test/redact.test.mjs"
|
|
121
|
+
],
|
|
77
122
|
"faults": [
|
|
78
123
|
{
|
|
79
124
|
"id": "F1",
|
|
@@ -82,17 +127,28 @@
|
|
|
82
127
|
"file": "src/redact.mjs",
|
|
83
128
|
"find": " return redacted;\n}",
|
|
84
129
|
"replace": " return new Promise(() => {});\n}",
|
|
85
|
-
"producedBy": {
|
|
130
|
+
"producedBy": {
|
|
131
|
+
"producer": "human",
|
|
132
|
+
"by": "fixture author"
|
|
133
|
+
}
|
|
86
134
|
}
|
|
87
135
|
]
|
|
88
136
|
},
|
|
89
137
|
{
|
|
90
138
|
"id": "REDACT-005",
|
|
91
139
|
"statement": "findRule() returns null for an unknown id.",
|
|
92
|
-
"source": {
|
|
140
|
+
"source": {
|
|
141
|
+
"kind": "spec",
|
|
142
|
+
"ref": "README.md#claims"
|
|
143
|
+
},
|
|
93
144
|
"severity": "low",
|
|
94
|
-
"producedBy": {
|
|
95
|
-
|
|
145
|
+
"producedBy": {
|
|
146
|
+
"producer": "human",
|
|
147
|
+
"by": "fixture author"
|
|
148
|
+
},
|
|
149
|
+
"defendedBy": [
|
|
150
|
+
"test/redact.test.mjs"
|
|
151
|
+
],
|
|
96
152
|
"faults": [
|
|
97
153
|
{
|
|
98
154
|
"id": "F1",
|
|
@@ -101,7 +157,10 @@
|
|
|
101
157
|
"file": "src/redact.mjs",
|
|
102
158
|
"find": "return undefined; // unknown rule",
|
|
103
159
|
"replace": "return rules[0];",
|
|
104
|
-
"producedBy": {
|
|
160
|
+
"producedBy": {
|
|
161
|
+
"producer": "human",
|
|
162
|
+
"by": "fixture author"
|
|
163
|
+
}
|
|
105
164
|
},
|
|
106
165
|
{
|
|
107
166
|
"id": "F2",
|
|
@@ -111,17 +170,28 @@
|
|
|
111
170
|
"find": "return null;",
|
|
112
171
|
"replace": "return rules[0];",
|
|
113
172
|
"expectHits": 1,
|
|
114
|
-
"producedBy": {
|
|
173
|
+
"producedBy": {
|
|
174
|
+
"producer": "human",
|
|
175
|
+
"by": "fixture author"
|
|
176
|
+
}
|
|
115
177
|
}
|
|
116
178
|
]
|
|
117
179
|
},
|
|
118
180
|
{
|
|
119
181
|
"id": "REDACT-006",
|
|
120
182
|
"statement": "mask() replaces each match with asterisks of equal length.",
|
|
121
|
-
"source": {
|
|
183
|
+
"source": {
|
|
184
|
+
"kind": "spec",
|
|
185
|
+
"ref": "README.md#claims"
|
|
186
|
+
},
|
|
122
187
|
"severity": "low",
|
|
123
|
-
"producedBy": {
|
|
124
|
-
|
|
188
|
+
"producedBy": {
|
|
189
|
+
"producer": "human",
|
|
190
|
+
"by": "fixture author"
|
|
191
|
+
},
|
|
192
|
+
"defendedBy": [
|
|
193
|
+
"test/redact.test.mjs"
|
|
194
|
+
],
|
|
125
195
|
"faults": [
|
|
126
196
|
{
|
|
127
197
|
"id": "F1",
|
|
@@ -130,17 +200,28 @@
|
|
|
130
200
|
"file": "src/redact.mjs",
|
|
131
201
|
"find": "(m) => '*'.repeat(m.length)",
|
|
132
202
|
"replace": "(m) => '*'.repeat(m.length",
|
|
133
|
-
"producedBy": {
|
|
203
|
+
"producedBy": {
|
|
204
|
+
"producer": "human",
|
|
205
|
+
"by": "fixture author"
|
|
206
|
+
}
|
|
134
207
|
}
|
|
135
208
|
]
|
|
136
209
|
},
|
|
137
210
|
{
|
|
138
211
|
"id": "EXPORT-001",
|
|
139
212
|
"statement": "Exported rows never include the content field.",
|
|
140
|
-
"source": {
|
|
213
|
+
"source": {
|
|
214
|
+
"kind": "spec",
|
|
215
|
+
"ref": "README.md#claims"
|
|
216
|
+
},
|
|
141
217
|
"severity": "medium",
|
|
142
|
-
"producedBy": {
|
|
143
|
-
|
|
218
|
+
"producedBy": {
|
|
219
|
+
"producer": "human",
|
|
220
|
+
"by": "fixture author"
|
|
221
|
+
},
|
|
222
|
+
"defendedBy": [
|
|
223
|
+
"test/export.test.mjs"
|
|
224
|
+
],
|
|
144
225
|
"faults": [
|
|
145
226
|
{
|
|
146
227
|
"id": "F1",
|
|
@@ -149,17 +230,27 @@
|
|
|
149
230
|
"file": "src/export.mjs",
|
|
150
231
|
"find": "return rows.map(({ content, ...rest }) => rest);",
|
|
151
232
|
"replace": "return rows.map((row) => ({ ...row }));",
|
|
152
|
-
"producedBy": {
|
|
233
|
+
"producedBy": {
|
|
234
|
+
"producer": "human",
|
|
235
|
+
"by": "fixture author"
|
|
236
|
+
}
|
|
153
237
|
}
|
|
154
238
|
]
|
|
155
239
|
},
|
|
156
240
|
{
|
|
157
241
|
"id": "FLAKY-001",
|
|
158
242
|
"statement": "A claim whose defender is not reliably green cannot be verified.",
|
|
159
|
-
"source": {
|
|
243
|
+
"source": {
|
|
244
|
+
"kind": "manual"
|
|
245
|
+
},
|
|
160
246
|
"severity": "low",
|
|
161
|
-
"producedBy": {
|
|
162
|
-
|
|
247
|
+
"producedBy": {
|
|
248
|
+
"producer": "human",
|
|
249
|
+
"by": "fixture author"
|
|
250
|
+
},
|
|
251
|
+
"defendedBy": [
|
|
252
|
+
"test/flaky.test.mjs"
|
|
253
|
+
],
|
|
163
254
|
"faults": [
|
|
164
255
|
{
|
|
165
256
|
"id": "F1",
|
|
@@ -168,7 +259,37 @@
|
|
|
168
259
|
"file": "src/redact.mjs",
|
|
169
260
|
"find": "action: 'MASK',",
|
|
170
261
|
"replace": "action: 'mask',",
|
|
171
|
-
"producedBy": {
|
|
262
|
+
"producedBy": {
|
|
263
|
+
"producer": "human",
|
|
264
|
+
"by": "fixture author"
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
]
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
"id": "DISCOVER-001",
|
|
271
|
+
"statement": "Every rule match is replaced in the returned text.",
|
|
272
|
+
"source": {
|
|
273
|
+
"kind": "spec",
|
|
274
|
+
"ref": "README.md#claims"
|
|
275
|
+
},
|
|
276
|
+
"severity": "high",
|
|
277
|
+
"producedBy": {
|
|
278
|
+
"producer": "human",
|
|
279
|
+
"by": "fixture author"
|
|
280
|
+
},
|
|
281
|
+
"faults": [
|
|
282
|
+
{
|
|
283
|
+
"id": "F1",
|
|
284
|
+
"description": "The replacement loop body is removed; nothing is masked. No defendedBy is declared: defenders must be discovered as the tests that import src/redact.mjs.",
|
|
285
|
+
"faultClass": "statement-deleted",
|
|
286
|
+
"file": "src/redact.mjs",
|
|
287
|
+
"find": "out = out.replace(rule.re, (m) => '*'.repeat(m.length));",
|
|
288
|
+
"replace": "/* fault: replacement removed */",
|
|
289
|
+
"producedBy": {
|
|
290
|
+
"producer": "human",
|
|
291
|
+
"by": "fixture author"
|
|
292
|
+
}
|
|
172
293
|
}
|
|
173
294
|
]
|
|
174
295
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.1.
|
|
9
|
+
"version": "0.1.3",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"ajv": "8.20.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
class Testguard < Formula
|
|
14
14
|
desc "Proves a test suite defends the claims a project makes"
|
|
15
15
|
homepage "https://github.com/raccioly/testguard"
|
|
16
|
-
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.
|
|
16
|
+
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.3.tgz"
|
|
17
17
|
sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
|
|
18
18
|
license "MIT"
|
|
19
19
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testguard-cli"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -35,10 +35,16 @@ Rules that follow from the table:
|
|
|
35
35
|
reporting `survived` would be wrong the other way. It is `flaky-defender`.
|
|
36
36
|
5. **Escalation never upgrades a verdict.** A fault that survives its declared
|
|
37
37
|
defenders may be re-run against the whole suite. If the wider suite kills
|
|
38
|
-
it, the verdict stays `survived` with reason `killed-by-undeclared-tests
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
it, the verdict stays `survived` with reason `killed-by-undeclared-tests`
|
|
39
|
+
and `detail.undeclaredKillers` names the tests, so the author can fix
|
|
40
|
+
`defendedBy`. The claim's stated evidence chain is broken even though the
|
|
41
|
+
suite is not blind. It gates, and ranks below a true survivor.
|
|
42
|
+
6. **A verdict names the commit it is about.** Evidence records `repo.head`;
|
|
43
|
+
when the working tree was probed instead, `repo.snapshot` holds the
|
|
44
|
+
throwaway commit that captured it. A tool must refuse to probe a commit
|
|
45
|
+
while defenders or targets have uncommitted changes, unless told to
|
|
46
|
+
snapshot the working tree — otherwise the answer looks right and is not.
|
|
47
|
+
7. **Never a single global score.** Output is per claim, ranked. Blindness is
|
|
42
48
|
concentrated, and one number hides where.
|
|
43
49
|
|
|
44
50
|
## Baseline and delta
|