testguard-cli 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/scripts/install-smoke.mjs +1 -1
  2. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CHANGELOG.md +44 -0
  3. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/PKG-INFO +29 -9
  4. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/README.md +28 -8
  5. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/action.yml +1 -1
  6. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/README.md +1 -0
  7. testguard_cli-0.1.3/fixtures/known-answer/expected.json +53 -0
  8. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/testguard.claims.json +155 -34
  9. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/package-lock.json +2 -2
  10. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/package.json +1 -1
  11. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/packaging/homebrew/testguard.rb +1 -1
  12. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/pyproject.toml +1 -1
  13. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/GATE-SEMANTICS.md +10 -4
  14. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/baseline.json +2 -1
  15. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/evidence.json +9 -3
  16. testguard_cli-0.1.3/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +537 -0
  17. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/lib/validate.mjs +3 -0
  18. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/baseline.schema.json +4 -0
  19. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/claims.schema.json +1 -1
  20. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/evidence.schema.json +26 -2
  21. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/baseline/baseline.mjs +1 -1
  22. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/brief/brief.mjs +1 -1
  23. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/cli.mjs +4 -0
  24. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/baseline.mjs +1 -1
  25. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/claims.mjs +4 -2
  26. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/probe.mjs +11 -2
  27. testguard_cli-0.1.3/src/git.mjs +58 -0
  28. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/classify.mjs +2 -1
  29. testguard_cli-0.1.3/src/probe/discover.mjs +12 -0
  30. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/probe.mjs +47 -10
  31. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/rank.mjs +14 -0
  32. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/render.mjs +13 -3
  33. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/brief.test.mjs +12 -1
  34. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/classify.test.mjs +2 -0
  35. testguard_cli-0.1.3/test/discover.test.mjs +22 -0
  36. testguard_cli-0.1.3/test/probe-preconditions.test.mjs +37 -0
  37. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/probe.fixture.test.mjs +56 -1
  38. testguard_cli-0.1.3/test/snapshot.test.mjs +38 -0
  39. testguard_cli-0.1.3/testguard.claims.json +316 -0
  40. testguard_cli-0.1.2/fixtures/known-answer/expected.json +0 -15
  41. testguard_cli-0.1.2/src/git.mjs +0 -32
  42. testguard_cli-0.1.2/testguard.claims.json +0 -120
  43. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.gitattributes +0 -0
  44. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  45. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  46. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  47. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/dependabot.yml +0 -0
  48. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/scripts/sync-release-version.mjs +0 -0
  49. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/auto-merge.yml +0 -0
  50. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/ci.yml +0 -0
  51. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/release.yml +0 -0
  52. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/scheduled-release.yml +0 -0
  53. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.github/workflows/supply-chain.yml +0 -0
  54. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.gitignore +0 -0
  55. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.npmignore +0 -0
  56. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.npmrc +0 -0
  57. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/.pre-commit-hooks.yaml +0 -0
  58. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/AGENTS.md +0 -0
  59. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CLAUDE.md +0 -0
  60. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CODE_OF_CONDUCT.md +0 -0
  61. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/CONTRIBUTING.md +0 -0
  62. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/LICENSE +0 -0
  63. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/PRIVACY.md +0 -0
  64. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/SECURITY.md +0 -0
  65. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/SUPPORT.md +0 -0
  66. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/bench/README.md +0 -0
  67. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/cli/testguard.mjs +0 -0
  68. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/.gitignore +0 -0
  69. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/package.json +0 -0
  70. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/src/export.mjs +0 -0
  71. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/src/redact.mjs +0 -0
  72. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/test/flaky.test.mjs +0 -0
  73. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/test/redact.test.mjs +0 -0
  74. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/fixtures/known-answer/vitest.config.mjs +0 -0
  75. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/README.md +0 -0
  76. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/brief.json +0 -0
  77. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/calibration.json +0 -0
  78. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/claims.json +0 -0
  79. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/examples/ignore.json +0 -0
  80. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
  81. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/baseline.zero-count.json +0 -0
  82. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
  83. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
  84. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
  85. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
  86. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.absolute-path.json +0 -0
  87. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
  88. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
  89. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.noop-fault.json +0 -0
  90. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
  91. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
  92. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
  93. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
  94. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
  95. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
  96. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
  97. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
  98. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
  99. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
  100. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
  101. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/invalid/ignore.short-reason.json +0 -0
  102. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/conformance/schemas.test.mjs +0 -0
  103. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/lib/fingerprint.mjs +0 -0
  104. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/brief.schema.json +0 -0
  105. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/calibration.schema.json +0 -0
  106. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/common.schema.json +0 -0
  107. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/spec/schemas/ignore.schema.json +0 -0
  108. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/claims/annotations.mjs +0 -0
  109. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/claims/load.mjs +0 -0
  110. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/commands/brief.mjs +0 -0
  111. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/evidence/writer.mjs +0 -0
  112. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/inject.mjs +0 -0
  113. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/runner-vitest.mjs +0 -0
  114. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/probe/worktree.mjs +0 -0
  115. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/util/glob.mjs +0 -0
  116. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/src/util/hash.mjs +0 -0
  117. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/annotations.test.mjs +0 -0
  118. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/baseline.test.mjs +0 -0
  119. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/glob.test.mjs +0 -0
  120. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/inject.test.mjs +0 -0
  121. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/rank-aliases.test.mjs +0 -0
  122. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/runner-command.test.mjs +0 -0
  123. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/runner-vitest.test.mjs +0 -0
  124. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/worktree.test.mjs +0 -0
  125. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/test/writer.test.mjs +0 -0
  126. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/testguard_cli/__init__.py +0 -0
  127. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/testguard_cli/wrapper.py +0 -0
  128. {testguard_cli-0.1.2 → testguard_cli-0.1.3}/vitest.config.mjs +0 -0
@@ -31,7 +31,7 @@ try {
31
31
  const version = run(bin, ['--version'], consumer).trim();
32
32
  const claims = run(bin, ['claims', '.'], consumer);
33
33
  if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
34
- if (!claims.includes('8 claims')) throw new Error(`claims did not list the fixture:\n${claims}`);
34
+ if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
35
35
  const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
36
36
  console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
37
37
  } finally {
@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.1.3] - 2026-09-17
11
+
12
+ From a second field report on a real codebase (456 tests, 27 claims, 35
13
+ faults; 13 survived on the first run, two critical claims with zero coverage).
14
+
15
+ ### Fixed
16
+
17
+ - **Worktree mode probed HEAD while reading the claims file from the working
18
+ tree**, so uncommitted defender changes were silently ignored — the same
19
+ survivors came back with no hint why. `probe` now refuses (exit 2) when any
20
+ resolved defender or fault target has uncommitted changes, naming the files
21
+ and the commit it would have probed. Every summary names the commit probed.
22
+ - `killed-by-undeclared-tests` never said which tests killed the fault; the
23
+ author could not fix `defendedBy` without grepping the suite. Evidence now
24
+ carries `detail.undeclaredKillers` and the CLI names the files.
25
+ - `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
26
+ expected }` is recorded and printed.
27
+ - A replacement with an unbalanced paren was `suite-failed-to-load`, not
28
+ `replacement-does-not-compile`: esbuild/vitest wording is now matched.
29
+ - The claims schema promised defender discovery for an absent `defendedBy`;
30
+ the tool answered `nocover`. Discovery is implemented: the test files that
31
+ import the fault's target (relative or alias), recorded as
32
+ `defenders.discovered`. `nocover` now means exactly "no test file imports
33
+ this source".
34
+ - `baseline.json` records `dirty`, as evidence already did.
35
+
36
+ ### Added
37
+
38
+ - `--include-dirty`: snapshot the working tree (tracked edits and untracked,
39
+ non-ignored files) into a throwaway commit and probe that. HEAD, index and
40
+ the user's tree are never touched; `run.repo.snapshot` records the commit.
41
+ - A one-line progress indicator on stderr (TTY only) so a minute of silence
42
+ is not mistaken for a hang.
43
+ - Fixture: a claim with no `defendedBy` whose defender is discovered.
44
+
45
+ ### Changed
46
+
47
+ - **Default output shows only unproven faults plus a killed count.**
48
+ `--verbose` restores the full stream.
49
+ - The `survived` hint reminds the author to check that the fault is
50
+ observable at all before writing a test for it.
51
+ - README: `$schema` path for consumers, `min-release-age` note, the
52
+ worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
53
+
10
54
  ## [0.1.2] - 2026-09-17
11
55
 
12
56
  From a field report on a real codebase (63 test files, 458 tests, 39 faults).
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: testguard-cli
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
5
5
  Project-URL: Homepage, https://github.com/raccioly/testguard
6
6
  Project-URL: Documentation, https://github.com/raccioly/testguard#readme
@@ -76,9 +76,13 @@ tests were written against the survivors, 39/39 were killed.
76
76
  | npm | `npm i -D testguard-cli` then `npx testguard probe` |
77
77
  | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
78
78
  | Homebrew | `brew tap raccioly/tap && brew install testguard` |
79
- | GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
79
+ | GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
80
80
  | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
81
81
 
82
+ Projects that set `min-release-age` in `.npmrc` cannot see a version published
83
+ less than that many days ago (`ENOVERSIONS`); install that one with
84
+ `npm i -D testguard-cli --min-release-age=0`.
85
+
82
86
  ## How it works
83
87
 
84
88
  ```bash
@@ -88,9 +92,11 @@ npx testguard-cli baseline # freeze today's unproven findings; from now on on
88
92
  npx testguard-cli brief # tell the agent where the suite is blind, before it writes
89
93
  ```
90
94
 
91
- 1. **Claims** live in `testguard.claims.json`: a statement, where it comes
92
- from, which tests supposedly defend it, and one or more *faults* — each a
93
- deterministic source change that would make the statement false. Every
95
+ 1. **Claims** live in `testguard.claims.json` (editors validate it against
96
+ `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
97
+ a statement, where it comes from, which tests supposedly defend it, and
98
+ one or more *faults* — each a deterministic source change that would make
99
+ the statement false. Every
94
100
  claim and every fault records who produced it. `testguard claims`
95
101
  validates the file and reports drift against `@claim <ID>` annotations in
96
102
  source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
@@ -115,10 +121,23 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
115
121
  written to `.testguard/evidence.json` — validated against the spec before
116
122
  it is written.
117
123
 
124
+ Worktree mode probes a **commit**. If a defender or target file has
125
+ uncommitted changes, `probe` refuses and says so — otherwise your new
126
+ tests would be silently absent and the same survivors would come back
127
+ with no hint why. `--include-dirty` snapshots the working tree (tracked
128
+ edits and new files) into a throwaway commit and probes that; your tree,
129
+ HEAD and index are never touched. Every summary names the commit probed.
130
+
131
+ A claim with no `defendedBy` has its defenders **discovered**: the test
132
+ files that import the fault's target, by relative path or resolved alias.
133
+ `NOCOVER` then means exactly "no test file imports this source".
134
+
118
135
  Practical loop: first pass `--no-escalate` (escalation re-runs the whole
119
- suite N times per survivor); iterate on one claim with
120
- `--claim <ID> --in-place` only the files faults are applied to must be
121
- committed, test files may be dirty; final pass with defaults. A custom
136
+ suite N times per survivor); iterate on one claim with `--claim <ID>` and
137
+ either `--include-dirty` or `--in-place` (only fault target files must be
138
+ clean there; test files may be dirty); final pass with defaults. By default
139
+ the stream shows only unproven faults plus a killed count — `--verbose`
140
+ shows every fault. A custom
122
141
  runner (`pnpm --filter`, a specific config) goes in
123
142
  `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
124
143
  see your `node_modules`, pass `--node-modules <dir>`.
@@ -127,7 +146,8 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
127
146
  source and defenders are unchanged reuse their prior verdict, so a probe
128
147
  in CI costs only what changed.
129
148
  4. **Brief** turns evidence plus baseline into a ranked, capped
130
- `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
149
+ `## TEST BLINDSPOT CONTEXT` block, printed and also written to
150
+ `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
131
151
  — for Claude Code, in `.claude/settings.json`:
132
152
 
133
153
  ```json
@@ -52,9 +52,13 @@ tests were written against the survivors, 39/39 were killed.
52
52
  | npm | `npm i -D testguard-cli` then `npx testguard probe` |
53
53
  | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
54
54
  | Homebrew | `brew tap raccioly/tap && brew install testguard` |
55
- | GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
55
+ | GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
56
56
  | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
57
57
 
58
+ Projects that set `min-release-age` in `.npmrc` cannot see a version published
59
+ less than that many days ago (`ENOVERSIONS`); install that one with
60
+ `npm i -D testguard-cli --min-release-age=0`.
61
+
58
62
  ## How it works
59
63
 
60
64
  ```bash
@@ -64,9 +68,11 @@ npx testguard-cli baseline # freeze today's unproven findings; from now on on
64
68
  npx testguard-cli brief # tell the agent where the suite is blind, before it writes
65
69
  ```
66
70
 
67
- 1. **Claims** live in `testguard.claims.json`: a statement, where it comes
68
- from, which tests supposedly defend it, and one or more *faults* — each a
69
- deterministic source change that would make the statement false. Every
71
+ 1. **Claims** live in `testguard.claims.json` (editors validate it against
72
+ `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
73
+ a statement, where it comes from, which tests supposedly defend it, and
74
+ one or more *faults* — each a deterministic source change that would make
75
+ the statement false. Every
70
76
  claim and every fault records who produced it. `testguard claims`
71
77
  validates the file and reports drift against `@claim <ID>` annotations in
72
78
  source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
@@ -91,10 +97,23 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
91
97
  written to `.testguard/evidence.json` — validated against the spec before
92
98
  it is written.
93
99
 
100
+ Worktree mode probes a **commit**. If a defender or target file has
101
+ uncommitted changes, `probe` refuses and says so — otherwise your new
102
+ tests would be silently absent and the same survivors would come back
103
+ with no hint why. `--include-dirty` snapshots the working tree (tracked
104
+ edits and new files) into a throwaway commit and probes that; your tree,
105
+ HEAD and index are never touched. Every summary names the commit probed.
106
+
107
+ A claim with no `defendedBy` has its defenders **discovered**: the test
108
+ files that import the fault's target, by relative path or resolved alias.
109
+ `NOCOVER` then means exactly "no test file imports this source".
110
+
94
111
  Practical loop: first pass `--no-escalate` (escalation re-runs the whole
95
- suite N times per survivor); iterate on one claim with
96
- `--claim <ID> --in-place` only the files faults are applied to must be
97
- committed, test files may be dirty; final pass with defaults. A custom
112
+ suite N times per survivor); iterate on one claim with `--claim <ID>` and
113
+ either `--include-dirty` or `--in-place` (only fault target files must be
114
+ clean there; test files may be dirty); final pass with defaults. By default
115
+ the stream shows only unproven faults plus a killed count — `--verbose`
116
+ shows every fault. A custom
98
117
  runner (`pnpm --filter`, a specific config) goes in
99
118
  `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
100
119
  see your `node_modules`, pass `--node-modules <dir>`.
@@ -103,7 +122,8 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
103
122
  source and defenders are unchanged reuse their prior verdict, so a probe
104
123
  in CI costs only what changed.
105
124
  4. **Brief** turns evidence plus baseline into a ranked, capped
106
- `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
125
+ `## TEST BLINDSPOT CONTEXT` block, printed and also written to
126
+ `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
107
127
  — for Claude Code, in `.claude/settings.json`:
108
128
 
109
129
  ```json
@@ -38,7 +38,7 @@ inputs:
38
38
  version:
39
39
  description: 'testguard-cli version to run'
40
40
  required: false
41
- default: '0.1.2'
41
+ default: '0.1.3'
42
42
  node-version:
43
43
  description: 'Node.js version (20+)'
44
44
  required: false
@@ -24,6 +24,7 @@ evidence comes from running against a real codebase with real history.
24
24
  | REDACT-006 | `mask()` replaces with equal-length asterisks | F1 syntax error | fault-invalid |
25
25
  | EXPORT-001 | Exported rows never include `content` | F1 keeps content | nocover |
26
26
  | FLAKY-001 | (defender is flaky) | F1 anything | flaky-defender |
27
+ | DISCOVER-001 | Every match is replaced (no `defendedBy`) | F1 loop body removed | killed — defenders **discovered** by import |
27
28
 
28
29
  ## The exhibit — REDACT-001/F1
29
30
 
@@ -0,0 +1,53 @@
1
+ {
2
+ "confirmRuns": 3,
3
+ "expected": {
4
+ "REDACT-001/F1": {
5
+ "verdict": "survived",
6
+ "why": "The audit-row test asserts with objectContaining and omits `content`; a row carrying the raw input passes."
7
+ },
8
+ "REDACT-001/F2": {
9
+ "verdict": "killed",
10
+ "why": "The masked-output test asserts the secret is absent."
11
+ },
12
+ "REDACT-002/F1": {
13
+ "verdict": "killed",
14
+ "why": "The invalid-rule test asserts compileRules does not throw."
15
+ },
16
+ "REDACT-003/F1": {
17
+ "verdict": "survived",
18
+ "why": "No test exercises a missing scope; the fail-closed guard is undefended."
19
+ },
20
+ "REDACT-004/F1": {
21
+ "verdict": "timeout",
22
+ "why": "Tests awaiting redact() hit the 1s test timeout; a timeout is not an assertion failure and is not a kill."
23
+ },
24
+ "REDACT-005/F1": {
25
+ "verdict": "unverifiable",
26
+ "reason": "anchor-missing",
27
+ "why": "The find string is not present in the source."
28
+ },
29
+ "REDACT-005/F2": {
30
+ "verdict": "unverifiable",
31
+ "reason": "anchor-ambiguous",
32
+ "why": "`return null;` occurs twice; expectHits is 1."
33
+ },
34
+ "REDACT-006/F1": {
35
+ "verdict": "fault-invalid",
36
+ "reason": "replacement-does-not-compile",
37
+ "why": "The replacement introduces a syntax error; the suite cannot load."
38
+ },
39
+ "EXPORT-001/F1": {
40
+ "verdict": "nocover",
41
+ "why": "test/export.test.mjs does not exist; no defender resolves."
42
+ },
43
+ "FLAKY-001/F1": {
44
+ "verdict": "flaky-defender",
45
+ "reason": "defenders-not-green",
46
+ "why": "test/flaky.test.mjs fails every other run; the baseline is not green N/N."
47
+ },
48
+ "DISCOVER-001/F1": {
49
+ "verdict": "killed",
50
+ "why": "No defendedBy; test/redact.test.mjs is discovered because it imports src/redact.mjs, and its masked-output test fails."
51
+ }
52
+ }
53
+ }
@@ -4,10 +4,19 @@
4
4
  {
5
5
  "id": "REDACT-001",
6
6
  "statement": "The audit row written for a masked message never contains the original input.",
7
- "source": { "kind": "spec", "ref": "README.md#claims" },
7
+ "source": {
8
+ "kind": "spec",
9
+ "ref": "README.md#claims"
10
+ },
8
11
  "severity": "critical",
9
- "producedBy": { "producer": "human", "by": "fixture author", "at": "2026-09-17T00:00:00Z" },
10
- "defendedBy": ["test/redact.test.mjs"],
12
+ "producedBy": {
13
+ "producer": "human",
14
+ "by": "fixture author",
15
+ "at": "2026-09-17T00:00:00Z"
16
+ },
17
+ "defendedBy": [
18
+ "test/redact.test.mjs"
19
+ ],
11
20
  "faults": [
12
21
  {
13
22
  "id": "F1",
@@ -16,7 +25,10 @@
16
25
  "file": "src/redact.mjs",
17
26
  "find": "content: redacted,",
18
27
  "replace": "content: input,",
19
- "producedBy": { "producer": "human", "by": "fixture author" }
28
+ "producedBy": {
29
+ "producer": "human",
30
+ "by": "fixture author"
31
+ }
20
32
  },
21
33
  {
22
34
  "id": "F2",
@@ -25,17 +37,28 @@
25
37
  "file": "src/redact.mjs",
26
38
  "find": "const redacted = mask(input, rules);",
27
39
  "replace": "const redacted = input;",
28
- "producedBy": { "producer": "human", "by": "fixture author" }
40
+ "producedBy": {
41
+ "producer": "human",
42
+ "by": "fixture author"
43
+ }
29
44
  }
30
45
  ]
31
46
  },
32
47
  {
33
48
  "id": "REDACT-002",
34
49
  "statement": "A rule whose pattern is invalid is skipped; it never aborts the scan.",
35
- "source": { "kind": "spec", "ref": "README.md#claims" },
50
+ "source": {
51
+ "kind": "spec",
52
+ "ref": "README.md#claims"
53
+ },
36
54
  "severity": "high",
37
- "producedBy": { "producer": "human", "by": "fixture author" },
38
- "defendedBy": ["test/redact.test.mjs"],
55
+ "producedBy": {
56
+ "producer": "human",
57
+ "by": "fixture author"
58
+ },
59
+ "defendedBy": [
60
+ "test/redact.test.mjs"
61
+ ],
39
62
  "faults": [
40
63
  {
41
64
  "id": "F1",
@@ -44,17 +67,28 @@
44
67
  "file": "src/redact.mjs",
45
68
  "find": " } catch {\n return null;\n }",
46
69
  "replace": " } catch (e) {\n throw e;\n }",
47
- "producedBy": { "producer": "human", "by": "fixture author" }
70
+ "producedBy": {
71
+ "producer": "human",
72
+ "by": "fixture author"
73
+ }
48
74
  }
49
75
  ]
50
76
  },
51
77
  {
52
78
  "id": "REDACT-003",
53
79
  "statement": "A missing scope is an error; the engine never performs an unscoped scan.",
54
- "source": { "kind": "annotation", "ref": "src/redact.mjs @claim REDACT-003" },
80
+ "source": {
81
+ "kind": "annotation",
82
+ "ref": "src/redact.mjs @claim REDACT-003"
83
+ },
55
84
  "severity": "high",
56
- "producedBy": { "producer": "human", "by": "fixture author" },
57
- "defendedBy": ["test/redact.test.mjs"],
85
+ "producedBy": {
86
+ "producer": "human",
87
+ "by": "fixture author"
88
+ },
89
+ "defendedBy": [
90
+ "test/redact.test.mjs"
91
+ ],
58
92
  "faults": [
59
93
  {
60
94
  "id": "F1",
@@ -63,17 +97,28 @@
63
97
  "file": "src/redact.mjs",
64
98
  "find": "if (!ctx || !ctx.scope) {",
65
99
  "replace": "if (false) {",
66
- "producedBy": { "producer": "human", "by": "fixture author" }
100
+ "producedBy": {
101
+ "producer": "human",
102
+ "by": "fixture author"
103
+ }
67
104
  }
68
105
  ]
69
106
  },
70
107
  {
71
108
  "id": "REDACT-004",
72
109
  "statement": "redact() settles for every input.",
73
- "source": { "kind": "spec", "ref": "README.md#claims" },
110
+ "source": {
111
+ "kind": "spec",
112
+ "ref": "README.md#claims"
113
+ },
74
114
  "severity": "medium",
75
- "producedBy": { "producer": "human", "by": "fixture author" },
76
- "defendedBy": ["test/redact.test.mjs"],
115
+ "producedBy": {
116
+ "producer": "human",
117
+ "by": "fixture author"
118
+ },
119
+ "defendedBy": [
120
+ "test/redact.test.mjs"
121
+ ],
77
122
  "faults": [
78
123
  {
79
124
  "id": "F1",
@@ -82,17 +127,28 @@
82
127
  "file": "src/redact.mjs",
83
128
  "find": " return redacted;\n}",
84
129
  "replace": " return new Promise(() => {});\n}",
85
- "producedBy": { "producer": "human", "by": "fixture author" }
130
+ "producedBy": {
131
+ "producer": "human",
132
+ "by": "fixture author"
133
+ }
86
134
  }
87
135
  ]
88
136
  },
89
137
  {
90
138
  "id": "REDACT-005",
91
139
  "statement": "findRule() returns null for an unknown id.",
92
- "source": { "kind": "spec", "ref": "README.md#claims" },
140
+ "source": {
141
+ "kind": "spec",
142
+ "ref": "README.md#claims"
143
+ },
93
144
  "severity": "low",
94
- "producedBy": { "producer": "human", "by": "fixture author" },
95
- "defendedBy": ["test/redact.test.mjs"],
145
+ "producedBy": {
146
+ "producer": "human",
147
+ "by": "fixture author"
148
+ },
149
+ "defendedBy": [
150
+ "test/redact.test.mjs"
151
+ ],
96
152
  "faults": [
97
153
  {
98
154
  "id": "F1",
@@ -101,7 +157,10 @@
101
157
  "file": "src/redact.mjs",
102
158
  "find": "return undefined; // unknown rule",
103
159
  "replace": "return rules[0];",
104
- "producedBy": { "producer": "human", "by": "fixture author" }
160
+ "producedBy": {
161
+ "producer": "human",
162
+ "by": "fixture author"
163
+ }
105
164
  },
106
165
  {
107
166
  "id": "F2",
@@ -111,17 +170,28 @@
111
170
  "find": "return null;",
112
171
  "replace": "return rules[0];",
113
172
  "expectHits": 1,
114
- "producedBy": { "producer": "human", "by": "fixture author" }
173
+ "producedBy": {
174
+ "producer": "human",
175
+ "by": "fixture author"
176
+ }
115
177
  }
116
178
  ]
117
179
  },
118
180
  {
119
181
  "id": "REDACT-006",
120
182
  "statement": "mask() replaces each match with asterisks of equal length.",
121
- "source": { "kind": "spec", "ref": "README.md#claims" },
183
+ "source": {
184
+ "kind": "spec",
185
+ "ref": "README.md#claims"
186
+ },
122
187
  "severity": "low",
123
- "producedBy": { "producer": "human", "by": "fixture author" },
124
- "defendedBy": ["test/redact.test.mjs"],
188
+ "producedBy": {
189
+ "producer": "human",
190
+ "by": "fixture author"
191
+ },
192
+ "defendedBy": [
193
+ "test/redact.test.mjs"
194
+ ],
125
195
  "faults": [
126
196
  {
127
197
  "id": "F1",
@@ -130,17 +200,28 @@
130
200
  "file": "src/redact.mjs",
131
201
  "find": "(m) => '*'.repeat(m.length)",
132
202
  "replace": "(m) => '*'.repeat(m.length",
133
- "producedBy": { "producer": "human", "by": "fixture author" }
203
+ "producedBy": {
204
+ "producer": "human",
205
+ "by": "fixture author"
206
+ }
134
207
  }
135
208
  ]
136
209
  },
137
210
  {
138
211
  "id": "EXPORT-001",
139
212
  "statement": "Exported rows never include the content field.",
140
- "source": { "kind": "spec", "ref": "README.md#claims" },
213
+ "source": {
214
+ "kind": "spec",
215
+ "ref": "README.md#claims"
216
+ },
141
217
  "severity": "medium",
142
- "producedBy": { "producer": "human", "by": "fixture author" },
143
- "defendedBy": ["test/export.test.mjs"],
218
+ "producedBy": {
219
+ "producer": "human",
220
+ "by": "fixture author"
221
+ },
222
+ "defendedBy": [
223
+ "test/export.test.mjs"
224
+ ],
144
225
  "faults": [
145
226
  {
146
227
  "id": "F1",
@@ -149,17 +230,27 @@
149
230
  "file": "src/export.mjs",
150
231
  "find": "return rows.map(({ content, ...rest }) => rest);",
151
232
  "replace": "return rows.map((row) => ({ ...row }));",
152
- "producedBy": { "producer": "human", "by": "fixture author" }
233
+ "producedBy": {
234
+ "producer": "human",
235
+ "by": "fixture author"
236
+ }
153
237
  }
154
238
  ]
155
239
  },
156
240
  {
157
241
  "id": "FLAKY-001",
158
242
  "statement": "A claim whose defender is not reliably green cannot be verified.",
159
- "source": { "kind": "manual" },
243
+ "source": {
244
+ "kind": "manual"
245
+ },
160
246
  "severity": "low",
161
- "producedBy": { "producer": "human", "by": "fixture author" },
162
- "defendedBy": ["test/flaky.test.mjs"],
247
+ "producedBy": {
248
+ "producer": "human",
249
+ "by": "fixture author"
250
+ },
251
+ "defendedBy": [
252
+ "test/flaky.test.mjs"
253
+ ],
163
254
  "faults": [
164
255
  {
165
256
  "id": "F1",
@@ -168,7 +259,37 @@
168
259
  "file": "src/redact.mjs",
169
260
  "find": "action: 'MASK',",
170
261
  "replace": "action: 'mask',",
171
- "producedBy": { "producer": "human", "by": "fixture author" }
262
+ "producedBy": {
263
+ "producer": "human",
264
+ "by": "fixture author"
265
+ }
266
+ }
267
+ ]
268
+ },
269
+ {
270
+ "id": "DISCOVER-001",
271
+ "statement": "Every rule match is replaced in the returned text.",
272
+ "source": {
273
+ "kind": "spec",
274
+ "ref": "README.md#claims"
275
+ },
276
+ "severity": "high",
277
+ "producedBy": {
278
+ "producer": "human",
279
+ "by": "fixture author"
280
+ },
281
+ "faults": [
282
+ {
283
+ "id": "F1",
284
+ "description": "The replacement loop body is removed; nothing is masked. No defendedBy is declared: defenders must be discovered as the tests that import src/redact.mjs.",
285
+ "faultClass": "statement-deleted",
286
+ "file": "src/redact.mjs",
287
+ "find": "out = out.replace(rule.re, (m) => '*'.repeat(m.length));",
288
+ "replace": "/* fault: replacement removed */",
289
+ "producedBy": {
290
+ "producer": "human",
291
+ "by": "fixture author"
292
+ }
172
293
  }
173
294
  ]
174
295
  }
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "testguard-cli",
3
- "version": "0.1.2",
3
+ "version": "0.1.3",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "testguard-cli",
9
- "version": "0.1.2",
9
+ "version": "0.1.3",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "ajv": "8.20.0"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testguard-cli",
3
- "version": "0.1.2",
3
+ "version": "0.1.3",
4
4
  "description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -13,7 +13,7 @@
13
13
  class Testguard < Formula
14
14
  desc "Proves a test suite defends the claims a project makes"
15
15
  homepage "https://github.com/raccioly/testguard"
16
- url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.2.tgz"
16
+ url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.3.tgz"
17
17
  sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
18
18
  license "MIT"
19
19
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "testguard-cli"
7
- version = "0.1.2"
7
+ version = "0.1.3"
8
8
  description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -35,10 +35,16 @@ Rules that follow from the table:
35
35
  reporting `survived` would be wrong the other way. It is `flaky-defender`.
36
36
  5. **Escalation never upgrades a verdict.** A fault that survives its declared
37
37
  defenders may be re-run against the whole suite. If the wider suite kills
38
- it, the verdict stays `survived` with reason `killed-by-undeclared-tests`:
39
- the claim's stated evidence chain is broken even though the suite is not
40
- blind. It gates, and ranks below a true survivor.
41
- 6. **Never a single global score.** Output is per claim, ranked. Blindness is
38
+ it, the verdict stays `survived` with reason `killed-by-undeclared-tests`
39
+ and `detail.undeclaredKillers` names the tests, so the author can fix
40
+ `defendedBy`. The claim's stated evidence chain is broken even though the
41
+ suite is not blind. It gates, and ranks below a true survivor.
42
+ 6. **A verdict names the commit it is about.** Evidence records `repo.head`;
43
+ when the working tree was probed instead, `repo.snapshot` holds the
44
+ throwaway commit that captured it. A tool must refuse to probe a commit
45
+ while defenders or targets have uncommitted changes, unless told to
46
+ snapshot the working tree — otherwise the answer looks right and is not.
47
+ 7. **Never a single global score.** Output is per claim, ranked. Blindness is
42
48
  concentrated, and one number hides where.
43
49
 
44
50
  ## Baseline and delta