testguard-cli 0.1.3__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. testguard_cli-0.2.1/.codex/hooks.json +71 -0
  2. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/scripts/install-smoke.mjs +2 -0
  3. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/AGENTS.md +2 -0
  4. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/CHANGELOG.md +32 -0
  5. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/PKG-INFO +39 -6
  6. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/README.md +38 -5
  7. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/action.yml +1 -1
  8. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/package-lock.json +2 -2
  9. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/package.json +1 -1
  10. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/packaging/homebrew/testguard.rb +1 -1
  11. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/pyproject.toml +1 -1
  12. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/GATE-SEMANTICS.md +5 -1
  13. testguard_cli-0.2.1/spec/conformance/examples/evidence-provisional.json +272 -0
  14. testguard_cli-0.2.1/spec/conformance/invalid/evidence.provisional-flag-on-confirmed-run.json +535 -0
  15. testguard_cli-0.2.1/spec/conformance/invalid/evidence.single-run-without-provisional.json +271 -0
  16. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/schemas.test.mjs +6 -4
  17. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/lib/validate.mjs +2 -0
  18. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/evidence.schema.json +4 -0
  19. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/brief/brief.mjs +2 -1
  20. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/cli.mjs +11 -3
  21. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/commands/baseline.mjs +5 -1
  22. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/commands/probe.mjs +10 -5
  23. testguard_cli-0.2.1/src/commands/scaffold.mjs +34 -0
  24. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/probe.mjs +1 -0
  25. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/render.mjs +7 -5
  26. testguard_cli-0.2.1/src/scaffold/producers.mjs +85 -0
  27. testguard_cli-0.2.1/src/scaffold/scaffold.mjs +118 -0
  28. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/brief.test.mjs +15 -0
  29. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/probe.fixture.test.mjs +21 -0
  30. testguard_cli-0.2.1/test/scaffold.test.mjs +133 -0
  31. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/testguard.claims.json +31 -0
  32. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.gitattributes +0 -0
  33. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  34. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  35. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  36. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/dependabot.yml +0 -0
  37. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/scripts/sync-release-version.mjs +0 -0
  38. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/workflows/auto-merge.yml +0 -0
  39. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/workflows/ci.yml +0 -0
  40. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/workflows/release.yml +0 -0
  41. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/workflows/scheduled-release.yml +0 -0
  42. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.github/workflows/supply-chain.yml +0 -0
  43. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.gitignore +0 -0
  44. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.npmignore +0 -0
  45. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.npmrc +0 -0
  46. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/.pre-commit-hooks.yaml +0 -0
  47. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/CLAUDE.md +0 -0
  48. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/CODE_OF_CONDUCT.md +0 -0
  49. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/CONTRIBUTING.md +0 -0
  50. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/LICENSE +0 -0
  51. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/PRIVACY.md +0 -0
  52. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/SECURITY.md +0 -0
  53. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/SUPPORT.md +0 -0
  54. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/bench/README.md +0 -0
  55. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/cli/testguard.mjs +0 -0
  56. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/.gitignore +0 -0
  57. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/README.md +0 -0
  58. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/expected.json +0 -0
  59. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/package.json +0 -0
  60. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/src/export.mjs +0 -0
  61. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/src/redact.mjs +0 -0
  62. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/test/flaky.test.mjs +0 -0
  63. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/test/redact.test.mjs +0 -0
  64. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/testguard.claims.json +0 -0
  65. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/fixtures/known-answer/vitest.config.mjs +0 -0
  66. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/README.md +0 -0
  67. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/baseline.json +0 -0
  68. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/brief.json +0 -0
  69. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/calibration.json +0 -0
  70. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/claims.json +0 -0
  71. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/evidence.json +0 -0
  72. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/examples/ignore.json +0 -0
  73. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
  74. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/baseline.zero-count.json +0 -0
  75. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
  76. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
  77. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
  78. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
  79. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.absolute-path.json +0 -0
  80. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
  81. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
  82. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.noop-fault.json +0 -0
  83. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
  84. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
  85. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
  86. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
  87. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
  88. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
  89. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
  90. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
  91. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
  92. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
  93. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
  94. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
  95. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/conformance/invalid/ignore.short-reason.json +0 -0
  96. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/lib/fingerprint.mjs +0 -0
  97. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/baseline.schema.json +0 -0
  98. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/brief.schema.json +0 -0
  99. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/calibration.schema.json +0 -0
  100. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/claims.schema.json +0 -0
  101. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/common.schema.json +0 -0
  102. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/spec/schemas/ignore.schema.json +0 -0
  103. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/baseline/baseline.mjs +0 -0
  104. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/claims/annotations.mjs +0 -0
  105. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/claims/load.mjs +0 -0
  106. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/commands/brief.mjs +0 -0
  107. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/commands/claims.mjs +0 -0
  108. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/evidence/writer.mjs +0 -0
  109. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/git.mjs +0 -0
  110. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/classify.mjs +0 -0
  111. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/discover.mjs +0 -0
  112. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/inject.mjs +0 -0
  113. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/rank.mjs +0 -0
  114. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/runner-vitest.mjs +0 -0
  115. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/probe/worktree.mjs +0 -0
  116. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/util/glob.mjs +0 -0
  117. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/src/util/hash.mjs +0 -0
  118. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/annotations.test.mjs +0 -0
  119. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/baseline.test.mjs +0 -0
  120. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/classify.test.mjs +0 -0
  121. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/discover.test.mjs +0 -0
  122. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/glob.test.mjs +0 -0
  123. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/inject.test.mjs +0 -0
  124. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/probe-preconditions.test.mjs +0 -0
  125. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/rank-aliases.test.mjs +0 -0
  126. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/runner-command.test.mjs +0 -0
  127. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/runner-vitest.test.mjs +0 -0
  128. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/snapshot.test.mjs +0 -0
  129. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/worktree.test.mjs +0 -0
  130. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/test/writer.test.mjs +0 -0
  131. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/testguard_cli/__init__.py +0 -0
  132. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/testguard_cli/wrapper.py +0 -0
  133. {testguard_cli-0.1.3 → testguard_cli-0.2.1}/vitest.config.mjs +0 -0
@@ -0,0 +1,71 @@
1
+ {
2
+ "hooks": {
3
+ "PreToolUse": [
4
+ {
5
+ "matcher": "Read",
6
+ "hooks": [
7
+ {
8
+ "type": "command",
9
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/pre-read.js\"",
10
+ "timeout": 5
11
+ }
12
+ ]
13
+ },
14
+ {
15
+ "matcher": "Write|Edit|MultiEdit",
16
+ "hooks": [
17
+ {
18
+ "type": "command",
19
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/pre-write.js\"",
20
+ "timeout": 5
21
+ }
22
+ ]
23
+ }
24
+ ],
25
+ "PostToolUse": [
26
+ {
27
+ "matcher": "Read",
28
+ "hooks": [
29
+ {
30
+ "type": "command",
31
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-read.js\"",
32
+ "timeout": 5
33
+ }
34
+ ]
35
+ },
36
+ {
37
+ "matcher": "Write|Edit|MultiEdit",
38
+ "hooks": [
39
+ {
40
+ "type": "command",
41
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-write.js\"",
42
+ "timeout": 10
43
+ }
44
+ ]
45
+ }
46
+ ],
47
+ "SessionStart": [
48
+ {
49
+ "matcher": "",
50
+ "hooks": [
51
+ {
52
+ "type": "command",
53
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/session-start.js\"",
54
+ "timeout": 5
55
+ }
56
+ ]
57
+ }
58
+ ],
59
+ "Stop": [
60
+ {
61
+ "hooks": [
62
+ {
63
+ "type": "command",
64
+ "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/stop.js\"",
65
+ "timeout": 10
66
+ }
67
+ ]
68
+ }
69
+ ]
70
+ }
71
+ }
@@ -30,6 +30,8 @@ try {
30
30
  const bin = join(consumer, 'node_modules', '.bin', process.platform === 'win32' ? 'testguard.cmd' : 'testguard');
31
31
  const version = run(bin, ['--version'], consumer).trim();
32
32
  const claims = run(bin, ['claims', '.'], consumer);
33
+ const draft = JSON.parse(run(bin, ['scaffold', 'src/redact.mjs', '--json'], consumer));
34
+ if (!draft.claims?.length) throw new Error('scaffold produced no claims from the installed tarball');
33
35
  if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
34
36
  if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
35
37
  const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
@@ -19,6 +19,7 @@ npm run test:spec # conformance suite only
19
19
  npm run self:probe # TestGuard probes its own claims; must exit 0
20
20
  npm run test:install # the packed tarball must run with production deps only
21
21
  node cli/testguard.mjs claims fixtures/known-answer
22
+ node cli/testguard.mjs scaffold src/probe/classify.mjs --json # what the producers propose for a file
22
23
  ```
23
24
 
24
25
  ## Key files
@@ -29,6 +30,7 @@ node cli/testguard.mjs claims fixtures/known-answer
29
30
  - `src/probe/probe.mjs` — orchestrator: isolation → baseline (cached per defender set) → apply → probe → escalate → restore → classify → rank.
30
31
  - `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
31
32
  - `testguard.claims.json` — claims about this codebase, probed in CI.
33
+ - `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
32
34
 
33
35
  ## Rules
34
36
 
@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.2.1] - 2026-09-17
11
+
12
+ ### Added
13
+
14
+ - **Provisional verdicts** (#5). A run with `--confirm` below 3 declares
15
+ `run.provisional: true` — a spec field whose equivalence with
16
+ `confirmRuns < 3` the validator enforces both ways. Provisional verdicts
17
+ print with a `?`, the summary is prefixed `PROVISIONAL`, a warning is
18
+ printed on stderr, evidence is written to `.testguard/evidence-provisional.json`
19
+ so the canonical file only ever holds confirmed runs, `brief` warns at the
20
+ top, and `baseline` refuses provisional evidence unless
21
+ `--allow-provisional`. A confirmed prior verdict is never reused by a
22
+ provisional run, nor the reverse.
23
+
24
+ ## [0.2.0] - 2026-09-17
25
+
26
+ ### Added
27
+
28
+ - **`testguard scaffold <file>`** — mechanical fault producer (#6). Proposes
29
+ the five shapes both field reports found behind ~80% of hand-written
30
+ faults: guard forced false, single-line guard or state change removed,
31
+ `return <check>` → `return true`, security literal weakened, check call
32
+ removed. Every proposal is an exact-line anchor with `expectHits` and
33
+ `occurrence` computed from the file (verifiable by construction; anything
34
+ `locate()` would reject is never emitted), `producedBy: derived`,
35
+ `defendedBy` prefilled from the tests that import the module, grouped
36
+ under a preceding `@claim` annotation or by enclosing function.
37
+ Statements are `TODO:` placeholders; the output is a draft under
38
+ `.testguard/`, never the claims file. `--claim <ID>` puts everything under
39
+ one claim and copies it if it exists; `--json` prints instead.
40
+ - Self-claim `TG-SCAFFOLD-ANCHORS-HIT`; install smoke exercises `scaffold`.
41
+
10
42
  ## [0.1.3] - 2026-09-17
11
43
 
12
44
  From a second field report on a real codebase (456 tests, 27 claims, 35
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: testguard-cli
3
- Version: 0.1.3
3
+ Version: 0.2.1
4
4
  Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
5
5
  Project-URL: Homepage, https://github.com/raccioly/testguard
6
6
  Project-URL: Documentation, https://github.com/raccioly/testguard#readme
@@ -76,7 +76,7 @@ tests were written against the survivors, 39/39 were killed.
76
76
  | npm | `npm i -D testguard-cli` then `npx testguard probe` |
77
77
  | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
78
78
  | Homebrew | `brew tap raccioly/tap && brew install testguard` |
79
- | GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
79
+ | GitHub Action | `uses: raccioly/testguard@v0.2.1` — see [`action.yml`](./action.yml) |
80
80
  | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
81
81
 
82
82
  Projects that set `min-release-age` in `.npmrc` cannot see a version published
@@ -90,6 +90,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
90
90
  npx testguard-cli probe # try to falsify each claim; report what the tests missed
91
91
  npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
92
92
  npx testguard-cli brief # tell the agent where the suite is blind, before it writes
93
+ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
93
94
  ```
94
95
 
95
96
  1. **Claims** live in `testguard.claims.json` (editors validate it against
@@ -135,7 +136,10 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
135
136
  Practical loop: first pass `--no-escalate` (escalation re-runs the whole
136
137
  suite N times per survivor); iterate on one claim with `--claim <ID>` and
137
138
  either `--include-dirty` or `--in-place` (only fault target files must be
138
- clean there; test files may be dirty); final pass with defaults. By default
139
+ clean there; test files may be dirty), optionally `--confirm 1` for a fast
140
+ **provisional** signal — verdicts print with a `?`, evidence goes to
141
+ `evidence-provisional.json`, and `baseline` refuses it; final pass with
142
+ defaults. By default
139
143
  the stream shows only unproven faults plus a killed count — `--verbose`
140
144
  shows every fault. A custom
141
145
  runner (`pnpm --filter`, a specific config) goes in
@@ -159,6 +163,34 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
159
163
  `--text` prints only, and exits 0 silently when there is no evidence yet,
160
164
  so the hook can never break a session.
161
165
 
166
+ ### Authoring faults mechanically
167
+
168
+ Writing faults by hand means reading the code to find exact anchors. Two
169
+ field reports found that ~80% of hand-written faults are one of five shapes,
170
+ so `scaffold` proposes them for you:
171
+
172
+ ```bash
173
+ npx testguard-cli scaffold src/auth.ts # → .testguard/scaffold-auth.json (a draft, never your claims file)
174
+ npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN # every proposal under one claim; copies it if it exists
175
+ ```
176
+
177
+ | Shape | What it proposes |
178
+ |---|---|
179
+ | `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
180
+ | `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
181
+ | `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
182
+ | `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
183
+ | `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
184
+
185
+ Every proposal's `find` is the exact line with `expectHits`/`occurrence`
186
+ computed from the file, so it is verifiable by construction; provenance is
187
+ `producer: derived`; `defendedBy` is prefilled from the tests that import
188
+ the module; proposals are grouped under a preceding `@claim <ID>` annotation
189
+ or by enclosing function. Statements are `TODO:` placeholders — a proposal
190
+ becomes a claim only when a human states what it defends. Deterministic
191
+ heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
192
+ emitted.
193
+
162
194
  **Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
163
195
  The baseline is the frozen contract; the other two are regenerated per run.
164
196
 
@@ -184,13 +216,14 @@ through every verdict.
184
216
 
185
217
  ## Status
186
218
 
187
- **v0.1.** Four commands, vitest runner, hand-authored faults. The contract
219
+ **v0.2.** Five commands, vitest runner, hand-authored faults plus a
220
+ mechanical scaffold for the five common shapes. The contract
188
221
  spine — six JSON Schemas shared with the other Guard tools — is under
189
222
  [`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
190
223
 
191
224
  Not yet: test generation (the two-gate acceptance loop), other runners,
192
- mechanical fault producers, and calibration of fault classes against real
193
- escaped bugs. Each is designed for; none is claimed.
225
+ AST-aware producers, and calibration of fault classes against real escaped
226
+ bugs. Each is designed for; none is claimed.
194
227
 
195
228
  ## Licence
196
229
 
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
52
52
  | npm | `npm i -D testguard-cli` then `npx testguard probe` |
53
53
  | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
54
54
  | Homebrew | `brew tap raccioly/tap && brew install testguard` |
55
- | GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
55
+ | GitHub Action | `uses: raccioly/testguard@v0.2.1` — see [`action.yml`](./action.yml) |
56
56
  | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
57
57
 
58
58
  Projects that set `min-release-age` in `.npmrc` cannot see a version published
@@ -66,6 +66,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
66
66
  npx testguard-cli probe # try to falsify each claim; report what the tests missed
67
67
  npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
68
68
  npx testguard-cli brief # tell the agent where the suite is blind, before it writes
69
+ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
69
70
  ```
70
71
 
71
72
  1. **Claims** live in `testguard.claims.json` (editors validate it against
@@ -111,7 +112,10 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
111
112
  Practical loop: first pass `--no-escalate` (escalation re-runs the whole
112
113
  suite N times per survivor); iterate on one claim with `--claim <ID>` and
113
114
  either `--include-dirty` or `--in-place` (only fault target files must be
114
- clean there; test files may be dirty); final pass with defaults. By default
115
+ clean there; test files may be dirty), optionally `--confirm 1` for a fast
116
+ **provisional** signal — verdicts print with a `?`, evidence goes to
117
+ `evidence-provisional.json`, and `baseline` refuses it; final pass with
118
+ defaults. By default
115
119
  the stream shows only unproven faults plus a killed count — `--verbose`
116
120
  shows every fault. A custom
117
121
  runner (`pnpm --filter`, a specific config) goes in
@@ -135,6 +139,34 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
135
139
  `--text` prints only, and exits 0 silently when there is no evidence yet,
136
140
  so the hook can never break a session.
137
141
 
142
+ ### Authoring faults mechanically
143
+
144
+ Writing faults by hand means reading the code to find exact anchors. Two
145
+ field reports found that ~80% of hand-written faults are one of five shapes,
146
+ so `scaffold` proposes them for you:
147
+
148
+ ```bash
149
+ npx testguard-cli scaffold src/auth.ts # → .testguard/scaffold-auth.json (a draft, never your claims file)
150
+ npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN # every proposal under one claim; copies it if it exists
151
+ ```
152
+
153
+ | Shape | What it proposes |
154
+ |---|---|
155
+ | `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
156
+ | `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
157
+ | `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
158
+ | `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
159
+ | `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
160
+
161
+ Every proposal's `find` is the exact line with `expectHits`/`occurrence`
162
+ computed from the file, so it is verifiable by construction; provenance is
163
+ `producer: derived`; `defendedBy` is prefilled from the tests that import
164
+ the module; proposals are grouped under a preceding `@claim <ID>` annotation
165
+ or by enclosing function. Statements are `TODO:` placeholders — a proposal
166
+ becomes a claim only when a human states what it defends. Deterministic
167
+ heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
168
+ emitted.
169
+
138
170
  **Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
139
171
  The baseline is the frozen contract; the other two are regenerated per run.
140
172
 
@@ -160,13 +192,14 @@ through every verdict.
160
192
 
161
193
  ## Status
162
194
 
163
- **v0.1.** Four commands, vitest runner, hand-authored faults. The contract
195
+ **v0.2.** Five commands, vitest runner, hand-authored faults plus a
196
+ mechanical scaffold for the five common shapes. The contract
164
197
  spine — six JSON Schemas shared with the other Guard tools — is under
165
198
  [`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
166
199
 
167
200
  Not yet: test generation (the two-gate acceptance loop), other runners,
168
- mechanical fault producers, and calibration of fault classes against real
169
- escaped bugs. Each is designed for; none is claimed.
201
+ AST-aware producers, and calibration of fault classes against real escaped
202
+ bugs. Each is designed for; none is claimed.
170
203
 
171
204
  ## Licence
172
205
 
@@ -38,7 +38,7 @@ inputs:
38
38
  version:
39
39
  description: 'testguard-cli version to run'
40
40
  required: false
41
- default: '0.1.3'
41
+ default: '0.2.1'
42
42
  node-version:
43
43
  description: 'Node.js version (20+)'
44
44
  required: false
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "testguard-cli",
3
- "version": "0.1.3",
3
+ "version": "0.2.1",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "testguard-cli",
9
- "version": "0.1.3",
9
+ "version": "0.2.1",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "ajv": "8.20.0"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testguard-cli",
3
- "version": "0.1.3",
3
+ "version": "0.2.1",
4
4
  "description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -13,7 +13,7 @@
13
13
  class Testguard < Formula
14
14
  desc "Proves a test suite defends the claims a project makes"
15
15
  homepage "https://github.com/raccioly/testguard"
16
- url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.3.tgz"
16
+ url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.2.1.tgz"
17
17
  sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
18
18
  license "MIT"
19
19
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "testguard-cli"
7
- version = "0.1.3"
7
+ version = "0.2.1"
8
8
  description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
9
9
  readme = "README.md"
10
10
  license = { text = "MIT" }
@@ -22,7 +22,11 @@ Rules that follow from the table:
22
22
  1. **Green baseline first.** Defenders run N times unmodified before any fault
23
23
  is applied. Anything short of N/N pass is `flaky-defender` and stops there.
24
24
  2. **Confirm over N runs.** `killed` and `survived` both require exactly N
25
- probe runs, all agreeing. Default N is 3.
25
+ probe runs, all agreeing. Default N is 3. **Fewer than three runs is
26
+ provisional**: the evidence declares `run.provisional: true`, every
27
+ rendering marks the verdicts as unconfirmed, and a provisional run is
28
+ never frozen into a baseline. Provisional runs exist for the fix loop —
29
+ a fast signal while writing a test — not for a gate.
26
30
  3. **Only a test body rejecting the behaviour kills.** A test that fails by
27
31
  assertion — or by an exception the fault provoked inside it — counts. A
28
32
  timeout does not, and a suite that fails to load does not: neither is
@@ -0,0 +1,272 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "tool": {
4
+ "name": "testguard",
5
+ "version": "0.1.0"
6
+ },
7
+ "run": {
8
+ "id": "run-provisional",
9
+ "startedAt": "2026-09-17T02:00:00Z",
10
+ "finishedAt": "2026-09-17T02:00:41Z",
11
+ "repo": {
12
+ "head": "deadbeefcafe",
13
+ "dirty": false,
14
+ "snapshot": "c0ffeeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
15
+ },
16
+ "runner": {
17
+ "name": "vitest",
18
+ "version": "5.0.1"
19
+ },
20
+ "confirmRuns": 1,
21
+ "mode": "worktree",
22
+ "provisional": true
23
+ },
24
+ "records": [
25
+ {
26
+ "fingerprint": "01876384493be6809da455cb1a1c4bc92e7564e9898506d790b9589200502163",
27
+ "claim": {
28
+ "id": "REDACT-001",
29
+ "statement": "The audit record written for a redacted message never contains the original secret.",
30
+ "severity": "critical",
31
+ "source": {
32
+ "kind": "spec",
33
+ "ref": "docs/spec.md#audit-records"
34
+ },
35
+ "producedBy": {
36
+ "producer": "human",
37
+ "by": "maintainer",
38
+ "at": "2026-09-17T02:00:00Z"
39
+ }
40
+ },
41
+ "subject": {
42
+ "kind": "fault",
43
+ "id": "F1",
44
+ "description": "Audit row is written with the raw input.",
45
+ "file": "src/redact.mjs",
46
+ "faultClass": "variable-swap",
47
+ "producedBy": {
48
+ "producer": "human",
49
+ "by": "maintainer",
50
+ "at": "2026-09-17T02:00:00Z"
51
+ }
52
+ },
53
+ "verdict": "survived",
54
+ "detail": {
55
+ "baselineRuns": [
56
+ {
57
+ "outcome": "pass",
58
+ "tests": {
59
+ "total": 4,
60
+ "passed": 4,
61
+ "failed": 0
62
+ },
63
+ "assertionFailures": 0,
64
+ "durationMs": 310
65
+ }
66
+ ],
67
+ "probeRuns": [
68
+ {
69
+ "outcome": "pass",
70
+ "tests": {
71
+ "total": 4,
72
+ "passed": 4,
73
+ "failed": 0
74
+ },
75
+ "assertionFailures": 0,
76
+ "durationMs": 310
77
+ }
78
+ ],
79
+ "escalated": true,
80
+ "escalationRuns": [
81
+ {
82
+ "outcome": "pass",
83
+ "tests": {
84
+ "total": 4,
85
+ "passed": 4,
86
+ "failed": 0
87
+ },
88
+ "assertionFailures": 0,
89
+ "durationMs": 310
90
+ }
91
+ ]
92
+ },
93
+ "defenders": {
94
+ "requested": [
95
+ "test/redact.test.mjs"
96
+ ],
97
+ "resolved": [
98
+ "test/redact.test.mjs"
99
+ ],
100
+ "nocover": false,
101
+ "discovered": false
102
+ },
103
+ "inputs": {
104
+ "targetHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
105
+ "defenderHashes": {
106
+ "test/redact.test.mjs": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
107
+ }
108
+ },
109
+ "rank": {
110
+ "score": 9.5,
111
+ "blastRadius": 7,
112
+ "tier": "critical"
113
+ },
114
+ "annotations": [
115
+ {
116
+ "kind": "tracked",
117
+ "text": "Assertion uses objectContaining without the content key.",
118
+ "by": "maintainer",
119
+ "at": "2026-09-17T02:00:00Z",
120
+ "ref": "issue#12"
121
+ }
122
+ ]
123
+ },
124
+ {
125
+ "fingerprint": "10d526ffb9535c8d99c39dcea29a8d6fa86c3370a2a5061de5320e028ba79b1f",
126
+ "claim": {
127
+ "id": "REDACT-001",
128
+ "statement": "The audit record written for a redacted message never contains the original secret.",
129
+ "severity": "critical",
130
+ "source": {
131
+ "kind": "spec",
132
+ "ref": "docs/spec.md#audit-records"
133
+ },
134
+ "producedBy": {
135
+ "producer": "human",
136
+ "by": "maintainer",
137
+ "at": "2026-09-17T02:00:00Z"
138
+ }
139
+ },
140
+ "subject": {
141
+ "kind": "fault",
142
+ "id": "F2",
143
+ "description": "Redaction skipped.",
144
+ "file": "src/redact.mjs",
145
+ "faultClass": "statement-deleted"
146
+ },
147
+ "verdict": "killed",
148
+ "detail": {
149
+ "baselineRuns": [
150
+ {
151
+ "outcome": "pass",
152
+ "tests": {
153
+ "total": 4,
154
+ "passed": 4,
155
+ "failed": 0
156
+ },
157
+ "assertionFailures": 0,
158
+ "durationMs": 310
159
+ }
160
+ ],
161
+ "probeRuns": [
162
+ {
163
+ "outcome": "fail",
164
+ "tests": {
165
+ "total": 4,
166
+ "passed": 3,
167
+ "failed": 1
168
+ },
169
+ "assertionFailures": 1,
170
+ "durationMs": 322
171
+ }
172
+ ]
173
+ },
174
+ "defenders": {
175
+ "requested": [
176
+ "test/redact.test.mjs"
177
+ ],
178
+ "resolved": [
179
+ "test/redact.test.mjs"
180
+ ],
181
+ "nocover": false
182
+ },
183
+ "inputs": {
184
+ "targetHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
185
+ "defenderHashes": {
186
+ "test/redact.test.mjs": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
187
+ }
188
+ }
189
+ },
190
+ {
191
+ "fingerprint": "0a0767ff8b124883e0bc042094c8494027b3d6d45bb8f4e40eda0930214a8970",
192
+ "claim": {
193
+ "id": "REDACT-002",
194
+ "statement": "A rule with an invalid pattern is skipped; it never aborts the scan.",
195
+ "severity": "high",
196
+ "source": {
197
+ "kind": "comment",
198
+ "ref": "src/redact.mjs:41"
199
+ }
200
+ },
201
+ "subject": {
202
+ "kind": "fault",
203
+ "id": "F1",
204
+ "description": "Invalid pattern rethrows.",
205
+ "file": "src/redact.mjs",
206
+ "faultClass": "exception-swallowed"
207
+ },
208
+ "verdict": "unverifiable",
209
+ "detail": {
210
+ "reason": "anchor-missing",
211
+ "baselineRuns": [],
212
+ "probeRuns": [],
213
+ "anchor": {
214
+ "hits": 0,
215
+ "expected": 1
216
+ }
217
+ },
218
+ "defenders": {
219
+ "requested": [
220
+ "test/redact.test.mjs"
221
+ ],
222
+ "resolved": [
223
+ "test/redact.test.mjs"
224
+ ],
225
+ "nocover": false
226
+ },
227
+ "inputs": {
228
+ "targetHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
229
+ "defenderHashes": {
230
+ "test/redact.test.mjs": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
231
+ }
232
+ }
233
+ },
234
+ {
235
+ "fingerprint": "81cb777d4cba8dd856ad3f0626fe58186906e5c18a01425d82957adaa963cb28",
236
+ "claim": {
237
+ "id": "REDACT-002",
238
+ "statement": "A rule with an invalid pattern is skipped; it never aborts the scan.",
239
+ "severity": "high",
240
+ "source": {
241
+ "kind": "comment",
242
+ "ref": "src/redact.mjs:41"
243
+ }
244
+ },
245
+ "subject": {
246
+ "kind": "fault",
247
+ "id": "F1",
248
+ "description": "Invalid pattern rethrows.",
249
+ "file": "src/redact.mjs",
250
+ "faultClass": "exception-swallowed"
251
+ },
252
+ "verdict": "nocover",
253
+ "detail": {
254
+ "baselineRuns": [],
255
+ "probeRuns": []
256
+ },
257
+ "defenders": {
258
+ "requested": [
259
+ "test/rules.test.mjs"
260
+ ],
261
+ "resolved": [],
262
+ "nocover": true
263
+ },
264
+ "inputs": {
265
+ "targetHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
266
+ "defenderHashes": {
267
+ "test/redact.test.mjs": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
268
+ }
269
+ }
270
+ }
271
+ ]
272
+ }