testguard-cli 0.1.2__tar.gz → 0.2.0__tar.gz

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/.github/scripts/install-smoke.mjs

@@ -30,8 +30,10 @@ try {
   const bin = join(consumer, 'node_modules', '.bin', process.platform === 'win32' ? 'testguard.cmd' : 'testguard');
   const version = run(bin, ['--version'], consumer).trim();
   const claims = run(bin, ['claims', '.'], consumer);
+  const draft = JSON.parse(run(bin, ['scaffold', 'src/redact.mjs', '--json'], consumer));
+  if (!draft.claims?.length) throw new Error('scaffold produced no claims from the installed tarball');
   if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
-  if (!claims.includes('8 claims')) throw new Error(`claims did not list the fixture:\n${claims}`);
+  if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
   const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
   console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
 } finally {

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/AGENTS.md

@@ -19,6 +19,7 @@ npm run test:spec      # conformance suite only
 npm run self:probe     # TestGuard probes its own claims; must exit 0
 npm run test:install   # the packed tarball must run with production deps only
 node cli/testguard.mjs claims fixtures/known-answer
+node cli/testguard.mjs scaffold src/probe/classify.mjs --json   # what the producers propose for a file
 ```
 
 ## Key files
@@ -29,6 +30,7 @@ node cli/testguard.mjs claims fixtures/known-answer
 - `src/probe/probe.mjs` — orchestrator: isolation → baseline (cached per defender set) → apply → probe → escalate → restore → classify → rank.
 - `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
 - `testguard.claims.json` — claims about this codebase, probed in CI.
+- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
 
 ## Rules
 

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/CHANGELOG.md

@@ -7,6 +7,68 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.0] - 2026-09-17
+
+### Added
+
+- **`testguard scaffold <file>`** — mechanical fault producer (#6). Proposes
+  the five shapes both field reports found behind ~80% of hand-written
+  faults: guard forced false, single-line guard or state change removed,
+  `return <check>` → `return true`, security literal weakened, check call
+  removed. Every proposal is an exact-line anchor with `expectHits` and
+  `occurrence` computed from the file (verifiable by construction; anything
+  `locate()` would reject is never emitted), `producedBy: derived`,
+  `defendedBy` prefilled from the tests that import the module, grouped
+  under a preceding `@claim` annotation or by enclosing function.
+  Statements are `TODO:` placeholders; the output is a draft under
+  `.testguard/`, never the claims file. `--claim <ID>` puts everything under
+  one claim and copies it if it exists; `--json` prints instead.
+- Self-claim `TG-SCAFFOLD-ANCHORS-HIT`; install smoke exercises `scaffold`.
+
+## [0.1.3] - 2026-09-17
+
+From a second field report on a real codebase (456 tests, 27 claims, 35
+faults; 13 survived on the first run, two critical claims with zero coverage).
+
+### Fixed
+
+- **Worktree mode probed HEAD while reading the claims file from the working
+  tree**, so uncommitted defender changes were silently ignored — the same
+  survivors came back with no hint why. `probe` now refuses (exit 2) when any
+  resolved defender or fault target has uncommitted changes, naming the files
+  and the commit it would have probed. Every summary names the commit probed.
+- `killed-by-undeclared-tests` never said which tests killed the fault; the
+  author could not fix `defendedBy` without grepping the suite. Evidence now
+  carries `detail.undeclaredKillers` and the CLI names the files.
+- `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
+  expected }` is recorded and printed.
+- A replacement with an unbalanced paren was `suite-failed-to-load`, not
+  `replacement-does-not-compile`: esbuild/vitest wording is now matched.
+- The claims schema promised defender discovery for an absent `defendedBy`;
+  the tool answered `nocover`. Discovery is implemented: the test files that
+  import the fault's target (relative or alias), recorded as
+  `defenders.discovered`. `nocover` now means exactly "no test file imports
+  this source".
+- `baseline.json` records `dirty`, as evidence already did.
+
+### Added
+
+- `--include-dirty`: snapshot the working tree (tracked edits and untracked,
+  non-ignored files) into a throwaway commit and probe that. HEAD, index and
+  the user's tree are never touched; `run.repo.snapshot` records the commit.
+- A one-line progress indicator on stderr (TTY only) so a minute of silence
+  is not mistaken for a hang.
+- Fixture: a claim with no `defendedBy` whose defender is discovered.
+
+### Changed
+
+- **Default output shows only unproven faults plus a killed count.**
+  `--verbose` restores the full stream.
+- The `survived` hint reminds the author to check that the fault is
+  observable at all before writing a test for it.
+- README: `$schema` path for consumers, `min-release-age` note, the
+  worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
+
 ## [0.1.2] - 2026-09-17
 
 From a field report on a real codebase (63 test files, 458 tests, 39 faults).

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: testguard-cli
-Version: 0.1.2
+Version: 0.2.0
 Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
 Project-URL: Homepage, https://github.com/raccioly/testguard
 Project-URL: Documentation, https://github.com/raccioly/testguard#readme
@@ -76,9 +76,13 @@ tests were written against the survivors, 39/39 were killed.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
+Projects that set `min-release-age` in `.npmrc` cannot see a version published
+less than that many days ago (`ENOVERSIONS`); install that one with
+`npm i -D testguard-cli --min-release-age=0`.
+
 ## How it works
 
 ```bash
@@ -86,11 +90,14 @@ npx testguard-cli claims      # what does this project claim, and is every claim
 npx testguard-cli probe       # try to falsify each claim; report what the tests missed
 npx testguard-cli baseline    # freeze today's unproven findings; from now on only new ones gate
 npx testguard-cli brief       # tell the agent where the suite is blind, before it writes
+npx testguard-cli scaffold src/x.ts   # propose faults for a file, as a draft to keep or drop
 ```
 
-1. **Claims** live in `testguard.claims.json`: a statement, where it comes
-   from, which tests supposedly defend it, and one or more *faults* — each a
-   deterministic source change that would make the statement false. Every
+1. **Claims** live in `testguard.claims.json` (editors validate it against
+   `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
+   a statement, where it comes from, which tests supposedly defend it, and
+   one or more *faults* — each a deterministic source change that would make
+   the statement false. Every
    claim and every fault records who produced it. `testguard claims`
    validates the file and reports drift against `@claim <ID>` annotations in
    source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
@@ -115,10 +122,23 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    written to `.testguard/evidence.json` — validated against the spec before
    it is written.
 
+   Worktree mode probes a **commit**. If a defender or target file has
+   uncommitted changes, `probe` refuses and says so — otherwise your new
+   tests would be silently absent and the same survivors would come back
+   with no hint why. `--include-dirty` snapshots the working tree (tracked
+   edits and new files) into a throwaway commit and probes that; your tree,
+   HEAD and index are never touched. Every summary names the commit probed.
+
+   A claim with no `defendedBy` has its defenders **discovered**: the test
+   files that import the fault's target, by relative path or resolved alias.
+   `NOCOVER` then means exactly "no test file imports this source".
+
    Practical loop: first pass `--no-escalate` (escalation re-runs the whole
-   suite N times per survivor); iterate on one claim with
-   `--claim <ID> --in-place` — only the files faults are applied to must be
-   committed, test files may be dirty; final pass with defaults. A custom
+   suite N times per survivor); iterate on one claim with `--claim <ID>` and
+   either `--include-dirty` or `--in-place` (only fault target files must be
+   clean there; test files may be dirty); final pass with defaults. By default
+   the stream shows only unproven faults plus a killed count — `--verbose`
+   shows every fault. A custom
    runner (`pnpm --filter`, a specific config) goes in
    `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
    see your `node_modules`, pass `--node-modules <dir>`.
@@ -127,7 +147,8 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    source and defenders are unchanged reuse their prior verdict, so a probe
    in CI costs only what changed.
 4. **Brief** turns evidence plus baseline into a ranked, capped
-   `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
+   `## TEST BLINDSPOT CONTEXT` block, printed and also written to
+   `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
    — for Claude Code, in `.claude/settings.json`:
 
    ```json
@@ -139,6 +160,34 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    `--text` prints only, and exits 0 silently when there is no evidence yet,
    so the hook can never break a session.
 
+### Authoring faults mechanically
+
+Writing faults by hand means reading the code to find exact anchors. Two
+field reports found that ~80% of hand-written faults are one of five shapes,
+so `scaffold` proposes them for you:
+
+```bash
+npx testguard-cli scaffold src/auth.ts          # → .testguard/scaffold-auth.json (a draft, never your claims file)
+npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN   # every proposal under one claim; copies it if it exists
+```
+
+| Shape | What it proposes |
+|---|---|
+| `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
+| `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
+| `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
+| `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
+| `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
+
+Every proposal's `find` is the exact line with `expectHits`/`occurrence`
+computed from the file, so it is verifiable by construction; provenance is
+`producer: derived`; `defendedBy` is prefilled from the tests that import
+the module; proposals are grouped under a preceding `@claim <ID>` annotation
+or by enclosing function. Statements are `TODO:` placeholders — a proposal
+becomes a claim only when a human states what it defends. Deterministic
+heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
+emitted.
+
 **Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
 The baseline is the frozen contract; the other two are regenerated per run.
 
@@ -164,13 +213,14 @@ through every verdict.
 
 ## Status
 
-**v0.1.** Four commands, vitest runner, hand-authored faults. The contract
+**v0.2.** Five commands, vitest runner, hand-authored faults plus a
+mechanical scaffold for the five common shapes. The contract
 spine — six JSON Schemas shared with the other Guard tools — is under
 [`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
 
 Not yet: test generation (the two-gate acceptance loop), other runners,
-mechanical fault producers, and calibration of fault classes against real
-escaped bugs. Each is designed for; none is claimed.
+AST-aware producers, and calibration of fault classes against real escaped
+bugs. Each is designed for; none is claimed.
 
 ## Licence
 

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/README.md

@@ -52,9 +52,13 @@ tests were written against the survivors, 39/39 were killed.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
+Projects that set `min-release-age` in `.npmrc` cannot see a version published
+less than that many days ago (`ENOVERSIONS`); install that one with
+`npm i -D testguard-cli --min-release-age=0`.
+
 ## How it works
 
 ```bash
@@ -62,11 +66,14 @@ npx testguard-cli claims      # what does this project claim, and is every claim
 npx testguard-cli probe       # try to falsify each claim; report what the tests missed
 npx testguard-cli baseline    # freeze today's unproven findings; from now on only new ones gate
 npx testguard-cli brief       # tell the agent where the suite is blind, before it writes
+npx testguard-cli scaffold src/x.ts   # propose faults for a file, as a draft to keep or drop
 ```
 
-1. **Claims** live in `testguard.claims.json`: a statement, where it comes
-   from, which tests supposedly defend it, and one or more *faults* — each a
-   deterministic source change that would make the statement false. Every
+1. **Claims** live in `testguard.claims.json` (editors validate it against
+   `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
+   a statement, where it comes from, which tests supposedly defend it, and
+   one or more *faults* — each a deterministic source change that would make
+   the statement false. Every
    claim and every fault records who produced it. `testguard claims`
    validates the file and reports drift against `@claim <ID>` annotations in
    source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
@@ -91,10 +98,23 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    written to `.testguard/evidence.json` — validated against the spec before
    it is written.
 
+   Worktree mode probes a **commit**. If a defender or target file has
+   uncommitted changes, `probe` refuses and says so — otherwise your new
+   tests would be silently absent and the same survivors would come back
+   with no hint why. `--include-dirty` snapshots the working tree (tracked
+   edits and new files) into a throwaway commit and probes that; your tree,
+   HEAD and index are never touched. Every summary names the commit probed.
+
+   A claim with no `defendedBy` has its defenders **discovered**: the test
+   files that import the fault's target, by relative path or resolved alias.
+   `NOCOVER` then means exactly "no test file imports this source".
+
    Practical loop: first pass `--no-escalate` (escalation re-runs the whole
-   suite N times per survivor); iterate on one claim with
-   `--claim <ID> --in-place` — only the files faults are applied to must be
-   committed, test files may be dirty; final pass with defaults. A custom
+   suite N times per survivor); iterate on one claim with `--claim <ID>` and
+   either `--include-dirty` or `--in-place` (only fault target files must be
+   clean there; test files may be dirty); final pass with defaults. By default
+   the stream shows only unproven faults plus a killed count — `--verbose`
+   shows every fault. A custom
    runner (`pnpm --filter`, a specific config) goes in
    `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
    see your `node_modules`, pass `--node-modules <dir>`.
@@ -103,7 +123,8 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    source and defenders are unchanged reuse their prior verdict, so a probe
    in CI costs only what changed.
 4. **Brief** turns evidence plus baseline into a ranked, capped
-   `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
+   `## TEST BLINDSPOT CONTEXT` block, printed and also written to
+   `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
    — for Claude Code, in `.claude/settings.json`:
 
    ```json
@@ -115,6 +136,34 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    `--text` prints only, and exits 0 silently when there is no evidence yet,
    so the hook can never break a session.
 
+### Authoring faults mechanically
+
+Writing faults by hand means reading the code to find exact anchors. Two
+field reports found that ~80% of hand-written faults are one of five shapes,
+so `scaffold` proposes them for you:
+
+```bash
+npx testguard-cli scaffold src/auth.ts          # → .testguard/scaffold-auth.json (a draft, never your claims file)
+npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN   # every proposal under one claim; copies it if it exists
+```
+
+| Shape | What it proposes |
+|---|---|
+| `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
+| `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
+| `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
+| `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
+| `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
+
+Every proposal's `find` is the exact line with `expectHits`/`occurrence`
+computed from the file, so it is verifiable by construction; provenance is
+`producer: derived`; `defendedBy` is prefilled from the tests that import
+the module; proposals are grouped under a preceding `@claim <ID>` annotation
+or by enclosing function. Statements are `TODO:` placeholders — a proposal
+becomes a claim only when a human states what it defends. Deterministic
+heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
+emitted.
+
 **Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
 The baseline is the frozen contract; the other two are regenerated per run.
 
@@ -140,13 +189,14 @@ through every verdict.
 
 ## Status
 
-**v0.1.** Four commands, vitest runner, hand-authored faults. The contract
+**v0.2.** Five commands, vitest runner, hand-authored faults plus a
+mechanical scaffold for the five common shapes. The contract
 spine — six JSON Schemas shared with the other Guard tools — is under
 [`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
 
 Not yet: test generation (the two-gate acceptance loop), other runners,
-mechanical fault producers, and calibration of fault classes against real
-escaped bugs. Each is designed for; none is claimed.
+AST-aware producers, and calibration of fault classes against real escaped
+bugs. Each is designed for; none is claimed.
 
 ## Licence
 

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/action.yml

@@ -38,7 +38,7 @@ inputs:
   version:
     description: 'testguard-cli version to run'
     required: false
-    default: '0.1.2'
+    default: '0.2.0'
   node-version:
     description: 'Node.js version (20+)'
     required: false

{testguard_cli-0.1.2 → testguard_cli-0.2.0}/fixtures/known-answer/README.md

@@ -24,6 +24,7 @@ evidence comes from running against a real codebase with real history.
 | REDACT-006 | `mask()` replaces with equal-length asterisks | F1 syntax error | fault-invalid |
 | EXPORT-001 | Exported rows never include `content` | F1 keeps content | nocover |
 | FLAKY-001 | (defender is flaky) | F1 anything | flaky-defender |
+| DISCOVER-001 | Every match is replaced (no `defendedBy`) | F1 loop body removed | killed — defenders **discovered** by import |
 
 ## The exhibit — REDACT-001/F1
 

testguard_cli-0.2.0/fixtures/known-answer/expected.json

@@ -0,0 +1,53 @@
+{
+  "confirmRuns": 3,
+  "expected": {
+    "REDACT-001/F1": {
+      "verdict": "survived",
+      "why": "The audit-row test asserts with objectContaining and omits `content`; a row carrying the raw input passes."
+    },
+    "REDACT-001/F2": {
+      "verdict": "killed",
+      "why": "The masked-output test asserts the secret is absent."
+    },
+    "REDACT-002/F1": {
+      "verdict": "killed",
+      "why": "The invalid-rule test asserts compileRules does not throw."
+    },
+    "REDACT-003/F1": {
+      "verdict": "survived",
+      "why": "No test exercises a missing scope; the fail-closed guard is undefended."
+    },
+    "REDACT-004/F1": {
+      "verdict": "timeout",
+      "why": "Tests awaiting redact() hit the 1s test timeout; a timeout is not an assertion failure and is not a kill."
+    },
+    "REDACT-005/F1": {
+      "verdict": "unverifiable",
+      "reason": "anchor-missing",
+      "why": "The find string is not present in the source."
+    },
+    "REDACT-005/F2": {
+      "verdict": "unverifiable",
+      "reason": "anchor-ambiguous",
+      "why": "`return null;` occurs twice; expectHits is 1."
+    },
+    "REDACT-006/F1": {
+      "verdict": "fault-invalid",
+      "reason": "replacement-does-not-compile",
+      "why": "The replacement introduces a syntax error; the suite cannot load."
+    },
+    "EXPORT-001/F1": {
+      "verdict": "nocover",
+      "why": "test/export.test.mjs does not exist; no defender resolves."
+    },
+    "FLAKY-001/F1": {
+      "verdict": "flaky-defender",
+      "reason": "defenders-not-green",
+      "why": "test/flaky.test.mjs fails every other run; the baseline is not green N/N."
+    },
+    "DISCOVER-001/F1": {
+      "verdict": "killed",
+      "why": "No defendedBy; test/redact.test.mjs is discovered because it imports src/redact.mjs, and its masked-output test fails."
+    }
+  }
+}