testguard-cli 0.1.1__tar.gz → 0.1.3__tar.gz

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/.github/scripts/install-smoke.mjs

@@ -31,7 +31,7 @@ try {
   const version = run(bin, ['--version'], consumer).trim();
   const claims = run(bin, ['claims', '.'], consumer);
   if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
-  if (!claims.includes('8 claims')) throw new Error(`claims did not list the fixture:\n${claims}`);
+  if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
   const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
   console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
 } finally {

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/.github/workflows/supply-chain.yml

@@ -17,8 +17,8 @@ permissions:
 jobs:
   osv-scan-pr:
     if: github.event_name == 'pull_request'
-    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@6e4298ebc4db23e847df9b2e2de2939d6f066c67" # v2.5.1
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc" # v2.6.0
 
   osv-scan-scheduled:
     if: github.event_name != 'pull_request'
-    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@6e4298ebc4db23e847df9b2e2de2939d6f066c67" # v2.5.1
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc" # v2.6.0

testguard_cli-0.1.3/CHANGELOG.md

@@ -0,0 +1,148 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.3] - 2026-09-17
+
+From a second field report on a real codebase (456 tests, 27 claims, 35
+faults; 13 survived on the first run, two critical claims with zero coverage).
+
+### Fixed
+
+- **Worktree mode probed HEAD while reading the claims file from the working
+  tree**, so uncommitted defender changes were silently ignored — the same
+  survivors came back with no hint why. `probe` now refuses (exit 2) when any
+  resolved defender or fault target has uncommitted changes, naming the files
+  and the commit it would have probed. Every summary names the commit probed.
+- `killed-by-undeclared-tests` never said which tests killed the fault; the
+  author could not fix `defendedBy` without grepping the suite. Evidence now
+  carries `detail.undeclaredKillers` and the CLI names the files.
+- `anchor-ambiguous` did not say how many hits; `detail.anchor { hits,
+  expected }` is recorded and printed.
+- A replacement with an unbalanced paren was `suite-failed-to-load`, not
+  `replacement-does-not-compile`: esbuild/vitest wording is now matched.
+- The claims schema promised defender discovery for an absent `defendedBy`;
+  the tool answered `nocover`. Discovery is implemented: the test files that
+  import the fault's target (relative or alias), recorded as
+  `defenders.discovered`. `nocover` now means exactly "no test file imports
+  this source".
+- `baseline.json` records `dirty`, as evidence already did.
+
+### Added
+
+- `--include-dirty`: snapshot the working tree (tracked edits and untracked,
+  non-ignored files) into a throwaway commit and probe that. HEAD, index and
+  the user's tree are never touched; `run.repo.snapshot` records the commit.
+- A one-line progress indicator on stderr (TTY only) so a minute of silence
+  is not mistaken for a hang.
+- Fixture: a claim with no `defendedBy` whose defender is discovered.
+
+### Changed
+
+- **Default output shows only unproven faults plus a killed count.**
+  `--verbose` restores the full stream.
+- The `survived` hint reminds the author to check that the fault is
+  observable at all before writing a test for it.
+- README: `$schema` path for consumers, `min-release-age` note, the
+  worktree-vs-working-tree rule, `brief` writes `brief.json` by default.
+
+## [0.1.2] - 2026-09-17
+
+From a field report on a real codebase (63 test files, 458 tests, 39 faults).
+
+### Fixed
+
+- **A runner that cannot be resolved was reported as `FLAKY-DEFENDER`.** In
+  the scratch worktree, a symlinked `node_modules` (the sibling/auto-worktree
+  layout) was invisible, vitest failed to load, and the load error was
+  classified as flaky tests — blaming the wrong party. Symlinked
+  `node_modules` are now linked to their resolved target; the runner is
+  checked before any verdict and an unresolvable one is a precondition
+  failure (exit 2) with the fix in the message; defenders that fail to
+  *load* during a baseline are `UNVERIFIABLE` (`defenders-failed-to-load`),
+  never flaky.
+- **Blast radius ignored path aliases.** `tsconfig`/`jsconfig` `paths` (with
+  `baseUrl` and relative `extends`) and `package.json#imports` are resolved,
+  so a module imported 83 times via `@/…` no longer ranks as if nothing
+  imported it. Bare package specifiers remain ignored, as documented.
+- Summary said "N unproven claims" when N counted faults; it now reports
+  unproven faults *and* the distinct claims they belong to.
+- `testguard claims` never showed which claims carry a `@claim` annotation;
+  it now reports the count and marks each annotated row.
+- Piping output to a closed reader (`testguard claims | head`) no longer
+  crashes with an `EPIPE` stack trace.
+
+### Added
+
+- `--claim <ID,ID>` probes only the named claims and writes
+  `.testguard/evidence-partial.json`, keeping the canonical evidence intact —
+  turns a fix-loop iteration from minutes into seconds.
+- `--runner-cmd "<cmd>"` with `{files}` and `{out}` placeholders for
+  monorepos, custom configs and other package managers.
+- `--node-modules <dir>` (or `TESTGUARD_NODE_MODULES`) to link a specific
+  `node_modules` into the scratch worktree.
+
+### Changed
+
+- The `--in-place` precondition message says what it means: only fault
+  target files must be clean; test files may be dirty.
+- `testguard baseline` prints the two `.gitignore` lines for the regenerated
+  files instead of leaving it to the README.
+- With no baseline, the ranked block is not printed a second time under the
+  per-fault stream.
+- `--quiet` is documented precisely.
+- Staged Homebrew formula carries the sha256 of the published 0.1.1 tarball.
+
+## [0.1.1] - 2026-09-17
+
+### Fixed
+
+- **0.1.0 did not run when installed.** `ajv`, which validates every document
+  against the spec, was declared as a devDependency, so `testguard` crashed on
+  startup from npm, `npx`, `pip` and Homebrew. It is now an exact-pinned
+  runtime dependency — the project's one dependency.
+- CI and the release job now pack the tarball, install it into a scratch
+  project with production dependencies only, and run the CLI from there
+  (`npm run test:install`). The suite alone runs from the checkout and could
+  not see this class of defect.
+
+### Changed
+
+- Homebrew formula carries the sha256 of the published tarball; description
+  shortened to satisfy `brew audit --strict`.
+
+## [0.1.0] - 2026-09-17
+
+First release. A claim verifier, not a test generator.
+
+### Added
+
+- **Contract spine** (`spec/`): six shared JSON Schemas — claims, evidence,
+  baseline, ignore, calibration, brief — identified as `urn:guard-spec:v1:*`,
+  with a validator that enforces the semantic rules a schema cannot express,
+  a single fingerprint derivation, `GATE-SEMANTICS.md`, and a conformance
+  corpus (one valid document per kind, 22 must-reject documents).
+- **`testguard claims`** — validates the claims file and reports drift
+  against `@claim <ID>` annotations in source.
+- **`testguard probe`** — applies each fault in a scratch git worktree,
+  confirms the defenders green N times, runs them N times with the fault,
+  escalates survivors to the whole suite with N-run attribution, restores,
+  classifies into a closed verdict set (`killed`, `survived`, `nocover`,
+  `unverifiable`, `timeout`, `fault-invalid`, `flaky-defender`), ranks, and
+  writes evidence validated against the spec. `--ref` pins the probed commit.
+  Verdicts are reused when the target, defenders and N are unchanged.
+- **`testguard baseline`** — freezes every non-passing fingerprint so only
+  new findings gate; severity floor via `--severity`.
+- **`testguard brief`** — a ranked, capped `## TEST BLINDSPOT CONTEXT` block
+  for an agent's session start; `--text` is hook-safe.
+- **Known-answer fixture** with a genuine `objectContaining` blind spot and a
+  case for every verdict; probed end to end in the test suite and in CI.
+- **Self-verification**: `testguard.claims.json` states invariants of the
+  tool itself, probed by the tool in CI.
+- Distribution: npm (`testguard-cli`), PyPI wrapper (`testguard-cli`),
+  GitHub Action, Homebrew formula (staged), pre-commit hooks.

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: testguard-cli
-Version: 0.1.1
+Version: 0.1.3
 Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
 Project-URL: Homepage, https://github.com/raccioly/testguard
 Project-URL: Documentation, https://github.com/raccioly/testguard#readme
@@ -59,6 +59,15 @@ on known-broken code. The largest gap was a compliance-critical path with
 100% coverage, where the one assertion that mattered used
 `expect.objectContaining({...})` and omitted the field carrying the data.
 
+A second, independent run on a different AI-authored codebase (63 test
+files, 458 tests, 24 hand-written security claims, 39 faults): **21 of 39
+faults survived a fully green suite — 9 of them critical.** Super-admin
+gating, membership checks, cookie flags and the whole authorization callback
+could be disabled without a single test noticing. One test file had
+re-implemented the authorization logic *inside the test* and asserted
+against the copy: fifteen green tests, zero detection. After wrapper-level
+tests were written against the survivors, 39/39 were killed.
+
 ## Install
 
 | How | Command |
@@ -67,9 +76,13 @@ on known-broken code. The largest gap was a compliance-critical path with
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.1.1` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
+Projects that set `min-release-age` in `.npmrc` cannot see a version published
+less than that many days ago (`ENOVERSIONS`); install that one with
+`npm i -D testguard-cli --min-release-age=0`.
+
 ## How it works
 
 ```bash
@@ -79,12 +92,14 @@ npx testguard-cli baseline    # freeze today's unproven findings; from now on on
 npx testguard-cli brief       # tell the agent where the suite is blind, before it writes
 ```
 
-1. **Claims** live in `testguard.claims.json`: a statement, where it comes
-   from, which tests supposedly defend it, and one or more *faults* — each a
-   deterministic source change that would make the statement false. Every
+1. **Claims** live in `testguard.claims.json` (editors validate it against
+   `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
+   a statement, where it comes from, which tests supposedly defend it, and
+   one or more *faults* — each a deterministic source change that would make
+   the statement false. Every
    claim and every fault records who produced it. `testguard claims`
    validates the file and reports drift against `@claim <ID>` annotations in
-   source.
+   source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
 2. **Probe** confirms the defenders are green N times unmodified, applies
    each fault in a scratch git worktree (your tree is never touched), runs
    the defenders N times, re-runs survivors against the whole suite with
@@ -101,14 +116,38 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    | `FLAKY-DEFENDER` | the defenders are not reliably green, or disagreed across runs |
 
    Never a single score. Findings are ranked by severity, claim provenance
-   and blast radius, and written to `.testguard/evidence.json` — validated
-   against the spec before it is written.
+   and blast radius (relative imports, `tsconfig` path aliases and
+   `package.json#imports` are resolved; bare package names are not), and
+   written to `.testguard/evidence.json` — validated against the spec before
+   it is written.
+
+   Worktree mode probes a **commit**. If a defender or target file has
+   uncommitted changes, `probe` refuses and says so — otherwise your new
+   tests would be silently absent and the same survivors would come back
+   with no hint why. `--include-dirty` snapshots the working tree (tracked
+   edits and new files) into a throwaway commit and probes that; your tree,
+   HEAD and index are never touched. Every summary names the commit probed.
+
+   A claim with no `defendedBy` has its defenders **discovered**: the test
+   files that import the fault's target, by relative path or resolved alias.
+   `NOCOVER` then means exactly "no test file imports this source".
+
+   Practical loop: first pass `--no-escalate` (escalation re-runs the whole
+   suite N times per survivor); iterate on one claim with `--claim <ID>` and
+   either `--include-dirty` or `--in-place` (only fault target files must be
+   clean there; test files may be dirty); final pass with defaults. By default
+   the stream shows only unproven faults plus a killed count — `--verbose`
+   shows every fault. A custom
+   runner (`pnpm --filter`, a specific config) goes in
+   `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
+   see your `node_modules`, pass `--node-modules <dir>`.
 3. **Baseline** freezes every non-passing fingerprint. Later probes suppress
    what was already known and exit non-zero only on what is new. Claims whose
    source and defenders are unchanged reuse their prior verdict, so a probe
    in CI costs only what changed.
 4. **Brief** turns evidence plus baseline into a ranked, capped
-   `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
+   `## TEST BLINDSPOT CONTEXT` block, printed and also written to
+   `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
    — for Claude Code, in `.claude/settings.json`:
 
    ```json

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/README.md

@@ -35,6 +35,15 @@ on known-broken code. The largest gap was a compliance-critical path with
 100% coverage, where the one assertion that mattered used
 `expect.objectContaining({...})` and omitted the field carrying the data.
 
+A second, independent run on a different AI-authored codebase (63 test
+files, 458 tests, 24 hand-written security claims, 39 faults): **21 of 39
+faults survived a fully green suite — 9 of them critical.** Super-admin
+gating, membership checks, cookie flags and the whole authorization callback
+could be disabled without a single test noticing. One test file had
+re-implemented the authorization logic *inside the test* and asserted
+against the copy: fifteen green tests, zero detection. After wrapper-level
+tests were written against the survivors, 39/39 were killed.
+
 ## Install
 
 | How | Command |
@@ -43,9 +52,13 @@ on known-broken code. The largest gap was a compliance-critical path with
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.1.1` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.1.3` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
+Projects that set `min-release-age` in `.npmrc` cannot see a version published
+less than that many days ago (`ENOVERSIONS`); install that one with
+`npm i -D testguard-cli --min-release-age=0`.
+
 ## How it works
 
 ```bash
@@ -55,12 +68,14 @@ npx testguard-cli baseline    # freeze today's unproven findings; from now on on
 npx testguard-cli brief       # tell the agent where the suite is blind, before it writes
 ```
 
-1. **Claims** live in `testguard.claims.json`: a statement, where it comes
-   from, which tests supposedly defend it, and one or more *faults* — each a
-   deterministic source change that would make the statement false. Every
+1. **Claims** live in `testguard.claims.json` (editors validate it against
+   `"$schema": "./node_modules/testguard-cli/spec/schemas/claims.schema.json"`):
+   a statement, where it comes from, which tests supposedly defend it, and
+   one or more *faults* — each a deterministic source change that would make
+   the statement false. Every
    claim and every fault records who produced it. `testguard claims`
    validates the file and reports drift against `@claim <ID>` annotations in
-   source.
+   source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
 2. **Probe** confirms the defenders are green N times unmodified, applies
    each fault in a scratch git worktree (your tree is never touched), runs
    the defenders N times, re-runs survivors against the whole suite with
@@ -77,14 +92,38 @@ npx testguard-cli brief       # tell the agent where the suite is blind, before
    | `FLAKY-DEFENDER` | the defenders are not reliably green, or disagreed across runs |
 
    Never a single score. Findings are ranked by severity, claim provenance
-   and blast radius, and written to `.testguard/evidence.json` — validated
-   against the spec before it is written.
+   and blast radius (relative imports, `tsconfig` path aliases and
+   `package.json#imports` are resolved; bare package names are not), and
+   written to `.testguard/evidence.json` — validated against the spec before
+   it is written.
+
+   Worktree mode probes a **commit**. If a defender or target file has
+   uncommitted changes, `probe` refuses and says so — otherwise your new
+   tests would be silently absent and the same survivors would come back
+   with no hint why. `--include-dirty` snapshots the working tree (tracked
+   edits and new files) into a throwaway commit and probes that; your tree,
+   HEAD and index are never touched. Every summary names the commit probed.
+
+   A claim with no `defendedBy` has its defenders **discovered**: the test
+   files that import the fault's target, by relative path or resolved alias.
+   `NOCOVER` then means exactly "no test file imports this source".
+
+   Practical loop: first pass `--no-escalate` (escalation re-runs the whole
+   suite N times per survivor); iterate on one claim with `--claim <ID>` and
+   either `--include-dirty` or `--in-place` (only fault target files must be
+   clean there; test files may be dirty); final pass with defaults. By default
+   the stream shows only unproven faults plus a killed count — `--verbose`
+   shows every fault. A custom
+   runner (`pnpm --filter`, a specific config) goes in
+   `--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
+   see your `node_modules`, pass `--node-modules <dir>`.
 3. **Baseline** freezes every non-passing fingerprint. Later probes suppress
    what was already known and exit non-zero only on what is new. Claims whose
    source and defenders are unchanged reuse their prior verdict, so a probe
    in CI costs only what changed.
 4. **Brief** turns evidence plus baseline into a ranked, capped
-   `## TEST BLINDSPOT CONTEXT` block. Wire it into an agent's session start
+   `## TEST BLINDSPOT CONTEXT` block, printed and also written to
+   `.testguard/brief.json` (`--text` prints only). Wire it into an agent's session start
    — for Claude Code, in `.claude/settings.json`:
 
    ```json

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/action.yml

@@ -38,7 +38,7 @@ inputs:
   version:
     description: 'testguard-cli version to run'
     required: false
-    default: '0.1.1'
+    default: '0.1.3'
   node-version:
     description: 'Node.js version (20+)'
     required: false

testguard_cli-0.1.3/cli/testguard.mjs

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import { main } from '../src/cli.mjs';
+
+// `testguard … | head` closes stdout early; that is not an error worth a stack trace.
+for (const stream of [process.stdout, process.stderr]) {
+  stream.on('error', (err) => {
+    if (err.code === 'EPIPE') process.exit(0);
+    throw err;
+  });
+}
+
+main(process.argv.slice(2)).then((code) => {
+  process.exitCode = code;
+});

{testguard_cli-0.1.1 → testguard_cli-0.1.3}/fixtures/known-answer/README.md

@@ -24,6 +24,7 @@ evidence comes from running against a real codebase with real history.
 | REDACT-006 | `mask()` replaces with equal-length asterisks | F1 syntax error | fault-invalid |
 | EXPORT-001 | Exported rows never include `content` | F1 keeps content | nocover |
 | FLAKY-001 | (defender is flaky) | F1 anything | flaky-defender |
+| DISCOVER-001 | Every match is replaced (no `defendedBy`) | F1 loop body removed | killed — defenders **discovered** by import |
 
 ## The exhibit — REDACT-001/F1
 

testguard_cli-0.1.3/fixtures/known-answer/expected.json

@@ -0,0 +1,53 @@
+{
+  "confirmRuns": 3,
+  "expected": {
+    "REDACT-001/F1": {
+      "verdict": "survived",
+      "why": "The audit-row test asserts with objectContaining and omits `content`; a row carrying the raw input passes."
+    },
+    "REDACT-001/F2": {
+      "verdict": "killed",
+      "why": "The masked-output test asserts the secret is absent."
+    },
+    "REDACT-002/F1": {
+      "verdict": "killed",
+      "why": "The invalid-rule test asserts compileRules does not throw."
+    },
+    "REDACT-003/F1": {
+      "verdict": "survived",
+      "why": "No test exercises a missing scope; the fail-closed guard is undefended."
+    },
+    "REDACT-004/F1": {
+      "verdict": "timeout",
+      "why": "Tests awaiting redact() hit the 1s test timeout; a timeout is not an assertion failure and is not a kill."
+    },
+    "REDACT-005/F1": {
+      "verdict": "unverifiable",
+      "reason": "anchor-missing",
+      "why": "The find string is not present in the source."
+    },
+    "REDACT-005/F2": {
+      "verdict": "unverifiable",
+      "reason": "anchor-ambiguous",
+      "why": "`return null;` occurs twice; expectHits is 1."
+    },
+    "REDACT-006/F1": {
+      "verdict": "fault-invalid",
+      "reason": "replacement-does-not-compile",
+      "why": "The replacement introduces a syntax error; the suite cannot load."
+    },
+    "EXPORT-001/F1": {
+      "verdict": "nocover",
+      "why": "test/export.test.mjs does not exist; no defender resolves."
+    },
+    "FLAKY-001/F1": {
+      "verdict": "flaky-defender",
+      "reason": "defenders-not-green",
+      "why": "test/flaky.test.mjs fails every other run; the baseline is not green N/N."
+    },
+    "DISCOVER-001/F1": {
+      "verdict": "killed",
+      "why": "No defendedBy; test/redact.test.mjs is discovered because it imports src/redact.mjs, and its masked-output test fails."
+    }
+  }
+}