testguard-cli 0.7.0__tar.gz → 0.8.0__tar.gz

testguard_cli-0.8.0/.github/scripts/check-cost-budget.mjs

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+/**
+ * Fail CI when the self-probe gate outgrows its budget.
+ *
+ * The decision lives in `src/probe/cost.mjs` as a pure function, unit-tested
+ * and covered by a self-claim; this script is only wiring — read the evidence,
+ * read the budget, print, choose an exit code.
+ *
+ * usage: check-cost-budget.mjs <evidence.json> [previous-evidence.json]
+ */
+import { readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { costReport, checkCostBudget, renderCostBudget } from '../../src/probe/cost.mjs';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const [evidencePath, previousPath] = process.argv.slice(2);
+if (!evidencePath) {
+  console.error('usage: check-cost-budget.mjs <evidence.json> [previous-evidence.json]');
+  process.exit(2);
+}
+
+const read = (p) => JSON.parse(readFileSync(resolve(p), 'utf8'));
+const budget = read(join(ROOT, 'testguard.cost-budget.json'));
+
+// A previous run is informational only. Its absence must never fail the check:
+// the first run on a branch, or a cold cache, has nothing to compare against.
+let previousMs;
+if (previousPath) {
+  try {
+    previousMs = costReport(read(previousPath).records).totalMs;
+  } catch {
+    previousMs = undefined;
+  }
+}
+
+const decision = checkCostBudget(costReport(read(evidencePath).records), { budgetSeconds: budget.seconds, previousMs });
+const text = renderCostBudget(decision);
+console.log(text);
+if (!decision.ok) {
+  console.log(`::error::${text.split('\n')[0]}`);
+  process.exit(1);
+}

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/.github/workflows/ci.yml

@@ -210,6 +210,14 @@ jobs:
         if: matrix.node-version == 22
         run: node cli/testguard.mjs probe . --budget 180000 --no-escalate --cost --out .testguard/ci-self-evidence.json
 
+      # The gate went 9.6 -> 23.6 min across one merged PR and nothing said a
+      # word, because --cost only ever printed into a log. `always()` so the
+      # cost is reported even when the probe itself failed: a run that got
+      # slower AND found a survivor should say both, not just the second.
+      - name: The gate must stay inside its cost budget
+        if: always() && matrix.node-version == 22
+        run: node .github/scripts/check-cost-budget.mjs .testguard/ci-self-evidence.json
+
       # A stable, branch-named artifact so a developer can fetch the evidence
       # CI produced without knowing a run id (README: read CI's evidence
       # locally). One Node leg is enough; the verdicts are identical.

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/CHANGELOG.md

@@ -7,6 +7,122 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.0] - 2026-09-18
+
+A calibration you can quote.
+
+A calibration document now says what its number means and where its labels
+came from, and it reproduces: `p` and every interval are recomputed by the
+validator from `n`, `positives`, `method` and `confidence`, at the document's
+own precision — the spec's own example had carried a truncated bound since
+the day it was written, and nothing could notice. `nocover` counts as the
+miss it is, so a project with no tests at all for a subsystem no longer
+outscores one with weak tests. The format also carries what
+websec-validator's shipped table already said — corpus, caveat, a producer
+floor, backoff tiers, a labelled prior — so a second tool's honesty survives
+translation; that table is now a conformance example, and "adoptable" means
+the file validates rather than a sentence saying so. Underneath: the
+self-probe gate is held to a cost budget (#89), and a reserved method has
+somewhere to put its data (#91).
+
+### Added
+- **A calibration can now say what it measures and where it came from** (#82).
+  websec-validator's shipped `calibration.json` already carried `corpus`,
+  `min_n`, `caveat`, `evidence_status` and `limitation` — the fields that say
+  "indicative, do not quote this number" — and claimspec had nowhere to put
+  them, so adopting it would have stripped that honesty in translation. It
+  also had nowhere to say what `p` *was*: testguard's is a miss rate where
+  high is bad, websec's is P(real) where high is good, and both validated
+  identically. The schema gains `measures` (a registry: `escape-missed`,
+  `finding-real`), provenance in `source` (`corpus`, `caveat`, `limitation`,
+  `evidenceStatus`, a `tool-oracle` kind, and an unconstrained `detail` on the
+  `methodDetail` pattern), a producer floor `minN` that consumers may raise
+  and never lower, ordered `backoff` tiers so a class→label→prior cascade is
+  expressible, a `fallback` whose shape says it is a guess, a per-cell
+  `breakdown` that sums to `n`, `p: null` at `n = 0`, and `|`-compound bucket
+  keys whose arity the validator checks. Every field is optional and absence
+  means *unattributed* — readable, not quotable — so no existing document
+  breaks; testguard's emitter always writes them and a self-claim guards it.
+  The proof of adoption is a file, not a sentence: websec-validator's table,
+  translated field for field, is now `spec/conformance/examples/calibration-websec.json`
+  and validates with every number reproducing. `GATE-SEMANTICS.md` records
+  the merge rule (sum counts, recompute, never average; only when `measures`
+  and `bucketBy` agree) that websec's shipped-plus-local overlay already
+  depends on.
+- **The gate is now gated on its own cost** (#89). `--cost` has printed the
+  self-probe's wall clock into every CI log since #67, and the gate still went
+  from 9.6 to 23.6 minutes across one merged change without anything saying a
+  word. A measurement nothing gates on is not a check — which is the premise
+  this tool rests on, applied to itself.
+  `testguard.cost-budget.json` carries the ceiling, its measurement and why it
+  is set where it is. CI fails when the probe exceeds it and opens with the
+  most expensive claims, so the first thing a failure shows is where to look.
+  Raising it is a committed diff with a reason: growth is allowed, going
+  unnoticed is not.
+  The budget is on the **total**, never a per-fault average. The regression
+  that prompted this took the corpus from 59 faults to 91 while per-fault cost
+  barely moved — an average would have reported everything fine while the gate
+  tripled, and a test says so. A budget that is missing or not a positive
+  number is refused rather than treated as unlimited, so a typo cannot quietly
+  disable the gate.
+
+
+### Fixed
+- **The cost budget is set against what CI measures.** 950 s was 18% over
+  the local 804 s figure and only 4.6% over the 906 s CI already measured on
+  `main`, so the first pull request to add any claim tripped it — #94 passed
+  its own leg only because the cache reused verdicts, and `main` went red at
+  981 s from scratch the moment it merged. Raised to 1100 s (~14% over the
+  962 s CI measures with #82's re-pointed defenders), with the CI figures
+  recorded beside the local one so the next person budgets against the
+  number the check actually sees. The heavy claims the failure now names are
+  the next #73-style cut.
+- **Calibration's pure tests moved off the replay fixture** (the #73 pattern).
+  Three calibration claims were defended by `test/replay.test.mjs`, which
+  builds a scripted git corpus on every run; their kills come from pure
+  `calibrationFrom` tests that need none of it. The cost budget fired on the
+  first full run — 1022 s against 950 s, the top rows being exactly those
+  claims — so the tests now live in `test/calibration.test.mjs`, the claims
+  point there, and every fault was re-probed after the move, not assumed.
+- **A cost-budget failure names its claims.** `checkCostBudget` read `id`
+  from a report whose claims carry `claimId`, so the one line meant to say
+  *which* claims to look at printed `undefined` five times — and its unit test
+  fed the shape the code wished for, so it passed. Found the first time the
+  budget actually fired. The test now goes through `costReport`, and
+  `TG-COST-BUDGET-NAMES-THE-CLAIM` guards it.
+- **A calibration counts `nocover` as a miss** (#82). `nocover` — no test
+  even imports the broken file — is the worst replay outcome, and the ratio
+  excluded it: a project with no tests at all for a subsystem scored *better*
+  than one with weak tests, because its worst outcomes left the denominator
+  before the ratio was taken. It now enters both sides, as it does in Stryker
+  and PIT, and the replay summary says how many misses were blind and how many
+  had no test at all. The schema description, the emitter's docstring and
+  `GATE-SEMANTICS.md` had also disagreed about what `p` was — "P(finding is
+  real)" in one place, the miss rate in another, and the docstring's two
+  sentences conditioning on different denominators — and now say one thing:
+  for testguard, the share of real escaped bugs the suite failed to catch.
+- **A calibration's arithmetic is checked, not trusted.** The validator
+  recomputed nothing: it checked that `p` sat inside `ci` and would pass a
+  document with every number invented. It now recomputes `p` from
+  `positives / n` and `ci` from `n`, `positives`, `method` and `confidence`
+  with `spec/lib/wilson.mjs` — one implementation, shared with the emitter —
+  at the document's own precision, accepting either the exact quantile or the
+  textbook 1.96. The spec's own example had carried a truncated lower bound
+  (6/30 → `0.09`; Wilson gives 0.0950 → `0.10`) since the day it was written,
+  and websec-validator's shipped table reproduces exactly.
+- **A reserved method can now say something.** #81 reserved `assertion` and
+  `scan` without required fields, and `fault` / `detail` are both
+  `additionalProperties: false` — so a probe using a reserved method could
+  declare that it existed and **nothing whatsoever about it**: not which rule
+  was checked, not where. That is not a reservation, it is a dead end, and
+  DocGuard's writer would have hit it on its first line.
+  Both now carry `methodDetail`: an object the spec deliberately does not
+  constrain, on the probe and on the record. It is **forbidden under
+  `fault-injection`**, whose shape is specified and must not acquire a junk
+  drawer — a self-claim fails if that ever stops being enforced — and it is a
+  staging area rather than a permanent home. What the first real consumer puts
+  there is the evidence for what the specified shape should become.
+
 ## [0.7.0] - 2026-09-18
 
 Python, and a fault that cannot hide.

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: testguard-cli
-Version: 0.7.0
+Version: 0.8.0
 Summary: Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest. Ships the Node.js CLI (requires Node.js 20+).
 Project-URL: Homepage, https://github.com/raccioly/testguard
 Project-URL: Documentation, https://github.com/raccioly/testguard#readme
@@ -150,9 +150,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.7.0` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.8.0` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
-| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.7.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
+| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.8.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
 
 Projects that set `min-release-age` in `.npmrc` cannot see a version published
 less than that many days ago (`ENOVERSIONS`); install that one with
@@ -369,15 +369,18 @@ most resembles, which is the join key that turns an uninterpretable mutation
 score into a statement with a sample size:
 
 ```
-  guard-removed          blind   7/9    p=0.78  ci [0.45, 0.94]
-  field-dropped          blind   4/4    p=1.00  ci [0.51, 1.00]
+  guard-removed          missed  7/9    p=0.78  ci [0.45, 0.94]
+  field-dropped          missed  4/4    p=1.00  ci [0.51, 1.00]
 ```
 
-Read as *"when a fault of this class survives, how often does that
-correspond to a bug that really escaped"*. It is written as a
-`calibration` document beside the replay one. Only `caught` and `blind`
-carry information; the other verdicts are excluded from both sides of the
-ratio.
+Read as *"when a real bug of this class escapes, how often does the suite
+miss it"* — a miss rate, so a high `p` is bad. It is written as a
+`calibration` document beside the replay one. `caught`, `blind` and
+`nocover` are measurements and enter the ratio, `nocover` as a miss: leaving
+it out would score a project with no tests at all for a subsystem *better*
+than one with weak tests. `flaky` and `unverifiable` are failed measurements
+and enter neither side. Every `p` and `ci` is recomputable from `n` and the
+miss count, and the validator recomputes them.
 
 **The open question this exists to answer.** Does a calibration learned on a
 repository *with* history transfer to a greenfield one that has none? AI
@@ -448,12 +451,12 @@ detected base that does not resolve is a warning for `status` and `brief`
 # GitHub Actions
 - uses: actions/checkout@v4
   with: { fetch-depth: 0 }
-- uses: raccioly/testguard@v0.7.0
+- uses: raccioly/testguard@v0.8.0
   with: { command: gate }
 
 # GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
 include:
-  - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.7.0/packaging/gitlab/testguard.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.8.0/packaging/gitlab/testguard.gitlab-ci.yml'
     inputs: { dir: backend, post_note: true }   # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
 ```
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/README.md

@@ -126,9 +126,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.7.0` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.8.0` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
-| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.7.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
+| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.8.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
 
 Projects that set `min-release-age` in `.npmrc` cannot see a version published
 less than that many days ago (`ENOVERSIONS`); install that one with
@@ -345,15 +345,18 @@ most resembles, which is the join key that turns an uninterpretable mutation
 score into a statement with a sample size:
 
 ```
-  guard-removed          blind   7/9    p=0.78  ci [0.45, 0.94]
-  field-dropped          blind   4/4    p=1.00  ci [0.51, 1.00]
+  guard-removed          missed  7/9    p=0.78  ci [0.45, 0.94]
+  field-dropped          missed  4/4    p=1.00  ci [0.51, 1.00]
 ```
 
-Read as *"when a fault of this class survives, how often does that
-correspond to a bug that really escaped"*. It is written as a
-`calibration` document beside the replay one. Only `caught` and `blind`
-carry information; the other verdicts are excluded from both sides of the
-ratio.
+Read as *"when a real bug of this class escapes, how often does the suite
+miss it"* — a miss rate, so a high `p` is bad. It is written as a
+`calibration` document beside the replay one. `caught`, `blind` and
+`nocover` are measurements and enter the ratio, `nocover` as a miss: leaving
+it out would score a project with no tests at all for a subsystem *better*
+than one with weak tests. `flaky` and `unverifiable` are failed measurements
+and enter neither side. Every `p` and `ci` is recomputable from `n` and the
+miss count, and the validator recomputes them.
 
 **The open question this exists to answer.** Does a calibration learned on a
 repository *with* history transfer to a greenfield one that has none? AI
@@ -424,12 +427,12 @@ detected base that does not resolve is a warning for `status` and `brief`
 # GitHub Actions
 - uses: actions/checkout@v4
   with: { fetch-depth: 0 }
-- uses: raccioly/testguard@v0.7.0
+- uses: raccioly/testguard@v0.8.0
   with: { command: gate }
 
 # GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
 include:
-  - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.7.0/packaging/gitlab/testguard.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.8.0/packaging/gitlab/testguard.gitlab-ci.yml'
     inputs: { dir: backend, post_note: true }   # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
 ```
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/action.yml

@@ -46,7 +46,7 @@ inputs:
   version:
     description: 'testguard-cli version to run'
     required: false
-    default: '0.7.0'
+    default: '0.8.0'
   node-version:
     description: 'Node.js version (20+)'
     required: false

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/docs-canonical/DATA-MODEL.md

@@ -42,7 +42,9 @@ defaults to `fault-injection`, whose required fields — `file`, `find`,
 baseline and probe runs on evidence — are unchanged and are enforced by the
 validator rather than merely by the schema. `assertion` and `scan` are
 reserved names for tools that verify by reading or scanning; their fields are
-defined by their first consumer, not in advance.
+defined by their first consumer, not in advance. Until then a reserved-method
+probe and record carry `methodDetail`, an unconstrained object that is
+forbidden under `fault-injection`.
 `spec/lib/validate.mjs` applies the schema **and** the semantic rules a schema
 cannot express. `spec/lib/fingerprint.mjs` is the single fingerprint
 implementation every tool must use.

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/docs-canonical/REQUIREMENTS.md

@@ -37,7 +37,7 @@
 | NFR-04 | Every emitted document conforms to the published schema before it is written | `writeSpecDoc` refuses non-conforming output |
 | NFR-05 | Verdicts are reproducible and attributable to a commit | evidence records `repo.head`, the snapshot when the working tree was probed, and the runner and its source |
 | NFR-06 | A verdict is never optimistic under uncertainty | contention recorded; flaky and timeout verdicts gate; escalation cannot upgrade a verdict; a survivor is unverifiable until the defenders are shown to fail because of the subject |
-| NFR-07 | Cost grows with what changed, not with the size of the project | verdict reuse on unchanged inputs; blast-radius-targeted escalation |
+| NFR-07 | Cost grows with what changed, not with the size of the project | verdict reuse on unchanged inputs; blast-radius-targeted escalation; a committed cost budget gated in CI, so growth is signed for rather than discovered |
 | NFR-08 | Portable contract | schemas are JSON Schema 2020-12, readable and implementable without this codebase |
 
 ## Success Criteria
@@ -97,7 +97,7 @@ fails the pull request naming the unclaimed file, and `status` reports
 | FR-10 | `src/probe/runners/` | fixture acceptance per runner, `TG-RUNNER-FROM-PROJECT-FIRST` |
 | FR-11 | `src/scaffold/producers.mjs` | `test/scaffold.test.mjs`, `TG-SCAFFOLD-ANCHORS-HIT` |
 | FR-12 | `src/admit/admit.mjs` | `test/admit.test.mjs`, `TG-ADMIT-NEEDS-ALL-KILLED` |
-| FR-13 | `src/replay/` | `test/replay.test.mjs`, `TG-REPLAY-FLAKY-IS-NEVER-CAUGHT` |
+| FR-13 | `src/replay/` | `test/replay.test.mjs`, `test/calibration.test.mjs`, `TG-REPLAY-FLAKY-IS-NEVER-CAUGHT`, `TG-CALIBRATION-COUNTS-NOCOVER-AS-A-MISS` |
 | FR-14 | `src/mcp/` | `test/mcp.test.mjs` |
 | NFR-01 | `src/init/init.mjs`, runner resolution | `TG-INIT-HOOK-NO-NETWORK`, `TG-README-HOOK-MATCHES-THE-CODE` |
 | NFR-02 | `package.json` | `npm run test:install` in CI |
@@ -105,8 +105,8 @@ fails the pull request naming the unclaimed file, and `status` reports
 | NFR-04 | `src/evidence/writer.mjs` | `test/writer.test.mjs` |
 | NFR-05 | `src/probe/probe.mjs` | `TG-FINGERPRINT-VERDICT`, `TG-FAULT-EDIT-VISIBLE` |
 | NFR-06 | `src/probe/contention.mjs`, `classify.mjs` | `TG-CONTENTION-NEVER-FAILS-A-PROBE`, `TG-ESCALATION-N-RUNS`, `TG-ONE-BAD-FAULT-KEEPS-THE-EVIDENCE`, `TG-A-PRECONDITION-STILL-REFUSES-THE-RUN`, `TG-A-PROBE-ERROR-IS-NEVER-REUSED`, `TG-SURVIVOR-PROVES-THE-SUBJECT-RUNS`, `TG-CONTROL-READS-ANY-RED-AS-REACHED`, `TG-FATAL-EDIT-CANNOT-COMPILE` |
-| NFR-07 | verdict reuse, `src/probe/rank.mjs` | `test/probe.fixture.test.mjs` reuse case |
-| NFR-08 | `spec/` | `spec/conformance/schemas.test.mjs` |
+| NFR-07 | verdict reuse, `src/probe/rank.mjs`, `src/probe/cost.mjs` | `test/probe.fixture.test.mjs` reuse case, `test/cost.test.mjs`, `TG-COST-BUDGET-ACTUALLY-GATES` |
+| NFR-08 | `spec/` | `spec/conformance/schemas.test.mjs`, `TG-WILSON-REPRODUCES-AT-THE-DOCUMENTS-PRECISION` |
 
 ## Revision History
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/docs-canonical/TEST-SPEC.md

@@ -20,7 +20,7 @@ the test that defends it, and CI fails if any injected fault survives.
 | Unit | `test/*.test.mjs` | One pure function's branches, without a runner | milliseconds |
 | Conformance | `spec/conformance/` | Every valid example validates; every must-reject document is rejected for the defect its filename names | ~1 s |
 | Fixture acceptance | `test/probe.fixture.test.mjs`, `test/probe.jest.test.mjs`, `test/probe.playwright.test.mjs` | The whole pipeline reproduces a known-answer oracle, one per runner | 10–50 s each |
-| Self-verification | `testguard.claims.json`, run by `npm run self:probe` | The tool's own invariants survive fault injection | **24 min** cold (measured 2026-09-18: 59 faults, `--serial`, idle machine; 1002 s baseline + 888 s probe); seconds when verdicts are reused, which is why CI restores the previous evidence from cache |
+| Self-verification | `testguard.claims.json`, run by `npm run self:probe` | The tool's own invariants survive fault injection | **13.4 min** cold (measured 2026-09-18 at `cf5a083`: 96 faults, 576 defender runs, 804 s, `--serial`, idle machine); seconds when verdicts are reused, which is why CI restores the previous evidence from cache. Budgeted at 950 s in `testguard.cost-budget.json` and **gated** in CI — the figure above went 9.6 → 23.6 min across one merged change while `--cost` printed it into a log nobody read |
 | Install smoke | `.github/scripts/install-smoke.mjs` | The **packed tarball** runs with production dependencies only | ~20 s |
 | Runtime budget | `ci.yml` | The suite has not silently started walking the wrong tree | gate, not a test |
 
@@ -77,7 +77,7 @@ What is gated instead:
 | `src/status/status.mjs` | `test/status.test.mjs` |
 | `src/brief/brief.mjs` | `test/brief.test.mjs` |
 | `src/init/init.mjs` | `test/init.test.mjs` |
-| `src/replay/` | `test/replay.test.mjs` |
+| `src/replay/` | `test/replay.test.mjs`, `test/calibration.test.mjs` (the pure half: `calibrationFrom`, `wilson`) |
 | `src/mcp/` | `test/mcp.test.mjs` |
 | `spec/lib/` | `spec/conformance/schemas.test.mjs` |
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "testguard-cli",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "testguard-cli",
-      "version": "0.6.0",
+      "version": "0.8.0",
       "license": "MIT",
       "dependencies": {
         "ajv": "8.20.0"

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testguard-cli",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
   "type": "module",
   "license": "MIT",

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/packaging/gitlab/testguard.gitlab-ci.yml

@@ -3,7 +3,7 @@
 # Include it from a project's .gitlab-ci.yml, pinned to a release:
 #
 #   include:
-#     - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.7.0/packaging/gitlab/testguard.gitlab-ci.yml'
+#     - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.8.0/packaging/gitlab/testguard.gitlab-ci.yml'
 #       inputs:
 #         dir: backend            # where testguard.claims.json lives (default ".")
 #         post_note: true         # post the brief as a merge-request note (needs TESTGUARD_GITLAB_TOKEN, api scope)
@@ -29,7 +29,7 @@ spec:
   inputs:
     version:
       description: testguard-cli version to run
-      default: "0.7.0"
+      default: "0.8.0"
     dir:
       description: directory whose test runner is invoked (where testguard.claims.json lives)
       default: "."

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/packaging/homebrew/testguard.rb

@@ -13,7 +13,7 @@
 class Testguard < Formula
   desc "Proves a test suite defends the claims a project makes"
   homepage "https://github.com/raccioly/testguard"
-  url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.7.0.tgz"
+  url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.8.0.tgz"
   sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
   license "MIT"
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "testguard-cli"
-version = "0.7.0"
+version = "0.8.0"
 description = "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest. Ships the Node.js CLI (requires Node.js 20+)."
 readme = "README.md"
 license = { text = "MIT" }

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/spec/GATE-SEMANTICS.md

@@ -167,6 +167,14 @@ Rules that follow from the table:
    under `fault-injection`, and the validator enforces them there in full: the
    relaxation is for a tool that injects nothing, never a licence for an
    injecting tool to stop showing its work.
+   A reserved method carries its own data in `methodDetail`, on the probe and
+   on the record: an object the spec deliberately does not constrain, so a tool
+   can say which rule it checked and where. Without it, "reserved" would mean a
+   probe that can declare it exists and nothing about it, which is not a
+   reservation but a dead end. It is **forbidden under `fault-injection`**,
+   whose shape is specified and must not acquire a junk drawer, and it is a
+   staging area rather than a permanent home: what the first consumer puts
+   there is the evidence for what the specified shape should become.
    `assertion` and `scan` are **reserved**. Their required fields are
    deliberately unspecified and will be defined by their first real consumer,
    with its own conformance examples written from the shape that tool actually
@@ -201,9 +209,96 @@ Two rules that follow:
 2. **The fix's own test is removed before the run.** It proves nothing about
    what the suite knew before the fix existed.
 
-Only `caught` and `blind` carry information. `nocover`, `flaky` and
-`unverifiable` are excluded from both sides of a calibration ratio, because a
-number computed over them would mean nothing.
+`caught`, `blind` and `nocover` are measurements and enter the calibration
+ratio: `caught` as a hit, the other two as misses. `nocover` counts as a miss
+because it is one — the worst kind. Leaving it out would let a project with
+**no tests at all** for a subsystem score better than one with weak tests,
+since its worst outcomes would leave the denominator before the ratio is
+taken; Stryker and PIT count no-coverage mutants against the headline score
+for the same reason. `flaky` and `unverifiable` are failed measurements and
+enter neither side.
+
+### Calibration arithmetic is reproducible
+
+A calibration cell declares `n` and `positives`, and with them has declared
+`p`; a document declares `method: wilson` at `confidence`, and with them has
+declared every `ci`. The validator recomputes both with `lib/wilson.mjs` —
+the single implementation, as `lib/fingerprint.mjs` is for fingerprints — and
+a cell that does not reproduce does not conform. Before this rule the
+validator checked only that `p` lay inside `ci`, which a document with every
+number invented satisfies, and the spec's own example shipped a truncated
+lower bound (6/30 → `0.09`, where Wilson gives 0.0950 → `0.10`) that nothing
+could notice.
+
+- **Precision is the document's own**: the most decimal places any `p` or
+  `ci` value shows, capped at 6. Values are compared after rounding to that
+  precision, so a producer at 3 dp and one at 4 dp both reproduce and a
+  truncated bound does not.
+- **z may be the exact quantile or the textbook rounding** (1.96, 2.576,
+  1.645). The two differ by up to 9.2e-6 in the interval — enough to flip a
+  rounding boundary at 4 dp — so either reproduces, and a document never fails
+  for having used the number in the textbook.
+- **`p` is `positives / n`.** A smoothed or shrunk point estimate is a
+  different method and must say so; under `wilson` the point estimate is the
+  observed proportion.
+
+### Calibration provenance, backoff and merging
+
+Everything beyond a document's primary buckets is optional and additive, in
+the way `method` is on claims and evidence: absence has a defined meaning, and
+no document written before a field existed becomes invalid. The meaning of
+absence is **unattributed** — readable, auditable, and not a source of
+numbers. A tool that emits calibrations always emits these fields; a
+self-claim guards that it does.
+
+- **`measures`** says what `p` is the probability of. It is the one field
+  that lets two calibrations be told apart: testguard's `escape-missed` is
+  P(a real escaped bug of this class was not caught), and high is bad;
+  websec-validator's `finding-real` is P(a reported finding of this class is a
+  real vulnerability), and high is good. A document without it cannot be
+  quoted and cannot be merged. Registry: `escape-missed`, `finding-real`. A
+  new value is added here by the tool that first emits it.
+- **`source`** carries provenance. `corpus` names external ground-truth
+  sources (a bug-replay's is the repository's own history, which `ref` already
+  names); `caveat` is the one sentence a consumer shows beside any quoted
+  number; `limitation` is the longer story; `evidenceStatus` says whether the
+  labels were reviewed; `kind: tool-oracle` is a tool confirming its own
+  findings, as websec-validator's dynamic probes do. `detail` is the tool's
+  own provenance, unconstrained, on the `methodDetail` pattern. A number
+  quoted without its `caveat` is a misquote.
+- **`minN`** is the producer's floor. A cell below it is not quoted on its
+  own; the consumer backs off a tier. A consumer may raise it — a gate that
+  blocks a merge wants more certainty than a report — and may never lower
+  it: lowering it resurrects a cell the producer suppressed. Absent, the
+  producer sets no floor and the consumer applies its own. A spec-level
+  constant would be wrong in both directions: websec-validator's 5 would
+  suppress every cell of a corpus the size of DocGuard's benchmark.
+- **`backoff`** is the ordered list of coarser tiers consulted below `minN`.
+  Each tier is held to the same arithmetic as the primary. Keys may be
+  compound (`attackClass|confidence` backs off to `confidence`), and every key
+  in a tier has as many `|`-parts as its `bucketBy` — a key with fewer was
+  translated from another table and lost a dimension on the way.
+- **`fallback`** is what remains when no tier has a cell above `minN`: a
+  labelled guess, not a measurement, and anything quoted from it carries
+  `n = 0`, `ci = [0, 1]` and its `basis`. It lives in the document so that
+  "what did it assume when it had no data?" has one auditable answer instead
+  of one per consumer, each unlabelled.
+- **`breakdown`** splits a cell's `n` by outcome when the producer can say
+  (testguard: `caught` / `blind` / `nocover`), and sums to `n`. It lets a
+  consumer recover a narrower rate — misses among covered code, say —
+  without the producer publishing a second, misleading headline.
+
+**Merging.** Two calibrations merge by summing `n` and `positives` per cell
+and recomputing `p` and `ci` from the sums — never by averaging `p` or `ci`,
+which have no meaning combined. They merge only when `measures`, `bucketBy`
+and `confidence` agree, and the merged document's `source.kind` is `mixed`.
+This is how websec-validator folds an operator's own confirmed samples over
+its shipped table, and it works only because cells carry raw counts.
+
+**The second producer is a conformance example.** websec-validator's shipped
+table, translated field for field, is `conformance/examples/calibration-websec.json`
+— corpus, caveat, limitation, floor, backoff tier and labelled fallback all
+present. That file, not a sentence in a README, is what "adoptable" means.
 
 ## Baseline and delta
 

{testguard_cli-0.7.0 → testguard_cli-0.8.0}/spec/README.md

@@ -28,7 +28,7 @@ that carry claims, baselines, scoping and briefs do not care.
 | `evidence` | [`schemas/evidence.schema.json`](schemas/evidence.schema.json) | One run's findings on disk, readable without the tool. Every verdict carries every run that produced it. |
 | `baseline` | [`schemas/baseline.schema.json`](schemas/baseline.schema.json) | Frozen fingerprints of existing debt. Gate only what is new. |
 | `ignore` | [`schemas/ignore.schema.json`](schemas/ignore.schema.json) | Reviewable scoping. Every entry has a reason. |
-| `calibration` | [`schemas/calibration.schema.json`](schemas/calibration.schema.json) | P(finding is real) per bucket, with a Wilson interval and the sample size behind it. |
+| `calibration` | [`schemas/calibration.schema.json`](schemas/calibration.schema.json) | The observed rate of a bucket's outcome — what it measures is in the document — with a Wilson interval and the sample size behind it. Every number is recomputable and the validator recomputes it; provenance, a producer floor, backoff tiers and a labelled fallback ride along so a second tool's honesty survives translation. |
 | `brief` | [`schemas/brief.schema.json`](schemas/brief.schema.json) | What to tell an agent before it writes code — ranked, capped, never a single score, and carrying the one next action. |
 | `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed, and changed files that carry no claim. |
 | `replay` | [`schemas/replay.schema.json`](schemas/replay.schema.json) | Would this suite have caught the bugs that already escaped? Replays real fix commits: revert the source, remove the test the fix shipped, run what remains. A replayed bug is ground truth — a human already confirmed it was a defect — which is what a fault model is calibrated against. |
@@ -65,7 +65,10 @@ consumer. It moves to its own repository only when a second tool adopts it.
 
 Adoption is additive and optional: an existing tool gains an output writer, and
 nothing it already detects or already writes changes. That holds today for
-`baseline`, `ignore`, `brief` and `status`. It does **not** yet hold for
+`baseline`, `ignore`, `brief`, `status` and `calibration` — websec-validator's
+shipped calibration table is `conformance/examples/calibration-websec.json`,
+translated field for field with its corpus, caveat, floor, backoff and
+fallback intact. It does **not** yet hold for
 `claims` and `evidence`, whose required fields assume fault injection
 (`find`/`replace` anchors, `confirmRuns`, `baselineRuns`/`probeRuns`,
 `defenders`) — a tool that scans or reads would have to emit empty arrays to