testguard-cli 0.5.0 → 0.6.0

package/spec/schemas/replay.schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:guard-spec:v1:replay",
+  "title": "Guard spec v1 — bug replay",
+  "description": "Would this suite have caught the bugs that actually escaped? For each fix commit in a range: revert only the source to its parent, remove the test the fix shipped, run what remains. `blind` means the suite was blind to a bug that really shipped. Unlike an injected fault, a replayed bug is ground truth — a human already confirmed it was a defect — which is what makes this the oracle a fault model is calibrated against.",
+  "type": "object",
+  "required": ["schemaVersion", "tool", "run", "records"],
+  "properties": {
+    "$schema": { "type": "string" },
+    "schemaVersion": { "const": 1 },
+    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
+    "run": {
+      "type": "object",
+      "required": ["id", "startedAt", "range", "confirmRuns"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1, "maxLength": 128 },
+        "startedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+        "finishedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+        "range": {
+          "description": "The commit range searched for fix commits, as given.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 256
+        },
+        "runner": { "type": "object", "required": ["name"], "properties": { "name": { "type": "string" }, "version": { "type": "string" } }, "additionalProperties": false },
+        "confirmRuns": { "type": "integer", "minimum": 1 },
+        "candidates": {
+          "description": "Fix commits found in the range before de-duplication by patch-id.",
+          "type": "integer",
+          "minimum": 0
+        },
+        "duplicates": {
+          "description": "Commits dropped as the same patch under another sha. A dual-branch topology carries the same fix two or three times; counting it twice corrupts the corpus.",
+          "type": "integer",
+          "minimum": 0
+        }
+      },
+      "additionalProperties": false
+    },
+    "records": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["commit", "patchId", "verdict", "source", "removedTests", "runs"],
+        "properties": {
+          "commit": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
+          "patchId": {
+            "description": "`git patch-id --stable` of the fix: the identity of the CHANGE rather than of the commit. A dual-branch topology carries the same fix under several shas, and counting it twice corrupts the corpus. Hex, 40 characters for a SHA-1 repository and 64 for a SHA-256 one; the commit sha when git could not compute one.",
+            "type": "string",
+            "pattern": "^[a-f0-9]{40}(?:[a-f0-9]{24})?$"
+          },
+          "subject": { "description": "The fix commit's subject line, for a human reading the corpus.", "type": "string", "maxLength": 512 },
+          "verdict": {
+            "description": "`caught` = a remaining test failed by assertion on the reverted source, every run. `blind` = the suite stayed green on known-broken code. `nocover` = no test exercises the reverted files at all, which is worse than blind. `unverifiable` = nothing could be concluded: the revert did not apply, the suite could not load, it timed out, or the commit only ADDS source (`no-prior-version` \u2014 there is no earlier behaviour to have been blind to, so it is not a bug the suite could have caught and it never enters a calibration). `flaky` = the runs disagreed, so nothing can be concluded.",
+            "enum": ["caught", "blind", "nocover", "unverifiable", "flaky"]
+          },
+          "reason": { "type": "string", "maxLength": 256 },
+          "source": {
+            "description": "Source files reverted to the fix's parent.",
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" }
+          },
+          "removedTests": {
+            "description": "Test files the fix shipped, removed before the run: the fix's own test proves nothing about what the suite knew before it.",
+            "type": "array",
+            "items": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" }
+          },
+          "ranTests": { "type": "integer", "minimum": 0 },
+          "faultClass": {
+            "description": "The injected-fault class this real diff most resembles, derived mechanically from the diff shape. This is the join key between ground truth and a fault model: it is what lets P(this fault class is a real bug) be computed.",
+            "$ref": "urn:guard-spec:v1:common#/$defs/faultClass"
+          },
+          "runs": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "urn:guard-spec:v1:evidence#/$defs/testRun" }
+          }
+        },
+        "additionalProperties": false
+      }
+    }
+  },
+  "additionalProperties": false
+}

package/spec/schemas/status.schema.json

@@ -1,27 +1,72 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "urn:guard-spec:v1:status",
-  "title": "Guard spec v1 — status",
+  "title": "Guard spec v1 \u2014 status",
   "description": "The single machine-readable answer to 'where is this project and what should happen next'. Every human rendering (CLI text, agent brief, skill instructions) derives from this document so they cannot disagree. Computed from the claims file, the evidence, the baseline and the current working tree; never from a cached verdict alone.",
   "type": "object",
-  "required": ["schemaVersion", "tool", "generatedAt", "state", "next", "counts"],
+  "required": [
+    "schemaVersion",
+    "tool",
+    "generatedAt",
+    "state",
+    "next",
+    "counts"
+  ],
   "properties": {
-    "$schema": { "type": "string" },
-    "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+    "$schema": {
+      "type": "string"
+    },
+    "schemaVersion": {
+      "const": 1
+    },
+    "tool": {
+      "$ref": "urn:guard-spec:v1:common#/$defs/tool"
+    },
+    "generatedAt": {
+      "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime"
+    },
     "state": {
       "description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
-      "enum": ["no-claims", "unclaimed-changes", "unprobed", "evidence-stale", "provisional-only", "unproven", "clean"]
+      "enum": [
+        "no-claims",
+        "unclaimed-changes",
+        "unprobed",
+        "evidence-stale",
+        "provisional-only",
+        "unproven",
+        "clean"
+      ]
     },
     "next": {
       "description": "The one action that moves the project forward. `command` is a shell line when the action is a tool command, or an instruction when it is not (write-test, claim for a test file).",
       "type": "object",
-      "required": ["action", "command", "why"],
+      "required": [
+        "action",
+        "command",
+        "why"
+      ],
       "properties": {
-        "action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "review-fault-change", "none"] },
-        "command": { "type": "string", "minLength": 1, "maxLength": 1024 },
-        "why": { "type": "string", "minLength": 1, "maxLength": 2000 },
+        "action": {
+          "enum": [
+            "scaffold",
+            "claim",
+            "probe",
+            "write-test",
+            "baseline",
+            "review-fault-change",
+            "none"
+          ]
+        },
+        "command": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 1024
+        },
+        "why": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
         "file": {
           "description": "For claim (and for scaffold when a change is known): the first uncovered changed file the action is about.",
           "$ref": "urn:guard-spec:v1:common#/$defs/repoPath"
@@ -29,38 +74,91 @@
         "target": {
           "description": "For write-test / review-fault-change: the finding the action is about.",
           "type": "object",
-          "required": ["claimId", "subjectId"],
+          "required": [
+            "claimId",
+            "subjectId"
+          ],
           "properties": {
-            "claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
-            "verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" }
+            "claimId": {
+              "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+            },
+            "subjectId": {
+              "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+            },
+            "file": {
+              "$ref": "urn:guard-spec:v1:common#/$defs/repoPath"
+            },
+            "verdict": {
+              "$ref": "urn:guard-spec:v1:common#/$defs/verdict"
+            }
           },
           "additionalProperties": false
         }
       },
       "additionalProperties": false
     },
-    "provisional": { "type": "boolean" },
+    "provisional": {
+      "type": "boolean"
+    },
     "counts": {
       "type": "object",
-      "required": ["claims", "faults"],
+      "required": [
+        "claims",
+        "faults"
+      ],
       "properties": {
-        "claims": { "type": "integer", "minimum": 0 },
-        "faults": { "type": "integer", "minimum": 0 },
+        "claims": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "faults": {
+          "type": "integer",
+          "minimum": 0
+        },
         "byVerdict": {
           "type": "object",
-          "propertyNames": { "enum": ["killed", "survived", "nocover", "unverifiable", "timeout", "fault-invalid", "flaky-defender"] },
-          "additionalProperties": { "type": "integer", "minimum": 0 }
+          "propertyNames": {
+            "enum": [
+              "killed",
+              "survived",
+              "nocover",
+              "unverifiable",
+              "timeout",
+              "fault-invalid",
+              "flaky-defender"
+            ]
+          },
+          "additionalProperties": {
+            "type": "integer",
+            "minimum": 0
+          }
+        },
+        "new": {
+          "type": "integer",
+          "minimum": 0
         },
-        "new": { "type": "integer", "minimum": 0 },
-        "baselined": { "type": "integer", "minimum": 0 }
+        "killedCoAuthored": { "description": "Kills whose defender was last touched by the same change (or author) as the code it guards — L3 independence. A signal, not a gate.", "type": "integer", "minimum": 0 },
+        "flakyDefenderSets": { "description": "Distinct defender sets that failed at least once on unmodified source.", "type": "integer", "minimum": 0 },
+        "worstFlakeRate": { "description": "Highest observed failures/runs on unmodified source across all records, 0–1. A number, so \"unstable\" stops being an adjective.", "type": "number", "minimum": 0, "maximum": 1 },
+        "baselined": {
+          "type": "integer",
+          "minimum": 0
+        }
       },
       "additionalProperties": false
     },
     "stale": {
       "description": "Why the evidence no longer describes the working tree. Empty when it does.",
       "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 512
+      }
+    },
+    "notes": {
+      "description": "Informational lines that never change `state`: e.g. a baseline frozen from a snapshot whose head predates the commit that carries its tests, with the way to re-stamp it.",
+      "type": "array",
       "items": { "type": "string", "minLength": 1, "maxLength": 512 }
     },
     "changedFaults": {
@@ -68,12 +166,24 @@
       "type": "array",
       "items": {
         "type": "object",
-        "required": ["claimId", "subjectId", "previousVerdict"],
+        "required": [
+          "claimId",
+          "subjectId",
+          "previousVerdict"
+        ],
         "properties": {
-          "claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "previousVerdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
-          "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" }
+          "claimId": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          },
+          "subjectId": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          },
+          "previousVerdict": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/verdict"
+          },
+          "file": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/repoPath"
+          }
         },
         "additionalProperties": false
       }
@@ -84,17 +194,43 @@
       "maxItems": 50,
       "items": {
         "type": "object",
-        "required": ["fingerprint", "claimId", "subjectId", "verdict", "severity", "isNew"],
+        "required": [
+          "fingerprint",
+          "claimId",
+          "subjectId",
+          "verdict",
+          "severity",
+          "isNew"
+        ],
         "properties": {
-          "fingerprint": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" },
-          "claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
-          "reason": { "type": "string", "maxLength": 256 },
-          "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
-          "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
-          "rank": { "type": "number" },
-          "isNew": { "type": "boolean" }
+          "fingerprint": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/sha256"
+          },
+          "claimId": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          },
+          "subjectId": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          },
+          "verdict": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/verdict"
+          },
+          "reason": {
+            "type": "string",
+            "maxLength": 256
+          },
+          "severity": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/severity"
+          },
+          "file": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/repoPath"
+          },
+          "rank": {
+            "type": "number"
+          },
+          "isNew": {
+            "type": "boolean"
+          }
         },
         "additionalProperties": false
       }
@@ -102,30 +238,96 @@
     "changes": {
       "description": "Claim coverage of the change since `ref`, present only when a reference was given (--changed) or detected from CI. Uncovered files come first in every rendering: the claim is written before more code. Same per-file shapes as the gate document.",
       "type": "object",
-      "required": ["ref", "base", "changed", "evaluated", "excluded", "uncovered", "reliedOn", "expired"],
+      "required": [
+        "ref",
+        "base",
+        "changed",
+        "evaluated",
+        "excluded",
+        "uncovered",
+        "reliedOn",
+        "expired"
+      ],
       "properties": {
-        "ref": { "type": "string", "minLength": 1, "maxLength": 256 },
-        "base": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
-        "includeDirty": { "type": "boolean" },
-        "changed": { "type": "integer", "minimum": 0 },
-        "evaluated": { "type": "integer", "minimum": 0 },
-        "excluded": { "type": "integer", "minimum": 0 },
-        "uncovered": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } },
-        "reliedOn": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } },
-        "expired": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance" } }
+        "ref": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 256
+        },
+        "base": {
+          "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+        },
+        "includeDirty": {
+          "type": "boolean"
+        },
+        "changed": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "evaluated": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "excluded": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "uncovered": {
+          "type": "array",
+          "items": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile"
+          }
+        },
+        "reliedOn": {
+          "type": "array",
+          "items": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance"
+          }
+        },
+        "expired": {
+          "type": "array",
+          "items": {
+            "$ref": "urn:guard-spec:v1:common#/$defs/ignoreReliance"
+          }
+        }
       },
       "additionalProperties": false
     },
     "paths": {
       "type": "object",
       "properties": {
-        "claims": { "type": "string" },
-        "ignore": { "type": "string" },
-        "evidence": { "type": "string" },
-        "provisionalEvidence": { "type": "string" },
-        "baseline": { "type": "string" }
+        "claims": {
+          "type": "string"
+        },
+        "ignore": {
+          "type": "string"
+        },
+        "evidence": {
+          "type": "string"
+        },
+        "provisionalEvidence": {
+          "type": "string"
+        },
+        "baseline": {
+          "type": "string"
+        }
       },
       "additionalProperties": false
+    },
+    "evidenceSource": {
+      "description": "Whether the evidence read was the project's own (`local`) or supplied with --evidence, e.g. fetched from CI (`provided`).",
+      "enum": [
+        "local",
+        "provided"
+      ]
+    },
+    "evidenceHead": {
+      "description": "The commit the evidence describes (its snapshot when the working tree was probed). Present whenever evidence was read.",
+      "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+    },
+    "head": {
+      "description": "HEAD of the working tree the status was computed in, when resolvable. Differs from evidenceHead when the evidence came from elsewhere.",
+      "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
     }
   },
   "additionalProperties": false

package/src/admit/admit.mjs

@@ -0,0 +1,17 @@
+/**
+ * The admission decision. Pure: records in, verdict out.
+ *
+ * A test is admitted only when every probed fault of its claim is `killed`
+ * — Gate A (green on unmodified HEAD, N/N) and Gate B (fails with the fault
+ * applied, N/N) are both inside `killed` by construction. Anything else
+ * blocks, and the first blocking record in gate order is named so the agent
+ * knows which fault to work on.
+ */
+const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender'];
+
+export function decideAdmission(records) {
+  if (records.length === 0) return { admitted: false, blocking: null };
+  const admitted = records.every((r) => r.verdict === 'killed');
+  const blocking = admitted ? null : [...records].filter((r) => r.verdict !== 'killed').sort((a, b) => ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict))[0];
+  return { admitted, blocking };
+}

package/src/baseline/baseline.mjs

@@ -1,13 +1,42 @@
 const SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
 
-/** Freeze every non-passing finding. `killed` is never debt, so it is never fingerprinted. */
+/**
+ * Freeze every non-passing finding. `killed` is never debt, so it is never
+ * fingerprinted. When the evidence came from a working-tree snapshot, the
+ * baseline says so (`snapshot`): its `head` is then the parent of the commit
+ * that will carry the tests, and `baseline --restamp` moves it once a clean
+ * probe of that commit reproduces the same fingerprints.
+ */
 export function buildBaseline(evidence, { createdAt = new Date().toISOString() } = {}) {
   const fingerprints = {};
   for (const r of evidence.records) {
     if (r.verdict === 'killed') continue;
     fingerprints[r.fingerprint] = (fingerprints[r.fingerprint] ?? 0) + 1;
   }
-  return { schemaVersion: 1, tool: evidence.tool, createdAt, head: evidence.run.repo.head, dirty: evidence.run.repo.dirty, fingerprints };
+  const repo = evidence.run.repo;
+  return { schemaVersion: 1, tool: evidence.tool, createdAt, head: repo.head, dirty: repo.dirty, ...(repo.snapshot ? { snapshot: repo.snapshot } : {}), fingerprints };
+}
+
+/** Same fingerprints, same counts. */
+export const sameFingerprints = (a, b) => {
+  const ka = Object.keys(a).sort();
+  const kb = Object.keys(b).sort();
+  return ka.length === kb.length && ka.every((k, i) => k === kb[i] && a[k] === b[k]);
+};
+
+/**
+ * Re-stamp a baseline's `head`/`dirty` from a later, CLEAN probe that
+ * reproduced exactly the same fingerprints. A frozen contract is never
+ * silently rewritten: different fingerprints, a dirty tree or a snapshot run
+ * are refused with the reason, and the caller re-probes and re-baselines.
+ */
+export function restampBaseline(baseline, evidence, { restampedAt = new Date().toISOString() } = {}) {
+  const repo = evidence.run.repo;
+  if (repo.snapshot || repo.dirty) return { ok: false, reason: `the evidence is from a ${repo.snapshot ? 'working-tree snapshot' : 'dirty tree'}; re-stamping needs a clean probe of the commit that carries the tests` };
+  const current = buildBaseline(evidence).fingerprints;
+  if (!sameFingerprints(baseline.fingerprints, current)) return { ok: false, reason: 'the current evidence does not reproduce the baseline\'s fingerprints; re-probe and run `testguard baseline` to freeze the new set instead' };
+  const { snapshot, ...rest } = baseline;
+  return { ok: true, baseline: { ...rest, head: repo.head, dirty: false, restampedAt } };
 }
 
 /**