testguard-cli 0.4.0 → 0.6.0

package/spec/lib/validate.mjs

@@ -6,7 +6,7 @@ import { fingerprint } from './fingerprint.mjs';
 
 const schemaDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'schemas');
 
-export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status']);
+export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status', 'gate', 'replay']);
 export const PASSING_VERDICTS = Object.freeze(new Set(['killed']));
 
 const ajv = new Ajv2020({ strict: true, allErrors: true });
@@ -47,6 +47,8 @@ const semantic = {
     const n = doc.run.confirmRuns;
     if (n < 3 && doc.run.provisional !== true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is below 3; the run must declare provisional: true` });
     if (n >= 3 && doc.run.provisional === true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is confirmed; provisional must be absent or false` });
+    if (doc.run.runners && doc.run.runner && !doc.run.runners.some((x) => x.name === doc.run.runner.name)) errors.push({ path: '/run/runners', message: 'runners must include the project runner named in run.runner' });
+    if (doc.run.repo.ignoredDirty && doc.run.repo.snapshot) errors.push({ path: '/run/repo/ignoredDirty', message: 'a working-tree snapshot has no ignored dirty files: the tree was probed as it is' });
     doc.records.forEach((r, i) => {
       const p = `/records/${i}`;
       const expected = fingerprint({ claimId: r.claim.id, subjectId: r.subject.id, file: r.subject.file ?? '', verdict: r.verdict });
@@ -74,9 +76,31 @@ const semantic = {
       if (r.verdict === 'nocover' && !r.defenders.nocover) {
         errors.push({ path: `${p}/defenders/nocover`, message: 'nocover verdict requires defenders.nocover = true' });
       }
+      if (r.detail.flakeRate) {
+        const { runs, failures } = r.detail.flakeRate;
+        if (failures > runs) errors.push({ path: `${p}/detail/flakeRate`, message: `flakeRate failures (${failures}) exceed runs (${runs})` });
+        if (failures > 0 && r.verdict !== 'flaky-defender') errors.push({ path: `${p}/detail/flakeRate`, message: 'a defender that failed on unmodified source makes the verdict flaky-defender' });
+      }
+      if (r.detail.independence) {
+        if (r.verdict !== 'killed') errors.push({ path: `${p}/detail/independence`, message: 'independence is a property of a kill; only killed records carry it' });
+        const { class: cls, defenderCommit, targetCommit } = r.detail.independence;
+        if (cls !== 'unknown' && !(defenderCommit && targetCommit)) errors.push({ path: `${p}/detail/independence`, message: `independence class ${cls} requires both defenderCommit and targetCommit` });
+      }
       if (r.detail.undeclaredKillers && r.detail.reason !== 'killed-by-undeclared-tests') {
         errors.push({ path: `${p}/detail/undeclaredKillers`, message: 'undeclaredKillers is only meaningful with reason killed-by-undeclared-tests' });
       }
+      if (r.defenders.byRunner) {
+        const resolved = new Set(r.defenders.resolved);
+        for (const [runner, files] of Object.entries(r.defenders.byRunner)) for (const f of files) if (!resolved.has(f)) errors.push({ path: `${p}/defenders/byRunner/${runner}`, message: `${f} ran under ${runner} but is not a resolved defender` });
+        if (Object.keys(r.defenders.byRunner).length < 2) errors.push({ path: `${p}/defenders/byRunner`, message: 'byRunner is only meaningful when more than one runner ran the defenders' });
+      }
+      if (r.defenders.mocking && r.defenders.discovered) {
+        for (const f of r.defenders.mocking) if (r.defenders.resolved.includes(f)) errors.push({ path: `${p}/defenders/mocking`, message: `${f} mocks the subject and was discovered; it cannot also be a resolved defender` });
+      }
+      for (const s of r.defenders.signals ?? []) {
+        if (s.signal === 'unasserted-annotated' && !s.reason) errors.push({ path: `${p}/defenders/signals`, message: `unasserted-annotated on ${s.file} requires the annotation's reason` });
+        if (!(r.defenders.mocking ?? []).includes(s.file)) errors.push({ path: `${p}/defenders/signals`, message: `${s.file} carries a mock signal but is not listed in defenders.mocking` });
+      }
       if (r.verdict === 'unverifiable' && !r.detail.reason) {
         errors.push({ path: `${p}/detail/reason`, message: 'unverifiable requires a reason (e.g. anchor-missing, anchor-ambiguous)' });
       }
@@ -99,8 +123,58 @@ const semantic = {
   status(doc) {
     const errors = [];
     if (['write-test', 'review-fault-change'].includes(doc.next.action) && !doc.next.target) errors.push({ path: '/next/target', message: `${doc.next.action} requires a target` });
+    if (doc.next.action === 'claim' && !doc.next.file) errors.push({ path: '/next/file', message: 'claim requires the file the claim is about' });
     if (doc.state === 'no-claims' && doc.counts.claims !== 0) errors.push({ path: '/counts/claims', message: 'no-claims with a non-zero claim count' });
     if (doc.state === 'clean' && (doc.counts.new ?? 0) > 0) errors.push({ path: '/state', message: 'clean with new findings' });
+    if (doc.state === 'unclaimed-changes' && !(doc.changes && doc.changes.uncovered.length > 0)) errors.push({ path: '/changes/uncovered', message: 'unclaimed-changes requires at least one uncovered changed file' });
+    if (doc.state === 'unclaimed-changes' && doc.next.action !== 'claim') errors.push({ path: '/next/action', message: 'unclaimed-changes requires next.action = claim' });
+    if (doc.evidenceSource && !doc.evidenceHead) errors.push({ path: '/evidenceHead', message: 'evidence was read, so the commit it describes must be recorded' });
+    if (doc.changes && doc.changes.uncovered.length > 0 && !['no-claims', 'unclaimed-changes'].includes(doc.state)) errors.push({ path: '/state', message: 'uncovered changed files are hidden behind a later state; unclaimed-changes precedes every evidence state' });
+    return errors;
+  },
+
+  replay(doc) {
+    const errors = [];
+    const n = doc.run.confirmRuns;
+    const seen = new Set();
+    doc.records.forEach((r, i) => {
+      const p = `/records/${i}`;
+      // One patch, one row: a dual-branch topology carries the same fix under
+      // two or three shas, and counting it twice corrupts the corpus the
+      // calibration is computed from.
+      if (seen.has(r.patchId)) errors.push({ path: `${p}/patchId`, message: `duplicate patch-id ${r.patchId.slice(0, 12)}: the same fix counted twice` });
+      seen.add(r.patchId);
+      if (['caught', 'blind'].includes(r.verdict) && r.runs.length !== n) {
+        errors.push({ path: `${p}/runs`, message: `${r.verdict} requires exactly confirmRuns (${n}) runs, got ${r.runs.length}` });
+      }
+      // Only a test body rejecting the reverted source counts as caught —
+      // the same rule as `killed`, for the same reason.
+      if (r.verdict === 'caught' && !r.runs.every((x) => x.outcome === 'fail' && (x.assertionFailures ?? 0) > 0)) {
+        errors.push({ path: `${p}/runs`, message: 'caught requires every run to fail with at least one assertion failure' });
+      }
+      if (r.verdict === 'blind' && !r.runs.every((x) => x.outcome === 'pass')) {
+        errors.push({ path: `${p}/runs`, message: 'blind requires every run to pass: the suite stayed green on known-broken code' });
+      }
+      if (r.verdict === 'flaky' && !(r.runs.some((x) => x.outcome === 'pass') && r.runs.some((x) => x.outcome === 'fail'))) {
+        errors.push({ path: `${p}/runs`, message: 'flaky requires runs that disagree' });
+      }
+      if (r.verdict === 'nocover' && r.ranTests) errors.push({ path: `${p}/ranTests`, message: 'nocover means no test exercises the reverted files; it cannot have run tests' });
+      if (r.verdict === 'unverifiable' && !r.reason) errors.push({ path: `${p}/reason`, message: 'unverifiable requires a reason (e.g. revert-did-not-apply, suite-failed-to-load)' });
+    });
+    return errors;
+  },
+
+  gate(doc) {
+    const errors = [];
+    if (doc.evaluated !== doc.covered.length + doc.uncovered.length) errors.push({ path: '/evaluated', message: `evaluated (${doc.evaluated}) must equal covered (${doc.covered.length}) + uncovered (${doc.uncovered.length})` });
+    if (doc.changed !== doc.evaluated + doc.excluded.length) errors.push({ path: '/changed', message: `changed (${doc.changed}) must equal evaluated (${doc.evaluated}) + excluded (${doc.excluded.length})` });
+    if (doc.uncovered.length > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'an uncovered changed file must exit 1; the gate never passes over unclaimed code' });
+    if (doc.uncovered.length === 0 && doc.exitCode === 1 && !(doc.strict && doc.changed > 0 && doc.evaluated === 0)) errors.push({ path: '/exitCode', message: 'exit 1 without uncovered files is only valid under --strict when a non-empty change evaluated nothing' });
+    doc.covered.forEach((c, i) => {
+      if (c.by === 'ignore' && !c.pattern) errors.push({ path: `/covered/${i}/pattern`, message: 'a file covered by an ignore entry must name the pattern that excused it' });
+      if (c.by !== 'ignore' && !c.claimIds?.length) errors.push({ path: `/covered/${i}/claimIds`, message: `a file covered by a ${c.by} must name the claim(s)` });
+    });
+    for (const e of doc.expired) if (!e.expires) errors.push({ path: '/expired', message: `expired entry "${e.pattern}" has no expires instant` });
     return errors;
   },
 

package/spec/schemas/baseline.schema.json

@@ -15,6 +15,14 @@
       "description": "Whether the working tree had uncommitted changes when the evidence behind this baseline was taken.",
       "type": "boolean"
     },
+    "snapshot": {
+      "description": "When the evidence behind this baseline was a working-tree snapshot: that throwaway commit. `head` is then the parent of the commit that will carry the tests; `baseline --restamp` moves `head` once a clean probe reproduces the same fingerprints.",
+      "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+    },
+    "restampedAt": {
+      "description": "Set when `head` was moved by `baseline --restamp` (same fingerprints, clean tree). `createdAt` keeps the original freeze time.",
+      "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime"
+    },
     "fingerprints": {
       "type": "object",
       "propertyNames": { "pattern": "^[a-f0-9]{64}$" },

package/spec/schemas/brief.schema.json

@@ -39,12 +39,22 @@
       "type": "object",
       "required": ["action", "command", "why"],
       "properties": {
-        "action": { "enum": ["scaffold", "probe", "write-test", "baseline", "review-fault-change", "none"] },
+        "action": { "enum": ["scaffold", "claim", "probe", "write-test", "baseline", "review-fault-change", "none"] },
         "command": { "type": "string", "minLength": 1, "maxLength": 1024 },
         "why": { "type": "string", "minLength": 1, "maxLength": 2000 }
       },
       "additionalProperties": false
     },
+    "unclaimed": {
+      "description": "Changed files that carry no claim, when the brief was built with a change reference. Rendered before the findings: an agent writes the claim before more code.",
+      "type": "object",
+      "required": ["ref", "files"],
+      "properties": {
+        "ref": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } }
+      },
+      "additionalProperties": false
+    },
     "items": {
       "type": "array",
       "maxItems": 50,

package/spec/schemas/common.schema.json

@@ -43,7 +43,7 @@
       ]
     },
     "faultClass": {
-      "description": "Taxonomy of the injected change. Required on every fault so calibration (fault class → real-bug predictiveness) can be computed later.",
+      "description": "Taxonomy of the injected change. Required on every fault so calibration (fault class → real-bug predictiveness) can be computed later. `field-dropped` = a field removed from a persisted/transmitted object, allow-list, schema or merge; `argument-swapped` = a call kept with a parameter-derived argument replaced by a trivial value; `element-removed` = a UI element no longer rendered; `handler-dropped` = an event handler no longer wired.",
       "enum": [
         "guard-removed",
         "condition-forced",
@@ -53,6 +53,10 @@
         "call-removed",
         "return-altered",
         "exception-swallowed",
+        "field-dropped",
+        "argument-swapped",
+        "element-removed",
+        "handler-dropped",
         "other"
       ]
     },
@@ -77,6 +81,33 @@
       },
       "additionalProperties": false
     },
+    "uncoveredFile": {
+      "description": "A changed source or test file that no claim covers and no unexpired ignore entry excuses. Shared by the gate and status documents.",
+      "type": "object",
+      "required": ["file", "kind", "suggestion"],
+      "properties": {
+        "file": { "$ref": "#/$defs/repoPath" },
+        "kind": { "enum": ["source", "test"] },
+        "nearestClaimId": {
+          "description": "The claim whose fault files share the longest directory prefix with this file — where to attach, not a verdict.",
+          "$ref": "#/$defs/identifier"
+        },
+        "suggestion": { "type": "string", "minLength": 1, "maxLength": 1024 }
+      },
+      "additionalProperties": false
+    },
+    "ignoreReliance": {
+      "description": "An ignore entry the gate relied on (or would have, had it not expired), with the changed files it matched.",
+      "type": "object",
+      "required": ["pattern", "reason", "files"],
+      "properties": {
+        "pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
+        "reason": { "type": "string", "minLength": 8, "maxLength": 2000 },
+        "expires": { "$ref": "#/$defs/isoDateTime" },
+        "files": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repoPath" } }
+      },
+      "additionalProperties": false
+    },
     "annotation": {
       "description": "Strictly additive. An annotation never changes, suppresses, or drops a finding; it only attaches context to it.",
       "type": "object",