testguard-cli 0.6.0 → 0.7.0

package/spec/lib/validate.mjs

@@ -7,6 +7,13 @@ import { fingerprint } from './fingerprint.mjs';
 const schemaDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'schemas');
 
 export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status', 'gate', 'replay']);
+/**
+ * How a document says it tried to falsify its claims. Absent means
+ * `fault-injection`, so every document written before the field existed is
+ * held to exactly the rules it was written against.
+ */
+export const methodOf = (x) => x?.method ?? 'fault-injection';
+const isInjection = (x) => methodOf(x) === 'fault-injection';
 export const PASSING_VERDICTS = Object.freeze(new Set(['killed']));
 
 const ajv = new Ajv2020({ strict: true, allErrors: true });
@@ -15,7 +22,7 @@ for (const f of readdirSync(schemaDir).filter((n) => n.endsWith('.schema.json'))
 }
 
 const schemaFor = (kind) => {
-  const v = ajv.getSchema(`urn:guard-spec:v1:${kind}`);
+  const v = ajv.getSchema(`urn:claimspec:v1:${kind}`);
   if (!v) throw new RangeError(`unknown Guard-spec kind: ${kind}`);
   return v;
 };
@@ -33,10 +40,15 @@ const semantic = {
         const p = `/claims/${ci}/faults/${fi}`;
         if (seenFault.has(f.id)) errors.push({ path: `${p}/id`, message: `duplicate fault id "${f.id}" in claim "${c.id}"` });
         seenFault.add(f.id);
-        const occ = f.occurrence ?? 1;
-        const hits = f.expectHits ?? 1;
-        if (occ > hits) errors.push({ path: `${p}/occurrence`, message: `occurrence ${occ} exceeds expectHits ${hits}` });
-        if (f.find === f.replace) errors.push({ path: `${p}/replace`, message: 'replace is identical to find; the fault is a no-op' });
+        // Anchor arithmetic is a property of fault injection. A probe that does
+        // not substitute text has no `find` to compare, and `undefined ===
+        // undefined` would report every one of them as a no-op.
+        if (isInjection(f)) {
+          const occ = f.occurrence ?? 1;
+          const hits = f.expectHits ?? 1;
+          if (occ > hits) errors.push({ path: `${p}/occurrence`, message: `occurrence ${occ} exceeds expectHits ${hits}` });
+          if (f.find === f.replace) errors.push({ path: `${p}/replace`, message: 'replace is identical to find; the fault is a no-op' });
+        }
       });
     });
     return errors;
@@ -44,9 +56,15 @@ const semantic = {
 
   evidence(doc) {
     const errors = [];
+    const injection = isInjection(doc.run);
     const n = doc.run.confirmRuns;
-    if (n < 3 && doc.run.provisional !== true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is below 3; the run must declare provisional: true` });
-    if (n >= 3 && doc.run.provisional === true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is confirmed; provisional must be absent or false` });
+    // N-run agreement is what fault injection means by "confirmed". A method
+    // that does not re-run anything has no N, and a rule about one would be a
+    // rule about a number that is not there.
+    if (injection) {
+      if (n < 3 && doc.run.provisional !== true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is below 3; the run must declare provisional: true` });
+      if (n >= 3 && doc.run.provisional === true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is confirmed; provisional must be absent or false` });
+    }
     if (doc.run.runners && doc.run.runner && !doc.run.runners.some((x) => x.name === doc.run.runner.name)) errors.push({ path: '/run/runners', message: 'runners must include the project runner named in run.runner' });
     if (doc.run.repo.ignoredDirty && doc.run.repo.snapshot) errors.push({ path: '/run/repo/ignoredDirty', message: 'a working-tree snapshot has no ignored dirty files: the tree was probed as it is' });
     doc.records.forEach((r, i) => {
@@ -54,6 +72,23 @@ const semantic = {
       const expected = fingerprint({ claimId: r.claim.id, subjectId: r.subject.id, file: r.subject.file ?? '', verdict: r.verdict });
       if (r.fingerprint !== expected) errors.push({ path: `${p}/fingerprint`, message: `fingerprint does not match spec derivation (expected ${expected})` });
 
+      // What the schema no longer requires, the validator still does — for the
+      // one method that means it. Fault injection without defenders, inputs or
+      // runs is not a leaner document, it is a verdict with nothing behind it;
+      // relaxing the schema was to let a scanner conform, never to let an
+      // injecting tool stop showing its work.
+      if (injection) {
+        for (const k of ['defenders', 'inputs']) {
+          if (!r[k]) errors.push({ path: `${p}/${k}`, message: `fault-injection requires ${k} on every record` });
+        }
+        for (const k of ['baselineRuns', 'probeRuns']) {
+          if (!Array.isArray(r.detail[k])) errors.push({ path: `${p}/detail/${k}`, message: `fault-injection requires detail.${k} on every record` });
+        }
+      }
+      // Every rule below reads those fields, so a record that is already
+      // malformed must not also throw on the way to being reported.
+      if (!injection || !r.defenders || !r.inputs || !Array.isArray(r.detail.baselineRuns) || !Array.isArray(r.detail.probeRuns)) return;
+
       const { baselineRuns, probeRuns } = r.detail;
       if (r.verdict === 'killed' || r.verdict === 'survived') {
         if (baselineRuns.length !== n) errors.push({ path: `${p}/detail/baselineRuns`, message: `${r.verdict} requires exactly confirmRuns (${n}) baseline runs, got ${baselineRuns.length}` });
@@ -99,11 +134,52 @@ const semantic = {
       }
       for (const s of r.defenders.signals ?? []) {
         if (s.signal === 'unasserted-annotated' && !s.reason) errors.push({ path: `${p}/defenders/signals`, message: `unasserted-annotated on ${s.file} requires the annotation's reason` });
-        if (!(r.defenders.mocking ?? []).includes(s.file)) errors.push({ path: `${p}/defenders/signals`, message: `${s.file} carries a mock signal but is not listed in defenders.mocking` });
+        // An attribute patch (Python) leaves the file a defender, so it belongs
+        // to `resolved` and not to `mocking`; every other signal is about a file
+        // that replaced the module and must be listed as mocking it.
+        if (s.signal === 'target-attribute-patched') {
+          if (!s.reason) errors.push({ path: `${p}/defenders/signals`, message: `target-attribute-patched on ${s.file} requires the patched attributes as its reason` });
+          if (!r.defenders.resolved.includes(s.file)) errors.push({ path: `${p}/defenders/signals`, message: `${s.file} patches attributes of the subject but is not a resolved defender` });
+        } else if (!(r.defenders.mocking ?? []).includes(s.file)) {
+          errors.push({ path: `${p}/defenders/signals`, message: `${s.file} carries a mock signal but is not listed in defenders.mocking` });
+        }
+      }
+      // A module the defenders never imported cannot have been killed by them.
+      if (r.detail.targetNotImported && r.verdict === 'killed') {
+        errors.push({ path: `${p}/detail/targetNotImported`, message: 'the subject was never imported, so the defenders cannot have killed the fault' });
+      }
+      // The control is only ever charged on a would-be survivor, so a `reached`
+      // record must be one, and `not-reached` must have become unverifiable.
+      if (r.detail.negativeControl === 'reached' && r.verdict !== 'survived') {
+        errors.push({ path: `${p}/detail/negativeControl`, message: `negativeControl "reached" belongs to a survivor, not to ${r.verdict}` });
+      }
+      if (r.detail.negativeControl === 'not-reached' && !(r.verdict === 'unverifiable' && r.detail.reason === 'subject-not-executed')) {
+        errors.push({ path: `${p}/detail/negativeControl`, message: 'negativeControl "not-reached" requires verdict unverifiable with reason subject-not-executed' });
+      }
+      if (r.detail.reason === 'subject-not-executed' && r.detail.negativeControl !== 'not-reached') {
+        errors.push({ path: `${p}/detail/reason`, message: 'subject-not-executed requires the negativeControl that established it' });
+      }
+      // Python's import provenance and the general control answer the same
+      // question. A document where they disagree describes a run that cannot
+      // have happened.
+      if (r.detail.targetNotImported && r.detail.negativeControl === 'reached') {
+        errors.push({ path: `${p}/detail/negativeControl`, message: 'the subject was never imported, so the negative control cannot have reached it' });
       }
       if (r.verdict === 'unverifiable' && !r.detail.reason) {
         errors.push({ path: `${p}/detail/reason`, message: 'unverifiable requires a reason (e.g. anchor-missing, anchor-ambiguous)' });
       }
+      // `probe-error` is the one reason that names nothing on its own: it says
+      // the run threw, not what threw. Without the message the record is a
+      // dead end for anyone reading the document instead of rerunning it.
+      if (r.detail.reason === 'probe-error' && !r.detail.message) {
+        errors.push({ path: `${p}/detail/message`, message: 'probe-error requires a message naming what threw' });
+      }
+      // In-place is the mode where a vanished target is the user's own file
+      // gone; it raises rather than records. A document claiming otherwise
+      // describes a run that cannot have happened.
+      if (r.detail.restoreSkipped && doc.run.mode === 'in-place') {
+        errors.push({ path: `${p}/detail/restoreSkipped`, message: 'restoreSkipped cannot occur in in-place mode, where a missing target is raised' });
+      }
     });
     return errors;
   },

package/spec/schemas/baseline.schema.json

@@ -1,27 +1,27 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:baseline",
-  "title": "Guard spec v1 — baseline",
+  "$id": "urn:claimspec:v1:baseline",
+  "title": "claimspec v1 — baseline",
   "description": "Frozen fingerprints of findings that existed when the baseline was taken. A finding whose fingerprint appears here is suppressed up to `count` occurrences; everything else is new and gates. Only non-passing verdicts are ever fingerprinted.",
   "type": "object",
   "required": ["schemaVersion", "tool", "createdAt", "fingerprints"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "createdAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
-    "head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "createdAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
+    "head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
     "dirty": {
       "description": "Whether the working tree had uncommitted changes when the evidence behind this baseline was taken.",
       "type": "boolean"
     },
     "snapshot": {
       "description": "When the evidence behind this baseline was a working-tree snapshot: that throwaway commit. `head` is then the parent of the commit that will carry the tests; `baseline --restamp` moves `head` once a clean probe reproduces the same fingerprints.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+      "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
     },
     "restampedAt": {
       "description": "Set when `head` was moved by `baseline --restamp` (same fingerprints, clean tree). `createdAt` keeps the original freeze time.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime"
+      "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
     },
     "fingerprints": {
       "type": "object",

package/spec/schemas/brief.schema.json

@@ -1,16 +1,16 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:brief",
-  "title": "Guard spec v1 — agent brief",
+  "$id": "urn:claimspec:v1:brief",
+  "title": "claimspec v1 — agent brief",
   "description": "What to tell an agent before it writes code. `text` is the rendered block for an additionalContext payload; `items` is the same content structured so other tools can compose briefs. Ranked, capped, and never a single score.",
   "type": "object",
   "required": ["schemaVersion", "tool", "generatedAt", "heading", "summary", "items", "text"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
-    "head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "generatedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
+    "head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
     "heading": {
       "description": "The markdown heading that opens `text`, e.g. `## TEST BLINDSPOT CONTEXT`.",
       "type": "string",
@@ -51,7 +51,7 @@
       "required": ["ref", "files"],
       "properties": {
         "ref": { "type": "string", "minLength": 1, "maxLength": 256 },
-        "files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } }
+        "files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:claimspec:v1:common#/$defs/uncoveredFile" } }
       },
       "additionalProperties": false
     },
@@ -62,13 +62,13 @@
         "type": "object",
         "required": ["fingerprint", "claimId", "statement", "verdict", "severity", "isNew"],
         "properties": {
-          "fingerprint": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" },
-          "claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
+          "fingerprint": { "$ref": "urn:claimspec:v1:common#/$defs/sha256" },
+          "claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
+          "subjectId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
           "statement": { "type": "string", "minLength": 1, "maxLength": 2000 },
-          "verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
-          "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
-          "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
+          "verdict": { "$ref": "urn:claimspec:v1:common#/$defs/verdict" },
+          "severity": { "$ref": "urn:claimspec:v1:common#/$defs/severity" },
+          "file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
           "rank": { "type": "number" },
           "hint": {
             "description": "One line the agent can act on, e.g. which assertion shape left the gap.",

package/spec/schemas/calibration.schema.json

@@ -1,15 +1,15 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:calibration",
-  "title": "Guard spec v1 — calibration",
+  "$id": "urn:claimspec:v1:calibration",
+  "title": "claimspec v1 — calibration",
   "description": "P(finding is real) per bucket, with a confidence interval and the sample size behind it. A label without n is a vibe; this is the format that stops it being one.",
   "type": "object",
   "required": ["schemaVersion", "tool", "computedAt", "method", "confidence", "bucketBy", "source", "buckets"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "computedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "computedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
     "method": {
       "description": "Interval method. `wilson` is the only method defined in v1.",
       "enum": ["wilson"]
@@ -17,7 +17,7 @@
     "confidence": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 },
     "bucketBy": {
       "description": "The field findings are bucketed on, e.g. `faultClass` or `severity`.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+      "$ref": "urn:claimspec:v1:common#/$defs/identifier"
     },
     "source": {
       "description": "Where the ground-truth labels came from.",

package/spec/schemas/claims.schema.json

@@ -1,27 +1,43 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:claims",
-  "title": "Guard spec v1 — claims file",
+  "$id": "urn:claimspec:v1:claims",
+  "title": "claimspec v1 \u2014 claims file",
   "description": "What must be true, and how to try to falsify it. A claim is a statement the project makes; each fault is one concrete way that statement could be false, expressed as a deterministic source change. This file is code: its `replace` strings execute under the test runner. Treat it with the same review discipline as source.",
   "type": "object",
-  "required": ["schemaVersion", "claims"],
+  "required": [
+    "schemaVersion",
+    "claims"
+  ],
   "properties": {
-    "$schema": { "type": "string" },
-    "schemaVersion": { "const": 1 },
+    "$schema": {
+      "type": "string"
+    },
+    "schemaVersion": {
+      "const": 1
+    },
     "claims": {
       "type": "array",
-      "items": { "$ref": "#/$defs/claim" }
+      "items": {
+        "$ref": "#/$defs/claim"
+      }
     }
   },
   "additionalProperties": false,
   "$defs": {
     "claim": {
       "type": "object",
-      "required": ["id", "statement", "source", "severity", "producedBy", "faults"],
+      "required": [
+        "id",
+        "statement",
+        "source",
+        "severity",
+        "producedBy",
+        "faults"
+      ],
       "properties": {
         "id": {
           "description": "Stable identifier. Renaming it changes every fingerprint derived from it.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
         },
         "statement": {
           "description": "The claim in plain language, as the project states it.",
@@ -32,46 +48,82 @@
         "source": {
           "description": "Where the claim comes from. Spec- and ADR-sourced claims are human-governed; annotation- and comment-sourced claims may have been written by the same agent that wrote the tests, and should be ranked accordingly.",
           "type": "object",
-          "required": ["kind"],
+          "required": [
+            "kind"
+          ],
           "properties": {
-            "kind": { "enum": ["spec", "adr", "annotation", "comment", "manual"] },
-            "ref": { "type": "string", "maxLength": 512 }
+            "kind": {
+              "enum": [
+                "spec",
+                "adr",
+                "annotation",
+                "comment",
+                "manual"
+              ]
+            },
+            "ref": {
+              "type": "string",
+              "maxLength": 512
+            }
           },
           "additionalProperties": false
         },
-        "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
-        "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
+        "severity": {
+          "$ref": "urn:claimspec:v1:common#/$defs/severity"
+        },
+        "producedBy": {
+          "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+        },
         "defendedBy": {
           "description": "Globs (project-relative) for the test files that supposedly defend this claim. Empty or absent means: discover defenders as the test files that import each fault's target file (relative paths and resolved aliases). If nothing resolves, the verdict is `nocover`.",
           "type": "array",
-          "items": { "type": "string", "minLength": 1 },
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
           "uniqueItems": true
         },
         "faults": {
           "type": "array",
           "minItems": 1,
-          "items": { "$ref": "#/$defs/fault" }
+          "items": {
+            "$ref": "#/$defs/fault"
+          }
         },
         "tags": {
           "type": "array",
-          "items": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
+          "items": {
+            "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+          },
           "uniqueItems": true
         }
       },
       "additionalProperties": false
     },
     "fault": {
-      "description": "One concrete way the claim could be false. Applied as an exact-string substitution; the schema is producer-agnostic — a human, an operator, an agent, or a derivation may all emit this shape.",
+      "description": "One concrete way the claim could be false, and how a tool would try to make it so. `method` says which; it defaults to `fault-injection`, whose shape is an exact-string substitution and is unchanged. Producer-agnostic: a human, an operator, an agent or a derivation may all emit this shape, and every artifact says which.",
       "type": "object",
-      "required": ["id", "description", "faultClass", "file", "find", "replace", "producedBy"],
+      "required": [
+        "id",
+        "description",
+        "producedBy"
+      ],
       "properties": {
         "id": {
           "description": "Unique within the claim.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "faultClass": {
+          "$ref": "urn:claimspec:v1:common#/$defs/faultClass"
+        },
+        "file": {
+          "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
         },
-        "description": { "type": "string", "minLength": 1, "maxLength": 2000 },
-        "faultClass": { "$ref": "urn:guard-spec:v1:common#/$defs/faultClass" },
-        "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
         "find": {
           "description": "Exact text to locate in `file`. Must occur exactly `expectHits` times or the fault is `unverifiable`.",
           "type": "string",
@@ -90,14 +142,59 @@
           "default": 1
         },
         "expectHits": {
-          "description": "How many times `find` must occur in `file`. Any other count means the anchor has rotted or is ambiguous → `unverifiable`, never a silent skip.",
+          "description": "How many times `find` must occur in `file`. Any other count means the anchor has rotted or is ambiguous \u2192 `unverifiable`, never a silent skip.",
           "type": "integer",
           "minimum": 1,
           "default": 1
         },
-        "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" }
+        "producedBy": {
+          "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+        },
+        "method": {
+          "$ref": "urn:claimspec:v1:common#/$defs/method"
+        }
       },
-      "additionalProperties": false
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {
+            "anyOf": [
+              {
+                "properties": {
+                  "method": {
+                    "const": "fault-injection"
+                  }
+                },
+                "required": [
+                  "method"
+                ]
+              },
+              {
+                "not": {
+                  "required": [
+                    "method"
+                  ]
+                }
+              }
+            ]
+          },
+          "then": {
+            "$comment": "The default method. Absent `method` means fault-injection, so every claims file written before the field existed is held to exactly the same requirements as before. The property stubs are what ajv's strict mode needs to accept `required` here; the real definitions stay above.",
+            "properties": {
+              "faultClass": true,
+              "file": true,
+              "find": true,
+              "replace": true
+            },
+            "required": [
+              "faultClass",
+              "file",
+              "find",
+              "replace"
+            ]
+          }
+        }
+      ]
     }
   }
 }