testguard-cli 0.6.0 → 0.8.0

package/spec/schemas/baseline.schema.json

@@ -1,27 +1,27 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:baseline",
-  "title": "Guard spec v1 — baseline",
+  "$id": "urn:claimspec:v1:baseline",
+  "title": "claimspec v1 — baseline",
   "description": "Frozen fingerprints of findings that existed when the baseline was taken. A finding whose fingerprint appears here is suppressed up to `count` occurrences; everything else is new and gates. Only non-passing verdicts are ever fingerprinted.",
   "type": "object",
   "required": ["schemaVersion", "tool", "createdAt", "fingerprints"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "createdAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
-    "head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "createdAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
+    "head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
     "dirty": {
       "description": "Whether the working tree had uncommitted changes when the evidence behind this baseline was taken.",
       "type": "boolean"
     },
     "snapshot": {
       "description": "When the evidence behind this baseline was a working-tree snapshot: that throwaway commit. `head` is then the parent of the commit that will carry the tests; `baseline --restamp` moves `head` once a clean probe reproduces the same fingerprints.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+      "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
     },
     "restampedAt": {
       "description": "Set when `head` was moved by `baseline --restamp` (same fingerprints, clean tree). `createdAt` keeps the original freeze time.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime"
+      "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
     },
     "fingerprints": {
       "type": "object",

package/spec/schemas/brief.schema.json

@@ -1,16 +1,16 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:brief",
-  "title": "Guard spec v1 — agent brief",
+  "$id": "urn:claimspec:v1:brief",
+  "title": "claimspec v1 — agent brief",
   "description": "What to tell an agent before it writes code. `text` is the rendered block for an additionalContext payload; `items` is the same content structured so other tools can compose briefs. Ranked, capped, and never a single score.",
   "type": "object",
   "required": ["schemaVersion", "tool", "generatedAt", "heading", "summary", "items", "text"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "generatedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
-    "head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "generatedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
+    "head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
     "heading": {
       "description": "The markdown heading that opens `text`, e.g. `## TEST BLINDSPOT CONTEXT`.",
       "type": "string",
@@ -51,7 +51,7 @@
       "required": ["ref", "files"],
       "properties": {
         "ref": { "type": "string", "minLength": 1, "maxLength": 256 },
-        "files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:guard-spec:v1:common#/$defs/uncoveredFile" } }
+        "files": { "type": "array", "minItems": 1, "items": { "$ref": "urn:claimspec:v1:common#/$defs/uncoveredFile" } }
       },
       "additionalProperties": false
     },
@@ -62,13 +62,13 @@
         "type": "object",
         "required": ["fingerprint", "claimId", "statement", "verdict", "severity", "isNew"],
         "properties": {
-          "fingerprint": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" },
-          "claimId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-          "subjectId": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
+          "fingerprint": { "$ref": "urn:claimspec:v1:common#/$defs/sha256" },
+          "claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
+          "subjectId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
           "statement": { "type": "string", "minLength": 1, "maxLength": 2000 },
-          "verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
-          "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
-          "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
+          "verdict": { "$ref": "urn:claimspec:v1:common#/$defs/verdict" },
+          "severity": { "$ref": "urn:claimspec:v1:common#/$defs/severity" },
+          "file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
           "rank": { "type": "number" },
           "hint": {
             "description": "One line the agent can act on, e.g. which assertion shape left the gap.",

package/spec/schemas/calibration.schema.json

@@ -1,54 +1,124 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:calibration",
-  "title": "Guard spec v1 — calibration",
-  "description": "P(finding is real) per bucket, with a confidence interval and the sample size behind it. A label without n is a vibe; this is the format that stops it being one.",
+  "$id": "urn:claimspec:v1:calibration",
+  "title": "claimspec v1 — calibration",
+  "description": "The observed rate of a bucket's outcome — for testguard, the share of real escaped bugs of a fault class that the suite failed to catch, so a high p is bad — with a confidence interval and the sample size behind it. A label without n is a vibe; this is the format that stops it being one. p and ci are recomputable from n, positives, method and confidence, and the validator recomputes them. Everything beyond the primary buckets is optional and additive, as `method` is on claims and evidence: a document that omits `measures` or `source.caveat` stays valid and is UNATTRIBUTED — a consumer may read it and audit it, merge nothing into it, and quote no number from it.",
   "type": "object",
   "required": ["schemaVersion", "tool", "computedAt", "method", "confidence", "bucketBy", "source", "buckets"],
   "properties": {
     "$schema": { "type": "string" },
     "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
-    "computedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+    "tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
+    "computedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
     "method": {
-      "description": "Interval method. `wilson` is the only method defined in v1.",
+      "description": "Interval method. `wilson` is the only method defined in v1; under it the point estimate is the observed proportion positives / n, and a smoothed or shrunk estimate is a different method.",
       "enum": ["wilson"]
     },
     "confidence": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 },
+    "measures": {
+      "description": "What `p` is the probability of — the one field that lets two calibrations be told apart. Registry in GATE-SEMANTICS.md: `escape-missed` (testguard: a real escaped bug of this class was not caught; high is bad) and `finding-real` (websec-validator: a reported finding of this class is a real vulnerability; high is good). Two documents merge only when this, `bucketBy` and `confidence` agree. Absent = unattributed: nothing may be quoted from the document.",
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9-]*$",
+      "maxLength": 64
+    },
     "bucketBy": {
-      "description": "The field findings are bucketed on, e.g. `faultClass` or `severity`.",
-      "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+      "description": "The field findings are bucketed on, e.g. `faultClass` or `severity`. A compound key joins fields with `|` (`attackClass|confidence`), and every bucket key then has the same number of parts.",
+      "$ref": "#/$defs/bucketKey"
+    },
+    "minN": {
+      "description": "The producer's floor. A cell with fewer samples is not quoted on its own; the consumer backs off a tier, then to `fallback`, then to nothing. A consumer may raise this — a gate that blocks a merge wants more certainty than a report — and may never lower it: lowering it resurrects a cell the producer suppressed. Absent = the producer sets no floor and the consumer applies its own.",
+      "type": "integer",
+      "minimum": 0
     },
     "source": {
-      "description": "Where the ground-truth labels came from.",
+      "description": "Where the ground-truth labels came from. `kind` is the mechanism (`tool-oracle`: a tool confirming its own findings, as websec-validator's dynamic probes do). `corpus` names external ground-truth sources — a bug-replay's ground truth is the repository's own history, already named by `ref`, so it is usually absent there. `caveat` is the one sentence a consumer must show beside any quoted number; `limitation` is the longer story; `evidenceStatus` says whether the labels were reviewed. `detail` is the tool's own provenance, deliberately unconstrained, on the `methodDetail` pattern: what the first consumers put there is the evidence for what should become specified.",
       "type": "object",
       "required": ["kind"],
       "properties": {
-        "kind": { "enum": ["bug-replay", "human-label", "mixed"] },
-        "ref": { "type": "string", "maxLength": 512 }
+        "kind": { "enum": ["bug-replay", "human-label", "tool-oracle", "mixed"] },
+        "ref": { "type": "string", "maxLength": 512 },
+        "corpus": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "minLength": 1, "maxLength": 128 }
+        },
+        "caveat": { "type": "string", "minLength": 1, "maxLength": 512 },
+        "limitation": { "type": "string", "minLength": 1, "maxLength": 1024 },
+        "evidenceStatus": { "enum": ["verified", "unverified", "quarantined"] },
+        "detail": { "type": "object", "minProperties": 1, "additionalProperties": true }
       },
       "additionalProperties": false
     },
-    "buckets": {
-      "type": "object",
-      "propertyNames": { "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" },
-      "additionalProperties": {
+    "buckets": { "$ref": "#/$defs/buckets" },
+    "backoff": {
+      "description": "Ordered coarser tiers consulted when a primary cell is below `minN`: websec-validator's `attackClass|confidence` backs off to `confidence`. Each tier is held to the same arithmetic as the primary. Absent = no backoff.",
+      "type": "array",
+      "minItems": 1,
+      "items": {
         "type": "object",
-        "required": ["n", "positives", "p", "ci"],
+        "required": ["bucketBy", "buckets"],
         "properties": {
-          "n": { "type": "integer", "minimum": 0 },
-          "positives": { "type": "integer", "minimum": 0 },
-          "p": { "type": "number", "minimum": 0, "maximum": 1 },
-          "ci": {
-            "type": "array",
-            "minItems": 2,
-            "maxItems": 2,
-            "items": { "type": "number", "minimum": 0, "maximum": 1 }
-          }
+          "bucketBy": { "$ref": "#/$defs/bucketKey" },
+          "buckets": { "$ref": "#/$defs/buckets" }
         },
         "additionalProperties": false
       }
+    },
+    "fallback": {
+      "description": "What remains when no tier has a cell above `minN`: a labelled guess, not a measurement, and the shape says so — it has a `basis` and no `n`, and anything quoted from it carries n = 0, ci = [0, 1] and the basis. It lives in the document so that \"what did it assume when it had no data?\" has one auditable answer instead of one per consumer.",
+      "type": "object",
+      "required": ["basis", "values"],
+      "properties": {
+        "basis": { "const": "uncalibrated-prior" },
+        "values": {
+          "type": "object",
+          "minProperties": 1,
+          "propertyNames": { "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*(\\|[A-Za-z0-9][A-Za-z0-9._-]*)*$" },
+          "additionalProperties": { "type": "number", "minimum": 0, "maximum": 1 }
+        }
+      },
+      "additionalProperties": false
     }
   },
-  "additionalProperties": false
+  "additionalProperties": false,
+  "$defs": {
+    "bucketKey": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*(\\|[A-Za-z0-9][A-Za-z0-9._-]*)*$",
+      "maxLength": 128
+    },
+    "cell": {
+      "type": "object",
+      "required": ["n", "positives", "p", "ci"],
+      "properties": {
+        "n": { "type": "integer", "minimum": 0 },
+        "positives": { "type": "integer", "minimum": 0 },
+        "p": {
+          "description": "positives / n at the document's precision, or null when n = 0: a cell with no trials has no proportion, and 0 would be a fabricated one.",
+          "type": ["number", "null"],
+          "minimum": 0,
+          "maximum": 1
+        },
+        "ci": {
+          "type": "array",
+          "minItems": 2,
+          "maxItems": 2,
+          "items": { "type": "number", "minimum": 0, "maximum": 1 }
+        },
+        "breakdown": {
+          "description": "How n splits by outcome, when the producer can say (testguard: caught / blind / nocover). Values sum to n. Lets a consumer recover a narrower rate — misses among covered code, say — without the producer publishing a second, misleading headline.",
+          "type": "object",
+          "minProperties": 1,
+          "propertyNames": { "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" },
+          "additionalProperties": { "type": "integer", "minimum": 0 }
+        }
+      },
+      "additionalProperties": false
+    },
+    "buckets": {
+      "type": "object",
+      "propertyNames": { "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*(\\|[A-Za-z0-9][A-Za-z0-9._-]*)*$" },
+      "additionalProperties": { "$ref": "#/$defs/cell" }
+    }
+  }
 }

package/spec/schemas/claims.schema.json

@@ -1,27 +1,43 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:claims",
-  "title": "Guard spec v1 — claims file",
+  "$id": "urn:claimspec:v1:claims",
+  "title": "claimspec v1 \u2014 claims file",
   "description": "What must be true, and how to try to falsify it. A claim is a statement the project makes; each fault is one concrete way that statement could be false, expressed as a deterministic source change. This file is code: its `replace` strings execute under the test runner. Treat it with the same review discipline as source.",
   "type": "object",
-  "required": ["schemaVersion", "claims"],
+  "required": [
+    "schemaVersion",
+    "claims"
+  ],
   "properties": {
-    "$schema": { "type": "string" },
-    "schemaVersion": { "const": 1 },
+    "$schema": {
+      "type": "string"
+    },
+    "schemaVersion": {
+      "const": 1
+    },
     "claims": {
       "type": "array",
-      "items": { "$ref": "#/$defs/claim" }
+      "items": {
+        "$ref": "#/$defs/claim"
+      }
     }
   },
   "additionalProperties": false,
   "$defs": {
     "claim": {
       "type": "object",
-      "required": ["id", "statement", "source", "severity", "producedBy", "faults"],
+      "required": [
+        "id",
+        "statement",
+        "source",
+        "severity",
+        "producedBy",
+        "faults"
+      ],
       "properties": {
         "id": {
           "description": "Stable identifier. Renaming it changes every fingerprint derived from it.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
         },
         "statement": {
           "description": "The claim in plain language, as the project states it.",
@@ -32,46 +48,82 @@
         "source": {
           "description": "Where the claim comes from. Spec- and ADR-sourced claims are human-governed; annotation- and comment-sourced claims may have been written by the same agent that wrote the tests, and should be ranked accordingly.",
           "type": "object",
-          "required": ["kind"],
+          "required": [
+            "kind"
+          ],
           "properties": {
-            "kind": { "enum": ["spec", "adr", "annotation", "comment", "manual"] },
-            "ref": { "type": "string", "maxLength": 512 }
+            "kind": {
+              "enum": [
+                "spec",
+                "adr",
+                "annotation",
+                "comment",
+                "manual"
+              ]
+            },
+            "ref": {
+              "type": "string",
+              "maxLength": 512
+            }
           },
           "additionalProperties": false
         },
-        "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
-        "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
+        "severity": {
+          "$ref": "urn:claimspec:v1:common#/$defs/severity"
+        },
+        "producedBy": {
+          "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+        },
         "defendedBy": {
           "description": "Globs (project-relative) for the test files that supposedly defend this claim. Empty or absent means: discover defenders as the test files that import each fault's target file (relative paths and resolved aliases). If nothing resolves, the verdict is `nocover`.",
           "type": "array",
-          "items": { "type": "string", "minLength": 1 },
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
           "uniqueItems": true
         },
         "faults": {
           "type": "array",
           "minItems": 1,
-          "items": { "$ref": "#/$defs/fault" }
+          "items": {
+            "$ref": "#/$defs/fault"
+          }
         },
         "tags": {
           "type": "array",
-          "items": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
+          "items": {
+            "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+          },
           "uniqueItems": true
         }
       },
       "additionalProperties": false
     },
     "fault": {
-      "description": "One concrete way the claim could be false. Applied as an exact-string substitution; the schema is producer-agnostic — a human, an operator, an agent, or a derivation may all emit this shape.",
+      "description": "One concrete way the claim could be false, and how a tool would try to make it so. `method` says which; it defaults to `fault-injection`, whose shape is an exact-string substitution and is unchanged. Producer-agnostic: a human, an operator, an agent or a derivation may all emit this shape, and every artifact says which.",
       "type": "object",
-      "required": ["id", "description", "faultClass", "file", "find", "replace", "producedBy"],
+      "required": [
+        "id",
+        "description",
+        "producedBy"
+      ],
       "properties": {
         "id": {
           "description": "Unique within the claim.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "faultClass": {
+          "$ref": "urn:claimspec:v1:common#/$defs/faultClass"
+        },
+        "file": {
+          "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
         },
-        "description": { "type": "string", "minLength": 1, "maxLength": 2000 },
-        "faultClass": { "$ref": "urn:guard-spec:v1:common#/$defs/faultClass" },
-        "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
         "find": {
           "description": "Exact text to locate in `file`. Must occur exactly `expectHits` times or the fault is `unverifiable`.",
           "type": "string",
@@ -90,14 +142,66 @@
           "default": 1
         },
         "expectHits": {
-          "description": "How many times `find` must occur in `file`. Any other count means the anchor has rotted or is ambiguous → `unverifiable`, never a silent skip.",
+          "description": "How many times `find` must occur in `file`. Any other count means the anchor has rotted or is ambiguous \u2192 `unverifiable`, never a silent skip.",
           "type": "integer",
           "minimum": 1,
           "default": 1
         },
-        "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" }
+        "producedBy": {
+          "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+        },
+        "method": {
+          "$ref": "urn:claimspec:v1:common#/$defs/method"
+        },
+        "methodDetail": {
+          "description": "Data belonging to a method the spec has not specified yet. `assertion` and `scan` are reserved names with no required fields, so a tool adopting one needs somewhere to put the rule it checked and the place it checked \u2014 without which it can declare that a probe exists and nothing about it. Deliberately unconstrained, deliberately namespaced, and **forbidden under `fault-injection`**, whose shape IS specified and must not acquire a junk drawer. What the first real consumer puts here is the evidence for what the specified shape should become; it is a staging area, not a permanent home.",
+          "type": "object",
+          "minProperties": 1,
+          "additionalProperties": true
+        }
       },
-      "additionalProperties": false
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {
+            "anyOf": [
+              {
+                "properties": {
+                  "method": {
+                    "const": "fault-injection"
+                  }
+                },
+                "required": [
+                  "method"
+                ]
+              },
+              {
+                "not": {
+                  "required": [
+                    "method"
+                  ]
+                }
+              }
+            ]
+          },
+          "then": {
+            "$comment": "The default method. Absent `method` means fault-injection, so every claims file written before the field existed is held to exactly the same requirements as before. The property stubs are what ajv's strict mode needs to accept `required` here; the real definitions stay above.",
+            "properties": {
+              "faultClass": true,
+              "file": true,
+              "find": true,
+              "replace": true,
+              "methodDetail": false
+            },
+            "required": [
+              "faultClass",
+              "file",
+              "find",
+              "replace"
+            ]
+          }
+        }
+      ]
     }
   }
 }