testguard-cli 0.5.0 → 0.7.0

package/spec/schemas/evidence.schema.json

@@ -1,45 +1,136 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "urn:guard-spec:v1:evidence",
-  "title": "Guard spec v1 — evidence file",
+  "$id": "urn:claimspec:v1:evidence",
+  "title": "claimspec v1 \u2014 evidence file",
   "description": "One run's findings, on disk, in a shape an auditor can read without the tool. Each record binds a claim to a subject (for TestGuard, an injected fault) and a verdict, with every run that produced the verdict.",
   "type": "object",
-  "required": ["schemaVersion", "tool", "run", "records"],
+  "required": [
+    "schemaVersion",
+    "tool",
+    "run",
+    "records"
+  ],
   "properties": {
-    "$schema": { "type": "string" },
-    "schemaVersion": { "const": 1 },
-    "tool": { "$ref": "urn:guard-spec:v1:common#/$defs/tool" },
+    "$schema": {
+      "type": "string"
+    },
+    "schemaVersion": {
+      "const": 1
+    },
+    "tool": {
+      "$ref": "urn:claimspec:v1:common#/$defs/tool"
+    },
     "run": {
       "type": "object",
-      "required": ["id", "startedAt", "repo", "confirmRuns", "mode"],
+      "required": [
+        "id",
+        "startedAt",
+        "repo"
+      ],
       "properties": {
-        "id": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-        "startedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
-        "finishedAt": { "$ref": "urn:guard-spec:v1:common#/$defs/isoDateTime" },
+        "id": {
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+        },
+        "startedAt": {
+          "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
+        },
+        "finishedAt": {
+          "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
+        },
         "repo": {
           "type": "object",
-          "required": ["head", "dirty"],
+          "required": [
+            "head",
+            "dirty"
+          ],
           "properties": {
-            "head": { "$ref": "urn:guard-spec:v1:common#/$defs/gitSha" },
-            "dirty": { "type": "boolean" },
+            "head": {
+              "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
+            },
+            "dirty": {
+              "type": "boolean"
+            },
             "snapshot": {
               "description": "When the working tree was probed instead of `head`: the dangling commit that captured it. Not a ref; may be garbage-collected.",
-              "$ref": "urn:guard-spec:v1:common#/$defs/gitSha"
+              "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
+            },
+            "ignoredDirty": {
+              "description": "Defender/target files that had uncommitted changes when `head` was probed as committed (an explicit --ref, or --ignore-dirty). The working-tree versions of these files are NOT what the verdicts describe.",
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+              }
             }
           },
           "additionalProperties": false
         },
         "runner": {
           "type": "object",
-          "required": ["name"],
+          "required": [
+            "name"
+          ],
           "properties": {
-            "name": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "version": { "type": "string", "maxLength": 64 }
+            "name": {
+              "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+            },
+            "version": {
+              "type": "string",
+              "maxLength": 64
+            },
+            "source": {
+              "description": "`project`: the runner package resolved from the project and its own bin script ran. `path`: no such package in the project; an executable of that name on PATH ran instead (a legitimate monorepo setup, but not the project's pinned version).",
+              "enum": [
+                "project",
+                "path"
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
+        "serial": {
+          "description": "True when the defenders were run one test file at a time. Recorded because a serial run is slower and its durations are not comparable with a parallel one.",
+          "type": "boolean"
+        },
+        "contention": {
+          "description": "Other test runners that were already running when this probe started. Best effort, never a verdict: it exists so a reader of a TIMEOUT or FLAKY-DEFENDER can tell the machine was loaded.",
+          "type": "object",
+          "required": [
+            "detected",
+            "runners"
+          ],
+          "properties": {
+            "detected": {
+              "const": true
+            },
+            "runners": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "object",
+                "required": [
+                  "pid",
+                  "command"
+                ],
+                "properties": {
+                  "pid": {
+                    "type": "integer",
+                    "minimum": 1
+                  },
+                  "command": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 200
+                  }
+                },
+                "additionalProperties": false
+              }
+            }
           },
           "additionalProperties": false
         },
         "confirmRuns": {
-          "description": "N — how many times the defenders were run both unmodified (baseline) and with each fault applied.",
+          "description": "N \u2014 how many times the defenders were run both unmodified (baseline) and with each fault applied.",
           "type": "integer",
           "minimum": 1
         },
@@ -49,33 +140,119 @@
         },
         "mode": {
           "description": "`worktree` = faults were applied in a scratch git worktree (default). `in-place` = the user's working tree was mutated and restored.",
-          "enum": ["worktree", "in-place"]
+          "enum": [
+            "worktree",
+            "in-place"
+          ]
+        },
+        "runners": {
+          "description": "Every runner that produced part of this evidence when defenders were split by file (the project runner plus, e.g., Playwright for the files under its testDir). Absent when one runner ran everything.",
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "object",
+            "required": [
+              "name"
+            ],
+            "properties": {
+              "name": {
+                "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+              },
+              "version": {
+                "type": "string",
+                "maxLength": 64
+              }
+            },
+            "additionalProperties": false
+          }
+        },
+        "method": {
+          "$ref": "urn:claimspec:v1:common#/$defs/method"
         }
       },
-      "additionalProperties": false
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {
+            "anyOf": [
+              {
+                "properties": {
+                  "method": {
+                    "const": "fault-injection"
+                  }
+                },
+                "required": [
+                  "method"
+                ]
+              },
+              {
+                "not": {
+                  "required": [
+                    "method"
+                  ]
+                }
+              }
+            ]
+          },
+          "then": {
+            "$comment": "Absent `method` means fault-injection, so every evidence file written before the field existed is held to exactly the same requirements as before. `confirmRuns` and `mode` are what injection means by a confirmed run and by where the mutation happened; a method that mutates nothing has neither.",
+            "properties": {
+              "confirmRuns": true,
+              "mode": true
+            },
+            "required": [
+              "confirmRuns",
+              "mode"
+            ]
+          }
+        }
+      ]
     },
     "records": {
       "type": "array",
-      "items": { "$ref": "#/$defs/record" }
+      "items": {
+        "$ref": "#/$defs/record"
+      }
     }
   },
   "additionalProperties": false,
   "$defs": {
     "testRun": {
       "type": "object",
-      "required": ["outcome", "durationMs"],
+      "required": [
+        "outcome",
+        "durationMs"
+      ],
       "properties": {
         "outcome": {
           "description": "`fail` means at least one test failed with the suite otherwise loading and running. `error` means the suite could not load or run (import/syntax error). `timeout` means the runner exceeded its budget.",
-          "enum": ["pass", "fail", "timeout", "error"]
+          "enum": [
+            "pass",
+            "fail",
+            "timeout",
+            "error"
+          ]
         },
         "tests": {
           "type": "object",
-          "required": ["total", "passed", "failed"],
+          "required": [
+            "total",
+            "passed",
+            "failed"
+          ],
           "properties": {
-            "total": { "type": "integer", "minimum": 0 },
-            "passed": { "type": "integer", "minimum": 0 },
-            "failed": { "type": "integer", "minimum": 0 }
+            "total": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "passed": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "failed": {
+              "type": "integer",
+              "minimum": 0
+            }
           },
           "additionalProperties": false
         },
@@ -84,60 +261,115 @@
           "type": "integer",
           "minimum": 0
         },
-        "durationMs": { "type": "integer", "minimum": 0 }
+        "durationMs": {
+          "type": "integer",
+          "minimum": 0
+        }
       },
       "additionalProperties": false
     },
     "record": {
       "type": "object",
-      "required": ["fingerprint", "claim", "subject", "verdict", "detail", "defenders", "inputs"],
+      "required": [
+        "fingerprint",
+        "claim",
+        "subject",
+        "verdict",
+        "detail"
+      ],
       "properties": {
         "fingerprint": {
           "description": "sha256 of `<claim.id>\\n<subject.id>\\n<subject.file>\\n<verdict>`. Derived from identities and outcome, never from `find`/`replace`, so repairing a rotted anchor does not churn the baseline.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/sha256"
+          "$ref": "urn:claimspec:v1:common#/$defs/sha256"
         },
         "claim": {
           "type": "object",
-          "required": ["id", "statement", "severity", "source"],
+          "required": [
+            "id",
+            "statement",
+            "severity",
+            "source"
+          ],
           "properties": {
-            "id": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "statement": { "type": "string", "minLength": 1, "maxLength": 2000 },
-            "severity": { "$ref": "urn:guard-spec:v1:common#/$defs/severity" },
+            "id": {
+              "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+            },
+            "statement": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 2000
+            },
+            "severity": {
+              "$ref": "urn:claimspec:v1:common#/$defs/severity"
+            },
             "source": {
               "type": "object",
-              "required": ["kind"],
+              "required": [
+                "kind"
+              ],
               "properties": {
-                "kind": { "enum": ["spec", "adr", "annotation", "comment", "manual"] },
-                "ref": { "type": "string", "maxLength": 512 }
+                "kind": {
+                  "enum": [
+                    "spec",
+                    "adr",
+                    "annotation",
+                    "comment",
+                    "manual"
+                  ]
+                },
+                "ref": {
+                  "type": "string",
+                  "maxLength": 512
+                }
               },
               "additionalProperties": false
             },
-            "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" }
+            "producedBy": {
+              "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+            }
           },
           "additionalProperties": false
         },
         "subject": {
           "description": "What was tried against the claim. For TestGuard `kind` is `fault`. Other adopters use their own kinds (e.g. `doc-statement`, `finding`); extra properties are permitted here and nowhere else.",
           "type": "object",
-          "required": ["kind", "id"],
+          "required": [
+            "kind",
+            "id"
+          ],
           "properties": {
-            "kind": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "id": { "$ref": "urn:guard-spec:v1:common#/$defs/identifier" },
-            "description": { "type": "string", "maxLength": 2000 },
-            "file": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" },
-            "faultClass": { "$ref": "urn:guard-spec:v1:common#/$defs/faultClass" },
-            "producedBy": { "$ref": "urn:guard-spec:v1:common#/$defs/provenance" },
+            "kind": {
+              "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+            },
+            "id": {
+              "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+            },
+            "description": {
+              "type": "string",
+              "maxLength": 2000
+            },
+            "file": {
+              "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+            },
+            "faultClass": {
+              "$ref": "urn:claimspec:v1:common#/$defs/faultClass"
+            },
+            "producedBy": {
+              "$ref": "urn:claimspec:v1:common#/$defs/provenance"
+            },
             "contentHash": {
-              "description": "sha256 of the subject's content (for a fault: find + newline + replace). Excluded from the fingerprint on purpose, so repairing an anchor does not churn a baseline — and recorded here on purpose, so weakening a fault to make a finding disappear is visible.",
-              "$ref": "urn:guard-spec:v1:common#/$defs/sha256"
+              "description": "sha256 of the subject's content (for a fault: find + newline + replace). Excluded from the fingerprint on purpose, so repairing an anchor does not churn a baseline \u2014 and recorded here on purpose, so weakening a fault to make a finding disappear is visible.",
+              "$ref": "urn:claimspec:v1:common#/$defs/sha256"
             }
           },
           "additionalProperties": true
         },
-        "verdict": { "$ref": "urn:guard-spec:v1:common#/$defs/verdict" },
+        "verdict": {
+          "$ref": "urn:claimspec:v1:common#/$defs/verdict"
+        },
         "detail": {
           "type": "object",
-          "required": ["baselineRuns", "probeRuns"],
+          "required": [],
           "properties": {
             "reason": {
               "description": "Machine-readable qualifier, e.g. `anchor-missing`, `anchor-ambiguous`, `defenders-not-green`, `replacement-does-not-compile`.",
@@ -147,50 +379,209 @@
             "baselineRuns": {
               "description": "Defenders run on unmodified source, N times. Anything short of N/N pass is `flaky-defender`.",
               "type": "array",
-              "items": { "$ref": "#/$defs/testRun" }
+              "items": {
+                "$ref": "#/$defs/testRun"
+              }
             },
             "probeRuns": {
               "description": "Defenders run with the fault applied, N times.",
               "type": "array",
-              "items": { "$ref": "#/$defs/testRun" }
+              "items": {
+                "$ref": "#/$defs/testRun"
+              }
             },
             "escalated": {
               "description": "True when the fault survived its declared defenders and a broader test set was then run.",
               "type": "boolean"
             },
+            "escalationStoppedEarly": {
+              "description": "Why the escalation loop ran fewer than N times. `no-common-failure`: no test failed in every run so far, so no further run could name an undeclared killer. Stopping can only fail to NAME a killer, never upgrade a verdict.",
+              "enum": [
+                "no-common-failure"
+              ]
+            },
             "escalationRuns": {
               "type": "array",
-              "items": { "$ref": "#/$defs/testRun" }
+              "items": {
+                "$ref": "#/$defs/testRun"
+              }
             },
             "undeclaredKillers": {
               "description": "With reason `killed-by-undeclared-tests`: the tests (as `file::name`) that failed in every escalation run. Names what to add to `defendedBy`.",
               "type": "array",
               "minItems": 1,
-              "items": { "type": "string", "minLength": 1, "maxLength": 1024 }
+              "items": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 1024
+              }
+            },
+            "flakeRate": {
+              "description": "Observed instability of the defenders on UNMODIFIED source: `failures` of `runs` baseline runs failed with the source untouched. Only runs that actually ran are counted (`pass`/`fail`); a load error or a timeout measures nothing about stability and is excluded from both numbers. `failures > 0` means the verdict is `flaky-defender`; the degree is for whoever fixes the defender, and it is the first input to calibration.",
+              "type": "object",
+              "required": [
+                "runs",
+                "failures"
+              ],
+              "properties": {
+                "runs": {
+                  "type": "integer",
+                  "minimum": 1
+                },
+                "failures": {
+                  "type": "integer",
+                  "minimum": 0
+                }
+              },
+              "additionalProperties": false
+            },
+            "independence": {
+              "description": "L3 signal, on `killed` records only: was the killing defender last touched by the same change (or the same author) as the fault's target? A signal for ranking and reporting; it never changes a verdict.",
+              "type": "object",
+              "required": [
+                "class"
+              ],
+              "properties": {
+                "class": {
+                  "enum": [
+                    "co-authored",
+                    "separate-change",
+                    "unknown"
+                  ]
+                },
+                "defenderCommit": {
+                  "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
+                },
+                "targetCommit": {
+                  "$ref": "urn:claimspec:v1:common#/$defs/gitSha"
+                },
+                "sameAuthor": {
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": false
+            },
+            "negativeControl": {
+              "description": "The result of replacing the subject with content its loader cannot parse, and running the defenders once. `reached`: they went red, so they do execute this file and a `survived` verdict is about their assertions. `not-reached`: they stayed green, so they never execute it and the verdict becomes `unverifiable` with reason `subject-not-executed`. Charged only where a false answer is expensive \u2014 on a would-be survivor \u2014 because a kill already proves the defenders reached the code. Absent when the control was not run or the runner cannot say what \"cannot compile\" means for its language.",
+              "enum": [
+                "reached",
+                "not-reached"
+              ]
+            },
+            "targetNotImported": {
+              "description": "True when, with the fault applied, no defender caused the subject's module to be imported at all. Reported by runners that can tell (Python), never inferred. It does not change the verdict, and it is the one thing that makes a `survived` unreadable: the tests were not blind to the fault, they never reached it. A module imported from OUTSIDE the tree being probed is not recorded here \u2014 that run is refused outright, because every verdict in it would be false.",
+              "const": true
+            },
+            "message": {
+              "description": "Human-readable detail for a reason that carries one \u2014 today `probe-error`, whose first line names what threw. Never parsed; a reader should not have to reproduce the run to learn what happened.",
+              "type": "string",
+              "maxLength": 1024
+            },
+            "restoreSkipped": {
+              "description": "The fault was applied but the restore found nothing to put back: the target file no longer existed. In worktree mode that is a disturbed run rather than a failed one \u2014 the mutation only ever existed inside a scratch worktree, and something outside the process deleted it \u2014 but the verdicts in that run were measured against a tree that was being removed underneath them, and a reader has to be able to see that. In `--in-place` mode it never appears: there a missing target is the user's own file gone, and the restore throws.",
+              "enum": [
+                "target-missing"
+              ]
             },
             "anchor": {
               "description": "With an anchor-related reason: how many times `find` occurred versus how many were expected.",
               "type": "object",
-              "required": ["hits", "expected"],
+              "required": [
+                "hits",
+                "expected"
+              ],
               "properties": {
-                "hits": { "type": "integer", "minimum": 0 },
-                "expected": { "type": "integer", "minimum": 1 }
+                "hits": {
+                  "type": "integer",
+                  "minimum": 0
+                },
+                "expected": {
+                  "type": "integer",
+                  "minimum": 1
+                }
               },
               "additionalProperties": false
             }
           },
-          "additionalProperties": false
+          "additionalProperties": false,
+          "description": "Everything behind the verdict. Under `fault-injection` the runs are mandatory and the validator enforces them; a method that runs no tests carries whatever its own evidence is."
         },
         "defenders": {
           "type": "object",
-          "required": ["requested", "resolved", "nocover"],
+          "required": [
+            "requested",
+            "resolved",
+            "nocover"
+          ],
           "properties": {
-            "requested": { "type": "array", "items": { "type": "string" } },
-            "resolved": { "type": "array", "items": { "$ref": "urn:guard-spec:v1:common#/$defs/repoPath" } },
-            "nocover": { "type": "boolean" },
+            "requested": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "resolved": {
+              "type": "array",
+              "items": {
+                "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+              }
+            },
+            "nocover": {
+              "type": "boolean"
+            },
             "discovered": {
               "description": "True when no defenders were declared and `resolved` was found by static import of the subject's file.",
               "type": "boolean"
+            },
+            "mocking": {
+              "description": "Candidate test files that import the subject's file but mock it (`vi.mock`/`jest.mock`). Discovered candidates that mock are excluded from `resolved`; a declared defender that mocks stays in `resolved` and is listed here, because a declared defender that mocks the subject is a broken evidence chain.",
+              "type": "array",
+              "items": {
+                "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+              }
+            },
+            "signals": {
+              "description": "Static, annotation-grade signals about the defenders. They never change a verdict. `mocked-never-asserted`: the file mocks the subject and never asserts on anything imported from it. `unasserted-annotated`: the same, but the author annotated the mock with a reason (`// unasserted: <why>`), recorded here \u2014 silenced, never hidden. `target-attribute-patched`: a Python defender that patches individual attributes of the subject (`patch(\"pkg.mod.fn\")`) rather than the module. Unlike a JavaScript module mock it does NOT disqualify the file \u2014 a fault anywhere else in the module is still detectable \u2014 so the file stays in `resolved`; `reason` names the patched attributes, which are where that test is least likely to notice anything.",
+              "type": "array",
+              "items": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "signal"
+                ],
+                "properties": {
+                  "file": {
+                    "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+                  },
+                  "signal": {
+                    "enum": [
+                      "mocked-never-asserted",
+                      "unasserted-annotated",
+                      "target-attribute-patched"
+                    ]
+                  },
+                  "reason": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 512
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
+            "byRunner": {
+              "description": "When the resolved defenders ran under more than one runner: runner name \u2192 the files it ran. Every file appears in `resolved`. Absent when the project runner ran them all.",
+              "type": "object",
+              "propertyNames": {
+                "$ref": "urn:claimspec:v1:common#/$defs/identifier"
+              },
+              "additionalProperties": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                  "$ref": "urn:claimspec:v1:common#/$defs/repoPath"
+                }
+              }
             }
           },
           "additionalProperties": false
@@ -198,38 +589,54 @@
         "inputs": {
           "description": "Content hashes of everything the verdict depends on, so an unchanged claim can reuse its prior verdict.",
           "type": "object",
-          "required": ["targetHash", "defenderHashes"],
+          "required": [
+            "targetHash",
+            "defenderHashes"
+          ],
           "properties": {
-            "targetHash": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" },
+            "targetHash": {
+              "$ref": "urn:claimspec:v1:common#/$defs/sha256"
+            },
             "defenderHashes": {
               "type": "object",
-              "additionalProperties": { "$ref": "urn:guard-spec:v1:common#/$defs/sha256" }
+              "additionalProperties": {
+                "$ref": "urn:claimspec:v1:common#/$defs/sha256"
+              }
             }
           },
           "additionalProperties": false
         },
         "reusedFrom": {
           "description": "Run id whose verdict was reused because `inputs` were unchanged.",
-          "$ref": "urn:guard-spec:v1:common#/$defs/identifier"
+          "$ref": "urn:claimspec:v1:common#/$defs/identifier"
         },
         "rank": {
           "description": "Additive ordering signal. Never alters `verdict`.",
           "type": "object",
-          "required": ["score"],
+          "required": [
+            "score"
+          ],
           "properties": {
-            "score": { "type": "number" },
+            "score": {
+              "type": "number"
+            },
             "blastRadius": {
               "description": "Number of source files that import the subject's file, transitively or directly as the tool documents.",
               "type": "integer",
               "minimum": 0
             },
-            "tier": { "type": "string", "maxLength": 64 }
+            "tier": {
+              "type": "string",
+              "maxLength": 64
+            }
           },
           "additionalProperties": false
         },
         "annotations": {
           "type": "array",
-          "items": { "$ref": "urn:guard-spec:v1:common#/$defs/annotation" }
+          "items": {
+            "$ref": "urn:claimspec:v1:common#/$defs/annotation"
+          }
         }
       },
       "additionalProperties": false