testguard-cli 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +628 -0
- package/README.md +536 -42
- package/package.json +2 -2
- package/spec/GATE-SEMANTICS.md +166 -2
- package/spec/README.md +29 -10
- package/spec/lib/validate.mjs +140 -8
- package/spec/schemas/baseline.schema.json +13 -5
- package/spec/schemas/brief.schema.json +12 -12
- package/spec/schemas/calibration.schema.json +5 -5
- package/spec/schemas/claims.schema.json +122 -25
- package/spec/schemas/common.schema.json +131 -29
- package/spec/schemas/evidence.schema.json +476 -69
- package/spec/schemas/gate.schema.json +12 -12
- package/spec/schemas/ignore.schema.json +4 -4
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +258 -56
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +71 -7
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +68 -10
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +13 -7
- package/src/commands/claims.mjs +55 -6
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +35 -7
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +3 -1
- package/src/gate/changed.mjs +9 -2
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +3 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/attribution.mjs +123 -0
- package/src/probe/classify.mjs +18 -2
- package/src/probe/contention.mjs +60 -0
- package/src/probe/cost.mjs +145 -0
- package/src/probe/discover.mjs +20 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/inject.mjs +41 -9
- package/src/probe/mocks.mjs +122 -0
- package/src/probe/probe.mjs +331 -40
- package/src/probe/progress.mjs +87 -0
- package/src/probe/pyimports.mjs +287 -0
- package/src/probe/rank.mjs +75 -15
- package/src/probe/runners/index.mjs +79 -8
- package/src/probe/runners/jest.mjs +15 -3
- package/src/probe/runners/playwright.mjs +142 -0
- package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
- package/src/probe/runners/python/_testguard_report.py +141 -0
- package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
- package/src/probe/runners/python.mjs +316 -0
- package/src/probe/runners/shared.mjs +80 -14
- package/src/probe/runners/vitest.mjs +16 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +310 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/producers.python.mjs +209 -0
- package/src/scaffold/scaffold.mjs +90 -30
- package/src/status/status.mjs +47 -6
- package/src/util/glob.mjs +7 -1
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:gate",
|
|
4
|
+
"title": "claimspec v1 — change gate",
|
|
5
5
|
"description": "Claim coverage of one change. `probe` asks whether the tests defend the claims that exist; `gate` asks whether the files a change touched carry any claim at all. Measured per file against the merge-base of a reference and HEAD (or the working tree). A file is covered when a fault anchors to it, when it is a test file that defends a claim, or when an unexpired ignore entry excuses it — and every reliance on an ignore entry is listed, so a reviewer sees why the gate passed.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": ["schemaVersion", "tool", "generatedAt", "ref", "base", "head", "includeDirty", "strict", "changed", "evaluated", "excluded", "covered", "uncovered", "reliedOn", "expired", "exitCode"],
|
|
8
8
|
"properties": {
|
|
9
9
|
"$schema": { "type": "string" },
|
|
10
10
|
"schemaVersion": { "const": 1 },
|
|
11
|
-
"tool": { "$ref": "urn:
|
|
12
|
-
"generatedAt": { "$ref": "urn:
|
|
11
|
+
"tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
|
|
12
|
+
"generatedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
13
13
|
"ref": { "description": "The reference the change was measured against, as given (a branch, a tag, a sha).", "type": "string", "minLength": 1, "maxLength": 256 },
|
|
14
|
-
"base": { "description": "merge-base(ref, HEAD): the commit the change is relative to.", "$ref": "urn:
|
|
15
|
-
"head": { "$ref": "urn:
|
|
14
|
+
"base": { "description": "merge-base(ref, HEAD): the commit the change is relative to.", "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
15
|
+
"head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
16
16
|
"includeDirty": { "description": "True when the working tree was compared instead of HEAD.", "type": "boolean" },
|
|
17
17
|
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
18
|
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
"type": "object",
|
|
24
24
|
"required": ["file", "by"],
|
|
25
25
|
"properties": {
|
|
26
|
-
"file": { "$ref": "urn:
|
|
26
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
27
27
|
"by": { "description": "`non-source`, `default:<glob>` or `exclude:<glob>`.", "type": "string", "minLength": 1, "maxLength": 600 }
|
|
28
28
|
},
|
|
29
29
|
"additionalProperties": false
|
|
@@ -35,17 +35,17 @@
|
|
|
35
35
|
"type": "object",
|
|
36
36
|
"required": ["file", "by"],
|
|
37
37
|
"properties": {
|
|
38
|
-
"file": { "$ref": "urn:
|
|
38
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
39
39
|
"by": { "enum": ["fault", "defender", "ignore"] },
|
|
40
|
-
"claimIds": { "type": "array", "minItems": 1, "items": { "$ref": "urn:
|
|
40
|
+
"claimIds": { "type": "array", "minItems": 1, "items": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" } },
|
|
41
41
|
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 }
|
|
42
42
|
},
|
|
43
43
|
"additionalProperties": false
|
|
44
44
|
}
|
|
45
45
|
},
|
|
46
|
-
"uncovered": { "type": "array", "items": { "$ref": "urn:
|
|
47
|
-
"reliedOn": { "description": "Unexpired ignore entries that excused at least one changed file. The gate passed *because of* these; a reviewer reads them.", "type": "array", "items": { "$ref": "urn:
|
|
48
|
-
"expired": { "description": "Expired ignore entries that would have matched a changed file. They excuse nothing; the files are reported as uncovered.", "type": "array", "items": { "$ref": "urn:
|
|
46
|
+
"uncovered": { "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/uncoveredFile" } },
|
|
47
|
+
"reliedOn": { "description": "Unexpired ignore entries that excused at least one changed file. The gate passed *because of* these; a reviewer reads them.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance" } },
|
|
48
|
+
"expired": { "description": "Expired ignore entries that would have matched a changed file. They excuse nothing; the files are reported as uncovered.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance" } },
|
|
49
49
|
"exitCode": { "enum": [0, 1] }
|
|
50
50
|
},
|
|
51
51
|
"additionalProperties": false
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:ignore",
|
|
4
|
+
"title": "claimspec v1 — structured ignore",
|
|
5
5
|
"description": "Reviewable scoping. Every entry carries a reason. Tools may also accept a plain gitignore-syntax text file as shorthand for path entries without reasons; this structured form is what an auditor reads.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": ["schemaVersion", "entries"],
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
|
|
22
22
|
"reason": { "type": "string", "minLength": 8, "maxLength": 2000 },
|
|
23
23
|
"by": { "type": "string", "maxLength": 256 },
|
|
24
|
-
"at": { "$ref": "urn:
|
|
24
|
+
"at": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
25
25
|
"expires": {
|
|
26
26
|
"description": "After this instant the entry no longer applies. Tools warn on expired entries.",
|
|
27
|
-
"$ref": "urn:
|
|
27
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
28
28
|
}
|
|
29
29
|
},
|
|
30
30
|
"additionalProperties": false
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "urn:claimspec:v1:replay",
|
|
4
|
+
"title": "claimspec v1 — bug replay",
|
|
5
|
+
"description": "Would this suite have caught the bugs that actually escaped? For each fix commit in a range: revert only the source to its parent, remove the test the fix shipped, run what remains. `blind` means the suite was blind to a bug that really shipped. Unlike an injected fault, a replayed bug is ground truth — a human already confirmed it was a defect — which is what makes this the oracle a fault model is calibrated against.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schemaVersion", "tool", "run", "records"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"$schema": { "type": "string" },
|
|
10
|
+
"schemaVersion": { "const": 1 },
|
|
11
|
+
"tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
|
|
12
|
+
"run": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"required": ["id", "startedAt", "range", "confirmRuns"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"id": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
17
|
+
"startedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
18
|
+
"finishedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
19
|
+
"range": {
|
|
20
|
+
"description": "The commit range searched for fix commits, as given.",
|
|
21
|
+
"type": "string",
|
|
22
|
+
"minLength": 1,
|
|
23
|
+
"maxLength": 256
|
|
24
|
+
},
|
|
25
|
+
"runner": { "type": "object", "required": ["name"], "properties": { "name": { "type": "string" }, "version": { "type": "string" } }, "additionalProperties": false },
|
|
26
|
+
"confirmRuns": { "type": "integer", "minimum": 1 },
|
|
27
|
+
"candidates": {
|
|
28
|
+
"description": "Fix commits found in the range before de-duplication by patch-id.",
|
|
29
|
+
"type": "integer",
|
|
30
|
+
"minimum": 0
|
|
31
|
+
},
|
|
32
|
+
"duplicates": {
|
|
33
|
+
"description": "Commits dropped as the same patch under another sha. A dual-branch topology carries the same fix two or three times; counting it twice corrupts the corpus.",
|
|
34
|
+
"type": "integer",
|
|
35
|
+
"minimum": 0
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"additionalProperties": false
|
|
39
|
+
},
|
|
40
|
+
"records": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"items": {
|
|
43
|
+
"type": "object",
|
|
44
|
+
"required": ["commit", "patchId", "verdict", "source", "removedTests", "runs"],
|
|
45
|
+
"properties": {
|
|
46
|
+
"commit": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
47
|
+
"patchId": {
|
|
48
|
+
"description": "`git patch-id --stable` of the fix: the identity of the CHANGE rather than of the commit. A dual-branch topology carries the same fix under several shas, and counting it twice corrupts the corpus. Hex, 40 characters for a SHA-1 repository and 64 for a SHA-256 one; the commit sha when git could not compute one.",
|
|
49
|
+
"type": "string",
|
|
50
|
+
"pattern": "^[a-f0-9]{40}(?:[a-f0-9]{24})?$"
|
|
51
|
+
},
|
|
52
|
+
"subject": { "description": "The fix commit's subject line, for a human reading the corpus.", "type": "string", "maxLength": 512 },
|
|
53
|
+
"verdict": {
|
|
54
|
+
"description": "`caught` = a remaining test failed by assertion on the reverted source, every run. `blind` = the suite stayed green on known-broken code. `nocover` = no test exercises the reverted files at all, which is worse than blind. `unverifiable` = nothing could be concluded: the revert did not apply, the suite could not load, it timed out, or the commit only ADDS source (`no-prior-version` \u2014 there is no earlier behaviour to have been blind to, so it is not a bug the suite could have caught and it never enters a calibration). `flaky` = the runs disagreed, so nothing can be concluded.",
|
|
55
|
+
"enum": ["caught", "blind", "nocover", "unverifiable", "flaky"]
|
|
56
|
+
},
|
|
57
|
+
"reason": { "type": "string", "maxLength": 256 },
|
|
58
|
+
"source": {
|
|
59
|
+
"description": "Source files reverted to the fix's parent.",
|
|
60
|
+
"type": "array",
|
|
61
|
+
"minItems": 1,
|
|
62
|
+
"items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" }
|
|
63
|
+
},
|
|
64
|
+
"removedTests": {
|
|
65
|
+
"description": "Test files the fix shipped, removed before the run: the fix's own test proves nothing about what the suite knew before it.",
|
|
66
|
+
"type": "array",
|
|
67
|
+
"items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" }
|
|
68
|
+
},
|
|
69
|
+
"ranTests": { "type": "integer", "minimum": 0 },
|
|
70
|
+
"faultClass": {
|
|
71
|
+
"description": "The injected-fault class this real diff most resembles, derived mechanically from the diff shape. This is the join key between ground truth and a fault model: it is what lets P(this fault class is a real bug) be computed.",
|
|
72
|
+
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
73
|
+
},
|
|
74
|
+
"runs": {
|
|
75
|
+
"type": "array",
|
|
76
|
+
"minItems": 1,
|
|
77
|
+
"items": { "$ref": "urn:claimspec:v1:evidence#/$defs/testRun" }
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
"additionalProperties": false
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
"additionalProperties": false
|
|
85
|
+
}
|
|
@@ -1,66 +1,164 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:status",
|
|
4
|
+
"title": "claimspec v1 \u2014 status",
|
|
5
5
|
"description": "The single machine-readable answer to 'where is this project and what should happen next'. Every human rendering (CLI text, agent brief, skill instructions) derives from this document so they cannot disagree. Computed from the claims file, the evidence, the baseline and the current working tree; never from a cached verdict alone.",
|
|
6
6
|
"type": "object",
|
|
7
|
-
"required": [
|
|
7
|
+
"required": [
|
|
8
|
+
"schemaVersion",
|
|
9
|
+
"tool",
|
|
10
|
+
"generatedAt",
|
|
11
|
+
"state",
|
|
12
|
+
"next",
|
|
13
|
+
"counts"
|
|
14
|
+
],
|
|
8
15
|
"properties": {
|
|
9
|
-
"$schema": {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
"
|
|
16
|
+
"$schema": {
|
|
17
|
+
"type": "string"
|
|
18
|
+
},
|
|
19
|
+
"schemaVersion": {
|
|
20
|
+
"const": 1
|
|
21
|
+
},
|
|
22
|
+
"tool": {
|
|
23
|
+
"$ref": "urn:claimspec:v1:common#/$defs/tool"
|
|
24
|
+
},
|
|
25
|
+
"generatedAt": {
|
|
26
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
27
|
+
},
|
|
13
28
|
"state": {
|
|
14
29
|
"description": "no-claims: no claims file. unclaimed-changes: the change measured against a reference touches source files that carry no claim and no excusing ignore entry (only computed when a reference is known; it precedes every evidence state because the claim is written before more code). unprobed: claims but no confirmed evidence. evidence-stale: evidence exists but a claim, fault, target or defender changed since. provisional-only: only sub-N evidence exists. unproven: current evidence has findings not covered by the baseline. clean: every finding is killed or baselined.",
|
|
15
|
-
"enum": [
|
|
30
|
+
"enum": [
|
|
31
|
+
"no-claims",
|
|
32
|
+
"unclaimed-changes",
|
|
33
|
+
"unprobed",
|
|
34
|
+
"evidence-stale",
|
|
35
|
+
"provisional-only",
|
|
36
|
+
"unproven",
|
|
37
|
+
"clean"
|
|
38
|
+
]
|
|
16
39
|
},
|
|
17
40
|
"next": {
|
|
18
41
|
"description": "The one action that moves the project forward. `command` is a shell line when the action is a tool command, or an instruction when it is not (write-test, claim for a test file).",
|
|
19
42
|
"type": "object",
|
|
20
|
-
"required": [
|
|
43
|
+
"required": [
|
|
44
|
+
"action",
|
|
45
|
+
"command",
|
|
46
|
+
"why"
|
|
47
|
+
],
|
|
21
48
|
"properties": {
|
|
22
|
-
"action": {
|
|
23
|
-
|
|
24
|
-
|
|
49
|
+
"action": {
|
|
50
|
+
"enum": [
|
|
51
|
+
"scaffold",
|
|
52
|
+
"claim",
|
|
53
|
+
"probe",
|
|
54
|
+
"write-test",
|
|
55
|
+
"baseline",
|
|
56
|
+
"review-fault-change",
|
|
57
|
+
"none"
|
|
58
|
+
]
|
|
59
|
+
},
|
|
60
|
+
"command": {
|
|
61
|
+
"type": "string",
|
|
62
|
+
"minLength": 1,
|
|
63
|
+
"maxLength": 1024
|
|
64
|
+
},
|
|
65
|
+
"why": {
|
|
66
|
+
"type": "string",
|
|
67
|
+
"minLength": 1,
|
|
68
|
+
"maxLength": 2000
|
|
69
|
+
},
|
|
25
70
|
"file": {
|
|
26
71
|
"description": "For claim (and for scaffold when a change is known): the first uncovered changed file the action is about.",
|
|
27
|
-
"$ref": "urn:
|
|
72
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
28
73
|
},
|
|
29
74
|
"target": {
|
|
30
75
|
"description": "For write-test / review-fault-change: the finding the action is about.",
|
|
31
76
|
"type": "object",
|
|
32
|
-
"required": [
|
|
77
|
+
"required": [
|
|
78
|
+
"claimId",
|
|
79
|
+
"subjectId"
|
|
80
|
+
],
|
|
33
81
|
"properties": {
|
|
34
|
-
"claimId": {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
"
|
|
82
|
+
"claimId": {
|
|
83
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
84
|
+
},
|
|
85
|
+
"subjectId": {
|
|
86
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
87
|
+
},
|
|
88
|
+
"file": {
|
|
89
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
90
|
+
},
|
|
91
|
+
"verdict": {
|
|
92
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
93
|
+
}
|
|
38
94
|
},
|
|
39
95
|
"additionalProperties": false
|
|
40
96
|
}
|
|
41
97
|
},
|
|
42
98
|
"additionalProperties": false
|
|
43
99
|
},
|
|
44
|
-
"provisional": {
|
|
100
|
+
"provisional": {
|
|
101
|
+
"type": "boolean"
|
|
102
|
+
},
|
|
45
103
|
"counts": {
|
|
46
104
|
"type": "object",
|
|
47
|
-
"required": [
|
|
105
|
+
"required": [
|
|
106
|
+
"claims",
|
|
107
|
+
"faults"
|
|
108
|
+
],
|
|
48
109
|
"properties": {
|
|
49
|
-
"claims": {
|
|
50
|
-
|
|
110
|
+
"claims": {
|
|
111
|
+
"type": "integer",
|
|
112
|
+
"minimum": 0
|
|
113
|
+
},
|
|
114
|
+
"faults": {
|
|
115
|
+
"type": "integer",
|
|
116
|
+
"minimum": 0
|
|
117
|
+
},
|
|
51
118
|
"byVerdict": {
|
|
52
119
|
"type": "object",
|
|
53
|
-
"propertyNames": {
|
|
54
|
-
|
|
120
|
+
"propertyNames": {
|
|
121
|
+
"enum": [
|
|
122
|
+
"killed",
|
|
123
|
+
"survived",
|
|
124
|
+
"nocover",
|
|
125
|
+
"unverifiable",
|
|
126
|
+
"timeout",
|
|
127
|
+
"fault-invalid",
|
|
128
|
+
"flaky-defender"
|
|
129
|
+
]
|
|
130
|
+
},
|
|
131
|
+
"additionalProperties": {
|
|
132
|
+
"type": "integer",
|
|
133
|
+
"minimum": 0
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
"new": {
|
|
137
|
+
"type": "integer",
|
|
138
|
+
"minimum": 0
|
|
55
139
|
},
|
|
56
|
-
"
|
|
57
|
-
"
|
|
140
|
+
"killedCoAuthored": { "description": "Kills whose defender was last touched by the same change (or author) as the code it guards — L3 independence. A signal, not a gate.", "type": "integer", "minimum": 0 },
|
|
141
|
+
"flakyDefenderSets": { "description": "Distinct defender sets that failed at least once on unmodified source.", "type": "integer", "minimum": 0 },
|
|
142
|
+
"worstFlakeRate": { "description": "Highest observed failures/runs on unmodified source across all records, 0–1. A number, so \"unstable\" stops being an adjective.", "type": "number", "minimum": 0, "maximum": 1 },
|
|
143
|
+
"baselined": {
|
|
144
|
+
"type": "integer",
|
|
145
|
+
"minimum": 0
|
|
146
|
+
}
|
|
58
147
|
},
|
|
59
148
|
"additionalProperties": false
|
|
60
149
|
},
|
|
61
150
|
"stale": {
|
|
62
151
|
"description": "Why the evidence no longer describes the working tree. Empty when it does.",
|
|
63
152
|
"type": "array",
|
|
153
|
+
"items": {
|
|
154
|
+
"type": "string",
|
|
155
|
+
"minLength": 1,
|
|
156
|
+
"maxLength": 512
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
"notes": {
|
|
160
|
+
"description": "Informational lines that never change `state`: e.g. a baseline frozen from a snapshot whose head predates the commit that carries its tests, with the way to re-stamp it.",
|
|
161
|
+
"type": "array",
|
|
64
162
|
"items": { "type": "string", "minLength": 1, "maxLength": 512 }
|
|
65
163
|
},
|
|
66
164
|
"changedFaults": {
|
|
@@ -68,12 +166,24 @@
|
|
|
68
166
|
"type": "array",
|
|
69
167
|
"items": {
|
|
70
168
|
"type": "object",
|
|
71
|
-
"required": [
|
|
169
|
+
"required": [
|
|
170
|
+
"claimId",
|
|
171
|
+
"subjectId",
|
|
172
|
+
"previousVerdict"
|
|
173
|
+
],
|
|
72
174
|
"properties": {
|
|
73
|
-
"claimId": {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
"
|
|
175
|
+
"claimId": {
|
|
176
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
177
|
+
},
|
|
178
|
+
"subjectId": {
|
|
179
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
180
|
+
},
|
|
181
|
+
"previousVerdict": {
|
|
182
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
183
|
+
},
|
|
184
|
+
"file": {
|
|
185
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
186
|
+
}
|
|
77
187
|
},
|
|
78
188
|
"additionalProperties": false
|
|
79
189
|
}
|
|
@@ -84,17 +194,43 @@
|
|
|
84
194
|
"maxItems": 50,
|
|
85
195
|
"items": {
|
|
86
196
|
"type": "object",
|
|
87
|
-
"required": [
|
|
197
|
+
"required": [
|
|
198
|
+
"fingerprint",
|
|
199
|
+
"claimId",
|
|
200
|
+
"subjectId",
|
|
201
|
+
"verdict",
|
|
202
|
+
"severity",
|
|
203
|
+
"isNew"
|
|
204
|
+
],
|
|
88
205
|
"properties": {
|
|
89
|
-
"fingerprint": {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
"
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
"
|
|
96
|
-
|
|
97
|
-
|
|
206
|
+
"fingerprint": {
|
|
207
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
208
|
+
},
|
|
209
|
+
"claimId": {
|
|
210
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
211
|
+
},
|
|
212
|
+
"subjectId": {
|
|
213
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
214
|
+
},
|
|
215
|
+
"verdict": {
|
|
216
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
217
|
+
},
|
|
218
|
+
"reason": {
|
|
219
|
+
"type": "string",
|
|
220
|
+
"maxLength": 256
|
|
221
|
+
},
|
|
222
|
+
"severity": {
|
|
223
|
+
"$ref": "urn:claimspec:v1:common#/$defs/severity"
|
|
224
|
+
},
|
|
225
|
+
"file": {
|
|
226
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
227
|
+
},
|
|
228
|
+
"rank": {
|
|
229
|
+
"type": "number"
|
|
230
|
+
},
|
|
231
|
+
"isNew": {
|
|
232
|
+
"type": "boolean"
|
|
233
|
+
}
|
|
98
234
|
},
|
|
99
235
|
"additionalProperties": false
|
|
100
236
|
}
|
|
@@ -102,30 +238,96 @@
|
|
|
102
238
|
"changes": {
|
|
103
239
|
"description": "Claim coverage of the change since `ref`, present only when a reference was given (--changed) or detected from CI. Uncovered files come first in every rendering: the claim is written before more code. Same per-file shapes as the gate document.",
|
|
104
240
|
"type": "object",
|
|
105
|
-
"required": [
|
|
241
|
+
"required": [
|
|
242
|
+
"ref",
|
|
243
|
+
"base",
|
|
244
|
+
"changed",
|
|
245
|
+
"evaluated",
|
|
246
|
+
"excluded",
|
|
247
|
+
"uncovered",
|
|
248
|
+
"reliedOn",
|
|
249
|
+
"expired"
|
|
250
|
+
],
|
|
106
251
|
"properties": {
|
|
107
|
-
"ref": {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
"
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
"
|
|
252
|
+
"ref": {
|
|
253
|
+
"type": "string",
|
|
254
|
+
"minLength": 1,
|
|
255
|
+
"maxLength": 256
|
|
256
|
+
},
|
|
257
|
+
"base": {
|
|
258
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
259
|
+
},
|
|
260
|
+
"includeDirty": {
|
|
261
|
+
"type": "boolean"
|
|
262
|
+
},
|
|
263
|
+
"changed": {
|
|
264
|
+
"type": "integer",
|
|
265
|
+
"minimum": 0
|
|
266
|
+
},
|
|
267
|
+
"evaluated": {
|
|
268
|
+
"type": "integer",
|
|
269
|
+
"minimum": 0
|
|
270
|
+
},
|
|
271
|
+
"excluded": {
|
|
272
|
+
"type": "integer",
|
|
273
|
+
"minimum": 0
|
|
274
|
+
},
|
|
275
|
+
"uncovered": {
|
|
276
|
+
"type": "array",
|
|
277
|
+
"items": {
|
|
278
|
+
"$ref": "urn:claimspec:v1:common#/$defs/uncoveredFile"
|
|
279
|
+
}
|
|
280
|
+
},
|
|
281
|
+
"reliedOn": {
|
|
282
|
+
"type": "array",
|
|
283
|
+
"items": {
|
|
284
|
+
"$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance"
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
"expired": {
|
|
288
|
+
"type": "array",
|
|
289
|
+
"items": {
|
|
290
|
+
"$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
116
293
|
},
|
|
117
294
|
"additionalProperties": false
|
|
118
295
|
},
|
|
119
296
|
"paths": {
|
|
120
297
|
"type": "object",
|
|
121
298
|
"properties": {
|
|
122
|
-
"claims": {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
"
|
|
126
|
-
|
|
299
|
+
"claims": {
|
|
300
|
+
"type": "string"
|
|
301
|
+
},
|
|
302
|
+
"ignore": {
|
|
303
|
+
"type": "string"
|
|
304
|
+
},
|
|
305
|
+
"evidence": {
|
|
306
|
+
"type": "string"
|
|
307
|
+
},
|
|
308
|
+
"provisionalEvidence": {
|
|
309
|
+
"type": "string"
|
|
310
|
+
},
|
|
311
|
+
"baseline": {
|
|
312
|
+
"type": "string"
|
|
313
|
+
}
|
|
127
314
|
},
|
|
128
315
|
"additionalProperties": false
|
|
316
|
+
},
|
|
317
|
+
"evidenceSource": {
|
|
318
|
+
"description": "Whether the evidence read was the project's own (`local`) or supplied with --evidence, e.g. fetched from CI (`provided`).",
|
|
319
|
+
"enum": [
|
|
320
|
+
"local",
|
|
321
|
+
"provided"
|
|
322
|
+
]
|
|
323
|
+
},
|
|
324
|
+
"evidenceHead": {
|
|
325
|
+
"description": "The commit the evidence describes (its snapshot when the working tree was probed). Present whenever evidence was read.",
|
|
326
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
327
|
+
},
|
|
328
|
+
"head": {
|
|
329
|
+
"description": "HEAD of the working tree the status was computed in, when resolvable. Differs from evidenceHead when the evidence came from elsewhere.",
|
|
330
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
129
331
|
}
|
|
130
332
|
},
|
|
131
333
|
"additionalProperties": false
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The admission decision. Pure: records in, verdict out.
|
|
3
|
+
*
|
|
4
|
+
* A test is admitted only when every probed fault of its claim is `killed`
|
|
5
|
+
* — Gate A (green on unmodified HEAD, N/N) and Gate B (fails with the fault
|
|
6
|
+
* applied, N/N) are both inside `killed` by construction. Anything else
|
|
7
|
+
* blocks, and the first blocking record in gate order is named so the agent
|
|
8
|
+
* knows which fault to work on.
|
|
9
|
+
*/
|
|
10
|
+
const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender'];
|
|
11
|
+
|
|
12
|
+
export function decideAdmission(records) {
|
|
13
|
+
if (records.length === 0) return { admitted: false, blocking: null };
|
|
14
|
+
const admitted = records.every((r) => r.verdict === 'killed');
|
|
15
|
+
const blocking = admitted ? null : [...records].filter((r) => r.verdict !== 'killed').sort((a, b) => ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict))[0];
|
|
16
|
+
return { admitted, blocking };
|
|
17
|
+
}
|
|
@@ -1,13 +1,42 @@
|
|
|
1
1
|
const SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
2
2
|
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* Freeze every non-passing finding. `killed` is never debt, so it is never
|
|
5
|
+
* fingerprinted. When the evidence came from a working-tree snapshot, the
|
|
6
|
+
* baseline says so (`snapshot`): its `head` is then the parent of the commit
|
|
7
|
+
* that will carry the tests, and `baseline --restamp` moves it once a clean
|
|
8
|
+
* probe of that commit reproduces the same fingerprints.
|
|
9
|
+
*/
|
|
4
10
|
export function buildBaseline(evidence, { createdAt = new Date().toISOString() } = {}) {
|
|
5
11
|
const fingerprints = {};
|
|
6
12
|
for (const r of evidence.records) {
|
|
7
13
|
if (r.verdict === 'killed') continue;
|
|
8
14
|
fingerprints[r.fingerprint] = (fingerprints[r.fingerprint] ?? 0) + 1;
|
|
9
15
|
}
|
|
10
|
-
|
|
16
|
+
const repo = evidence.run.repo;
|
|
17
|
+
return { schemaVersion: 1, tool: evidence.tool, createdAt, head: repo.head, dirty: repo.dirty, ...(repo.snapshot ? { snapshot: repo.snapshot } : {}), fingerprints };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Same fingerprints, same counts. */
|
|
21
|
+
export const sameFingerprints = (a, b) => {
|
|
22
|
+
const ka = Object.keys(a).sort();
|
|
23
|
+
const kb = Object.keys(b).sort();
|
|
24
|
+
return ka.length === kb.length && ka.every((k, i) => k === kb[i] && a[k] === b[k]);
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Re-stamp a baseline's `head`/`dirty` from a later, CLEAN probe that
|
|
29
|
+
* reproduced exactly the same fingerprints. A frozen contract is never
|
|
30
|
+
* silently rewritten: different fingerprints, a dirty tree or a snapshot run
|
|
31
|
+
* are refused with the reason, and the caller re-probes and re-baselines.
|
|
32
|
+
*/
|
|
33
|
+
export function restampBaseline(baseline, evidence, { restampedAt = new Date().toISOString() } = {}) {
|
|
34
|
+
const repo = evidence.run.repo;
|
|
35
|
+
if (repo.snapshot || repo.dirty) return { ok: false, reason: `the evidence is from a ${repo.snapshot ? 'working-tree snapshot' : 'dirty tree'}; re-stamping needs a clean probe of the commit that carries the tests` };
|
|
36
|
+
const current = buildBaseline(evidence).fingerprints;
|
|
37
|
+
if (!sameFingerprints(baseline.fingerprints, current)) return { ok: false, reason: 'the current evidence does not reproduce the baseline\'s fingerprints; re-probe and run `testguard baseline` to freeze the new set instead' };
|
|
38
|
+
const { snapshot, ...rest } = baseline;
|
|
39
|
+
return { ok: true, baseline: { ...rest, head: repo.head, dirty: false, restampedAt } };
|
|
11
40
|
}
|
|
12
41
|
|
|
13
42
|
/**
|