testguard-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +489 -0
  2. package/README.md +287 -34
  3. package/package.json +2 -2
  4. package/spec/GATE-SEMANTICS.md +210 -8
  5. package/spec/README.md +32 -11
  6. package/spec/lib/validate.mjs +125 -14
  7. package/spec/lib/wilson.mjs +112 -0
  8. package/spec/schemas/baseline.schema.json +7 -7
  9. package/spec/schemas/brief.schema.json +12 -12
  10. package/spec/schemas/calibration.schema.json +96 -26
  11. package/spec/schemas/claims.schema.json +129 -25
  12. package/spec/schemas/common.schema.json +127 -29
  13. package/spec/schemas/evidence.schema.json +175 -73
  14. package/spec/schemas/gate.schema.json +12 -12
  15. package/spec/schemas/ignore.schema.json +4 -4
  16. package/spec/schemas/replay.schema.json +10 -10
  17. package/spec/schemas/status.schema.json +25 -25
  18. package/src/cli.mjs +20 -5
  19. package/src/commands/claims.mjs +22 -2
  20. package/src/commands/probe.mjs +24 -4
  21. package/src/commands/replay.mjs +1 -1
  22. package/src/gate/changed.mjs +9 -2
  23. package/src/probe/attribution.mjs +123 -0
  24. package/src/probe/classify.mjs +18 -2
  25. package/src/probe/cost.mjs +205 -0
  26. package/src/probe/discover.mjs +14 -6
  27. package/src/probe/inject.mjs +41 -9
  28. package/src/probe/mocks.mjs +2 -0
  29. package/src/probe/probe.mjs +235 -52
  30. package/src/probe/progress.mjs +87 -0
  31. package/src/probe/pyimports.mjs +287 -0
  32. package/src/probe/rank.mjs +2 -0
  33. package/src/probe/runners/index.mjs +38 -11
  34. package/src/probe/runners/jest.mjs +13 -1
  35. package/src/probe/runners/playwright.mjs +9 -0
  36. package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
  37. package/src/probe/runners/python/_testguard_report.py +141 -0
  38. package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
  39. package/src/probe/runners/python.mjs +316 -0
  40. package/src/probe/runners/shared.mjs +10 -1
  41. package/src/probe/runners/vitest.mjs +13 -1
  42. package/src/replay/replay.mjs +50 -22
  43. package/src/scaffold/producers.python.mjs +209 -0
  44. package/src/scaffold/scaffold.mjs +86 -32
  45. package/src/util/glob.mjs +7 -1
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "$schema": "https://json-schema.org/draft/2020-12/schema",
3
- "$id": "urn:guard-spec:v1:common",
4
- "title": "Guard spec v1 — shared definitions",
5
- "description": "Definitions shared by every Guard-spec format. Referenced as urn:guard-spec:v1:common#/$defs/<name>.",
3
+ "$id": "urn:claimspec:v1:common",
4
+ "title": "claimspec v1 \u2014 shared definitions",
5
+ "description": "Definitions shared by every Guard-spec format. Referenced as urn:claimspec:v1:common#/$defs/<name>.",
6
6
  "$defs": {
7
7
  "sha256": {
8
8
  "type": "string",
@@ -28,7 +28,12 @@
28
28
  "pattern": "^(?![/\\\\])(?!\\.\\.(?:[/\\\\]|$))(?!.*[/\\\\]\\.\\.(?:[/\\\\]|$)).*$"
29
29
  },
30
30
  "severity": {
31
- "enum": ["critical", "high", "medium", "low"]
31
+ "enum": [
32
+ "critical",
33
+ "high",
34
+ "medium",
35
+ "low"
36
+ ]
32
37
  },
33
38
  "verdict": {
34
39
  "description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md). `unverifiable` covers a fault that could not be applied (anchor missing/ambiguous) and a claim whose defenders failed to load (`defenders-failed-to-load`).",
@@ -43,7 +48,7 @@
43
48
  ]
44
49
  },
45
50
  "faultClass": {
46
- "description": "Taxonomy of the injected change. Required on every fault so calibration (fault class → real-bug predictiveness) can be computed later. `field-dropped` = a field removed from a persisted/transmitted object, allow-list, schema or merge; `argument-swapped` = a call kept with a parameter-derived argument replaced by a trivial value; `element-removed` = a UI element no longer rendered; `handler-dropped` = an event handler no longer wired.",
51
+ "description": "Taxonomy of the injected change. Required on every fault so calibration (fault class \u2192 real-bug predictiveness) can be computed later. `field-dropped` = a field removed from a persisted/transmitted object, allow-list, schema or merge; `argument-swapped` = a call kept with a parameter-derived argument replaced by a trivial value; `element-removed` = a UI element no longer rendered; `handler-dropped` = an event handler no longer wired.",
47
52
  "enum": [
48
53
  "guard-removed",
49
54
  "condition-forced",
@@ -63,63 +68,156 @@
63
68
  "provenance": {
64
69
  "description": "Who or what produced an artifact. `human` = authored by a person; `agent` = proposed by an LLM agent; `operator` = emitted by a mechanical mutation operator; `derived` = computed from another artifact (e.g. a bug log).",
65
70
  "type": "object",
66
- "required": ["producer"],
71
+ "required": [
72
+ "producer"
73
+ ],
67
74
  "properties": {
68
- "producer": { "enum": ["human", "agent", "operator", "derived"] },
69
- "by": { "type": "string", "maxLength": 256 },
70
- "reviewedBy": { "type": "string", "maxLength": 256 },
71
- "at": { "$ref": "#/$defs/isoDateTime" }
75
+ "producer": {
76
+ "enum": [
77
+ "human",
78
+ "agent",
79
+ "operator",
80
+ "derived"
81
+ ]
82
+ },
83
+ "by": {
84
+ "type": "string",
85
+ "maxLength": 256
86
+ },
87
+ "reviewedBy": {
88
+ "type": "string",
89
+ "maxLength": 256
90
+ },
91
+ "at": {
92
+ "$ref": "#/$defs/isoDateTime"
93
+ }
72
94
  },
73
95
  "additionalProperties": false
74
96
  },
75
97
  "tool": {
76
98
  "type": "object",
77
- "required": ["name", "version"],
99
+ "required": [
100
+ "name",
101
+ "version"
102
+ ],
78
103
  "properties": {
79
- "name": { "$ref": "#/$defs/identifier" },
80
- "version": { "type": "string", "minLength": 1, "maxLength": 64 }
104
+ "name": {
105
+ "$ref": "#/$defs/identifier"
106
+ },
107
+ "version": {
108
+ "type": "string",
109
+ "minLength": 1,
110
+ "maxLength": 64
111
+ }
81
112
  },
82
113
  "additionalProperties": false
83
114
  },
84
115
  "uncoveredFile": {
85
116
  "description": "A changed source or test file that no claim covers and no unexpired ignore entry excuses. Shared by the gate and status documents.",
86
117
  "type": "object",
87
- "required": ["file", "kind", "suggestion"],
118
+ "required": [
119
+ "file",
120
+ "kind",
121
+ "suggestion"
122
+ ],
88
123
  "properties": {
89
- "file": { "$ref": "#/$defs/repoPath" },
90
- "kind": { "enum": ["source", "test"] },
124
+ "file": {
125
+ "$ref": "#/$defs/repoPath"
126
+ },
127
+ "kind": {
128
+ "enum": [
129
+ "source",
130
+ "test"
131
+ ]
132
+ },
91
133
  "nearestClaimId": {
92
- "description": "The claim whose fault files share the longest directory prefix with this file — where to attach, not a verdict.",
134
+ "description": "The claim whose fault files share the longest directory prefix with this file \u2014 where to attach, not a verdict.",
93
135
  "$ref": "#/$defs/identifier"
94
136
  },
95
- "suggestion": { "type": "string", "minLength": 1, "maxLength": 1024 }
137
+ "suggestion": {
138
+ "type": "string",
139
+ "minLength": 1,
140
+ "maxLength": 1024
141
+ }
96
142
  },
97
143
  "additionalProperties": false
98
144
  },
99
145
  "ignoreReliance": {
100
146
  "description": "An ignore entry the gate relied on (or would have, had it not expired), with the changed files it matched.",
101
147
  "type": "object",
102
- "required": ["pattern", "reason", "files"],
148
+ "required": [
149
+ "pattern",
150
+ "reason",
151
+ "files"
152
+ ],
103
153
  "properties": {
104
- "pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
105
- "reason": { "type": "string", "minLength": 8, "maxLength": 2000 },
106
- "expires": { "$ref": "#/$defs/isoDateTime" },
107
- "files": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repoPath" } }
154
+ "pattern": {
155
+ "type": "string",
156
+ "minLength": 1,
157
+ "maxLength": 512
158
+ },
159
+ "reason": {
160
+ "type": "string",
161
+ "minLength": 8,
162
+ "maxLength": 2000
163
+ },
164
+ "expires": {
165
+ "$ref": "#/$defs/isoDateTime"
166
+ },
167
+ "files": {
168
+ "type": "array",
169
+ "minItems": 1,
170
+ "items": {
171
+ "$ref": "#/$defs/repoPath"
172
+ }
173
+ }
108
174
  },
109
175
  "additionalProperties": false
110
176
  },
111
177
  "annotation": {
112
178
  "description": "Strictly additive. An annotation never changes, suppresses, or drops a finding; it only attaches context to it.",
113
179
  "type": "object",
114
- "required": ["kind", "text"],
180
+ "required": [
181
+ "kind",
182
+ "text"
183
+ ],
115
184
  "properties": {
116
- "kind": { "enum": ["note", "accepted-risk", "false-positive", "tracked", "wontfix"] },
117
- "text": { "type": "string", "minLength": 1, "maxLength": 4000 },
118
- "by": { "type": "string", "maxLength": 256 },
119
- "at": { "$ref": "#/$defs/isoDateTime" },
120
- "ref": { "type": "string", "maxLength": 512 }
185
+ "kind": {
186
+ "enum": [
187
+ "note",
188
+ "accepted-risk",
189
+ "false-positive",
190
+ "tracked",
191
+ "wontfix"
192
+ ]
193
+ },
194
+ "text": {
195
+ "type": "string",
196
+ "minLength": 1,
197
+ "maxLength": 4000
198
+ },
199
+ "by": {
200
+ "type": "string",
201
+ "maxLength": 256
202
+ },
203
+ "at": {
204
+ "$ref": "#/$defs/isoDateTime"
205
+ },
206
+ "ref": {
207
+ "type": "string",
208
+ "maxLength": 512
209
+ }
121
210
  },
122
211
  "additionalProperties": false
212
+ },
213
+ "method": {
214
+ "description": "How a tool tries to falsify a claim. `fault-injection` mutates source and runs the claim's defenders; it is the only method fully specified in v1, and the default when the field is absent, so every document written before this field existed stays valid. `assertion` (a claim checked by reading code, as a documentation guard does) and `scan` (a surface checked by a scanner) are RESERVED: their required fields are deliberately not specified here and will be defined by their first real consumer. Inventing them for an absent tool is the mistake that made this discriminator necessary.",
215
+ "enum": [
216
+ "fault-injection",
217
+ "assertion",
218
+ "scan"
219
+ ],
220
+ "default": "fault-injection"
123
221
  }
124
222
  }
125
223
  }