testguard-cli 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +373 -0
- package/README.md +277 -27
- package/package.json +2 -2
- package/spec/GATE-SEMANTICS.md +112 -5
- package/spec/README.md +28 -10
- package/spec/lib/validate.mjs +84 -8
- package/spec/schemas/baseline.schema.json +7 -7
- package/spec/schemas/brief.schema.json +12 -12
- package/spec/schemas/calibration.schema.json +5 -5
- package/spec/schemas/claims.schema.json +122 -25
- package/spec/schemas/common.schema.json +127 -29
- package/spec/schemas/evidence.schema.json +169 -73
- package/spec/schemas/gate.schema.json +12 -12
- package/spec/schemas/ignore.schema.json +4 -4
- package/spec/schemas/replay.schema.json +10 -10
- package/spec/schemas/status.schema.json +25 -25
- package/src/cli.mjs +20 -5
- package/src/commands/claims.mjs +22 -2
- package/src/commands/probe.mjs +24 -4
- package/src/gate/changed.mjs +9 -2
- package/src/probe/attribution.mjs +123 -0
- package/src/probe/classify.mjs +18 -2
- package/src/probe/cost.mjs +145 -0
- package/src/probe/discover.mjs +14 -6
- package/src/probe/inject.mjs +41 -9
- package/src/probe/mocks.mjs +2 -0
- package/src/probe/probe.mjs +235 -52
- package/src/probe/progress.mjs +87 -0
- package/src/probe/pyimports.mjs +287 -0
- package/src/probe/rank.mjs +2 -0
- package/src/probe/runners/index.mjs +38 -11
- package/src/probe/runners/jest.mjs +13 -1
- package/src/probe/runners/playwright.mjs +9 -0
- package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
- package/src/probe/runners/python/_testguard_report.py +141 -0
- package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
- package/src/probe/runners/python.mjs +316 -0
- package/src/probe/runners/shared.mjs +10 -1
- package/src/probe/runners/vitest.mjs +13 -1
- package/src/replay/replay.mjs +3 -2
- package/src/scaffold/producers.python.mjs +209 -0
- package/src/scaffold/scaffold.mjs +86 -32
- package/src/util/glob.mjs +7 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
5
|
-
"description": "Definitions shared by every Guard-spec format. Referenced as urn:
|
|
3
|
+
"$id": "urn:claimspec:v1:common",
|
|
4
|
+
"title": "claimspec v1 \u2014 shared definitions",
|
|
5
|
+
"description": "Definitions shared by every Guard-spec format. Referenced as urn:claimspec:v1:common#/$defs/<name>.",
|
|
6
6
|
"$defs": {
|
|
7
7
|
"sha256": {
|
|
8
8
|
"type": "string",
|
|
@@ -28,7 +28,12 @@
|
|
|
28
28
|
"pattern": "^(?![/\\\\])(?!\\.\\.(?:[/\\\\]|$))(?!.*[/\\\\]\\.\\.(?:[/\\\\]|$)).*$"
|
|
29
29
|
},
|
|
30
30
|
"severity": {
|
|
31
|
-
"enum": [
|
|
31
|
+
"enum": [
|
|
32
|
+
"critical",
|
|
33
|
+
"high",
|
|
34
|
+
"medium",
|
|
35
|
+
"low"
|
|
36
|
+
]
|
|
32
37
|
},
|
|
33
38
|
"verdict": {
|
|
34
39
|
"description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md). `unverifiable` covers a fault that could not be applied (anchor missing/ambiguous) and a claim whose defenders failed to load (`defenders-failed-to-load`).",
|
|
@@ -43,7 +48,7 @@
|
|
|
43
48
|
]
|
|
44
49
|
},
|
|
45
50
|
"faultClass": {
|
|
46
|
-
"description": "Taxonomy of the injected change. Required on every fault so calibration (fault class
|
|
51
|
+
"description": "Taxonomy of the injected change. Required on every fault so calibration (fault class \u2192 real-bug predictiveness) can be computed later. `field-dropped` = a field removed from a persisted/transmitted object, allow-list, schema or merge; `argument-swapped` = a call kept with a parameter-derived argument replaced by a trivial value; `element-removed` = a UI element no longer rendered; `handler-dropped` = an event handler no longer wired.",
|
|
47
52
|
"enum": [
|
|
48
53
|
"guard-removed",
|
|
49
54
|
"condition-forced",
|
|
@@ -63,63 +68,156 @@
|
|
|
63
68
|
"provenance": {
|
|
64
69
|
"description": "Who or what produced an artifact. `human` = authored by a person; `agent` = proposed by an LLM agent; `operator` = emitted by a mechanical mutation operator; `derived` = computed from another artifact (e.g. a bug log).",
|
|
65
70
|
"type": "object",
|
|
66
|
-
"required": [
|
|
71
|
+
"required": [
|
|
72
|
+
"producer"
|
|
73
|
+
],
|
|
67
74
|
"properties": {
|
|
68
|
-
"producer": {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
75
|
+
"producer": {
|
|
76
|
+
"enum": [
|
|
77
|
+
"human",
|
|
78
|
+
"agent",
|
|
79
|
+
"operator",
|
|
80
|
+
"derived"
|
|
81
|
+
]
|
|
82
|
+
},
|
|
83
|
+
"by": {
|
|
84
|
+
"type": "string",
|
|
85
|
+
"maxLength": 256
|
|
86
|
+
},
|
|
87
|
+
"reviewedBy": {
|
|
88
|
+
"type": "string",
|
|
89
|
+
"maxLength": 256
|
|
90
|
+
},
|
|
91
|
+
"at": {
|
|
92
|
+
"$ref": "#/$defs/isoDateTime"
|
|
93
|
+
}
|
|
72
94
|
},
|
|
73
95
|
"additionalProperties": false
|
|
74
96
|
},
|
|
75
97
|
"tool": {
|
|
76
98
|
"type": "object",
|
|
77
|
-
"required": [
|
|
99
|
+
"required": [
|
|
100
|
+
"name",
|
|
101
|
+
"version"
|
|
102
|
+
],
|
|
78
103
|
"properties": {
|
|
79
|
-
"name": {
|
|
80
|
-
|
|
104
|
+
"name": {
|
|
105
|
+
"$ref": "#/$defs/identifier"
|
|
106
|
+
},
|
|
107
|
+
"version": {
|
|
108
|
+
"type": "string",
|
|
109
|
+
"minLength": 1,
|
|
110
|
+
"maxLength": 64
|
|
111
|
+
}
|
|
81
112
|
},
|
|
82
113
|
"additionalProperties": false
|
|
83
114
|
},
|
|
84
115
|
"uncoveredFile": {
|
|
85
116
|
"description": "A changed source or test file that no claim covers and no unexpired ignore entry excuses. Shared by the gate and status documents.",
|
|
86
117
|
"type": "object",
|
|
87
|
-
"required": [
|
|
118
|
+
"required": [
|
|
119
|
+
"file",
|
|
120
|
+
"kind",
|
|
121
|
+
"suggestion"
|
|
122
|
+
],
|
|
88
123
|
"properties": {
|
|
89
|
-
"file": {
|
|
90
|
-
|
|
124
|
+
"file": {
|
|
125
|
+
"$ref": "#/$defs/repoPath"
|
|
126
|
+
},
|
|
127
|
+
"kind": {
|
|
128
|
+
"enum": [
|
|
129
|
+
"source",
|
|
130
|
+
"test"
|
|
131
|
+
]
|
|
132
|
+
},
|
|
91
133
|
"nearestClaimId": {
|
|
92
|
-
"description": "The claim whose fault files share the longest directory prefix with this file
|
|
134
|
+
"description": "The claim whose fault files share the longest directory prefix with this file \u2014 where to attach, not a verdict.",
|
|
93
135
|
"$ref": "#/$defs/identifier"
|
|
94
136
|
},
|
|
95
|
-
"suggestion": {
|
|
137
|
+
"suggestion": {
|
|
138
|
+
"type": "string",
|
|
139
|
+
"minLength": 1,
|
|
140
|
+
"maxLength": 1024
|
|
141
|
+
}
|
|
96
142
|
},
|
|
97
143
|
"additionalProperties": false
|
|
98
144
|
},
|
|
99
145
|
"ignoreReliance": {
|
|
100
146
|
"description": "An ignore entry the gate relied on (or would have, had it not expired), with the changed files it matched.",
|
|
101
147
|
"type": "object",
|
|
102
|
-
"required": [
|
|
148
|
+
"required": [
|
|
149
|
+
"pattern",
|
|
150
|
+
"reason",
|
|
151
|
+
"files"
|
|
152
|
+
],
|
|
103
153
|
"properties": {
|
|
104
|
-
"pattern": {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
154
|
+
"pattern": {
|
|
155
|
+
"type": "string",
|
|
156
|
+
"minLength": 1,
|
|
157
|
+
"maxLength": 512
|
|
158
|
+
},
|
|
159
|
+
"reason": {
|
|
160
|
+
"type": "string",
|
|
161
|
+
"minLength": 8,
|
|
162
|
+
"maxLength": 2000
|
|
163
|
+
},
|
|
164
|
+
"expires": {
|
|
165
|
+
"$ref": "#/$defs/isoDateTime"
|
|
166
|
+
},
|
|
167
|
+
"files": {
|
|
168
|
+
"type": "array",
|
|
169
|
+
"minItems": 1,
|
|
170
|
+
"items": {
|
|
171
|
+
"$ref": "#/$defs/repoPath"
|
|
172
|
+
}
|
|
173
|
+
}
|
|
108
174
|
},
|
|
109
175
|
"additionalProperties": false
|
|
110
176
|
},
|
|
111
177
|
"annotation": {
|
|
112
178
|
"description": "Strictly additive. An annotation never changes, suppresses, or drops a finding; it only attaches context to it.",
|
|
113
179
|
"type": "object",
|
|
114
|
-
"required": [
|
|
180
|
+
"required": [
|
|
181
|
+
"kind",
|
|
182
|
+
"text"
|
|
183
|
+
],
|
|
115
184
|
"properties": {
|
|
116
|
-
"kind": {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
185
|
+
"kind": {
|
|
186
|
+
"enum": [
|
|
187
|
+
"note",
|
|
188
|
+
"accepted-risk",
|
|
189
|
+
"false-positive",
|
|
190
|
+
"tracked",
|
|
191
|
+
"wontfix"
|
|
192
|
+
]
|
|
193
|
+
},
|
|
194
|
+
"text": {
|
|
195
|
+
"type": "string",
|
|
196
|
+
"minLength": 1,
|
|
197
|
+
"maxLength": 4000
|
|
198
|
+
},
|
|
199
|
+
"by": {
|
|
200
|
+
"type": "string",
|
|
201
|
+
"maxLength": 256
|
|
202
|
+
},
|
|
203
|
+
"at": {
|
|
204
|
+
"$ref": "#/$defs/isoDateTime"
|
|
205
|
+
},
|
|
206
|
+
"ref": {
|
|
207
|
+
"type": "string",
|
|
208
|
+
"maxLength": 512
|
|
209
|
+
}
|
|
121
210
|
},
|
|
122
211
|
"additionalProperties": false
|
|
212
|
+
},
|
|
213
|
+
"method": {
|
|
214
|
+
"description": "How a tool tries to falsify a claim. `fault-injection` mutates source and runs the claim's defenders; it is the only method fully specified in v1, and the default when the field is absent, so every document written before this field existed stays valid. `assertion` (a claim checked by reading code, as a documentation guard does) and `scan` (a surface checked by a scanner) are RESERVED: their required fields are deliberately not specified here and will be defined by their first real consumer. Inventing them for an absent tool is the mistake that made this discriminator necessary.",
|
|
215
|
+
"enum": [
|
|
216
|
+
"fault-injection",
|
|
217
|
+
"assertion",
|
|
218
|
+
"scan"
|
|
219
|
+
],
|
|
220
|
+
"default": "fault-injection"
|
|
123
221
|
}
|
|
124
222
|
}
|
|
125
223
|
}
|