testguard-cli 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/README.md +141 -4
- package/package.json +1 -1
- package/spec/lib/validate.mjs +49 -1
- package/spec/schemas/concerns.schema.json +41 -0
- package/spec/schemas/sweep.schema.json +288 -49
- package/src/cli.mjs +12 -2
- package/src/commands/concerns.mjs +32 -0
- package/src/commands/sweep.mjs +59 -24
- package/src/supply/concerns.mjs +162 -0
- package/src/supply/feedback.mjs +146 -0
- package/src/supply/persistence.mjs +53 -0
- package/src/supply/savepath.mjs +216 -0
- package/src/sweep/sweep.mjs +111 -7
- package/src/util/glob.mjs +8 -0
|
@@ -4,35 +4,135 @@
|
|
|
4
4
|
"title": "claimspec v1 — sweep",
|
|
5
5
|
"description": "What a sweep proposed and what survived it. A sweep probes faults NOBODY STATED: it takes the changed source files that carry no claim, proposes faults with the mechanical producers, and reports the ones a green suite did not notice. Its findings are therefore weaker evidence than a probe's — no human said the behaviour mattered — and stronger than nothing, which is what a repository with no claims has. The drafts it carries are proposals for a human to keep or drop; they are not claims, and nothing here belongs in a claims file until someone states the sentence it defends.",
|
|
6
6
|
"type": "object",
|
|
7
|
-
"required": [
|
|
7
|
+
"required": [
|
|
8
|
+
"schemaVersion",
|
|
9
|
+
"tool",
|
|
10
|
+
"generatedAt",
|
|
11
|
+
"ref",
|
|
12
|
+
"base",
|
|
13
|
+
"head",
|
|
14
|
+
"includeDirty",
|
|
15
|
+
"scope",
|
|
16
|
+
"selection",
|
|
17
|
+
"counts",
|
|
18
|
+
"findings",
|
|
19
|
+
"exitCode"
|
|
20
|
+
],
|
|
8
21
|
"additionalProperties": false,
|
|
9
22
|
"properties": {
|
|
10
|
-
"$schema": {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
"
|
|
17
|
-
|
|
23
|
+
"$schema": {
|
|
24
|
+
"type": "string"
|
|
25
|
+
},
|
|
26
|
+
"schemaVersion": {
|
|
27
|
+
"const": 1
|
|
28
|
+
},
|
|
29
|
+
"tool": {
|
|
30
|
+
"$ref": "urn:claimspec:v1:common#/$defs/tool"
|
|
31
|
+
},
|
|
32
|
+
"generatedAt": {
|
|
33
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
34
|
+
},
|
|
35
|
+
"ref": {
|
|
36
|
+
"type": "string",
|
|
37
|
+
"description": "The reference the change was measured against."
|
|
38
|
+
},
|
|
39
|
+
"base": {
|
|
40
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
41
|
+
},
|
|
42
|
+
"head": {
|
|
43
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
44
|
+
},
|
|
45
|
+
"includeDirty": {
|
|
46
|
+
"type": "boolean"
|
|
47
|
+
},
|
|
18
48
|
"scope": {
|
|
19
49
|
"type": "object",
|
|
20
|
-
"required": [
|
|
50
|
+
"required": [
|
|
51
|
+
"mode",
|
|
52
|
+
"changed",
|
|
53
|
+
"targets",
|
|
54
|
+
"swept"
|
|
55
|
+
],
|
|
21
56
|
"additionalProperties": false,
|
|
22
57
|
"properties": {
|
|
23
|
-
"
|
|
24
|
-
|
|
25
|
-
|
|
58
|
+
"mode": {
|
|
59
|
+
"enum": [
|
|
60
|
+
"changed",
|
|
61
|
+
"save-paths"
|
|
62
|
+
],
|
|
63
|
+
"description": "What the sweep was pointed at. `changed` proposes for the changed source files that carry no claim. `save-paths` proposes for every file that writes to storage, because a diff cannot answer how many save paths exist — and a clean report over an unstated denominator is a sample of unknown size, not a guarantee."
|
|
64
|
+
},
|
|
65
|
+
"concern": {
|
|
66
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier",
|
|
67
|
+
"description": "The concern this sweep was aimed by, when one was. A concern narrows both the targets and the fault classes, so the same surface swept under two concerns is two different questions."
|
|
68
|
+
},
|
|
69
|
+
"changed": {
|
|
70
|
+
"type": "integer",
|
|
71
|
+
"minimum": 0,
|
|
72
|
+
"description": "Files in the change, before any filter."
|
|
73
|
+
},
|
|
74
|
+
"targets": {
|
|
75
|
+
"type": "integer",
|
|
76
|
+
"minimum": 0,
|
|
77
|
+
"description": "Changed SOURCE files carrying no claim — what the gate calls uncovered. A sweep proposes against these and nothing else."
|
|
78
|
+
},
|
|
79
|
+
"swept": {
|
|
80
|
+
"type": "integer",
|
|
81
|
+
"minimum": 0,
|
|
82
|
+
"description": "Targets that yielded at least one proposal. Never more than `targets`: a file the producers cannot touch is skipped, not swept."
|
|
83
|
+
},
|
|
26
84
|
"skipped": {
|
|
27
85
|
"type": "array",
|
|
28
86
|
"description": "Targets that produced nothing, and why. Recorded rather than dropped: a file that silently yielded no proposal reads as a file with no findings.",
|
|
29
87
|
"items": {
|
|
30
88
|
"type": "object",
|
|
31
|
-
"required": [
|
|
89
|
+
"required": [
|
|
90
|
+
"file",
|
|
91
|
+
"reason"
|
|
92
|
+
],
|
|
32
93
|
"additionalProperties": false,
|
|
33
94
|
"properties": {
|
|
34
|
-
"file": {
|
|
35
|
-
|
|
95
|
+
"file": {
|
|
96
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
97
|
+
},
|
|
98
|
+
"reason": {
|
|
99
|
+
"type": "string",
|
|
100
|
+
"minLength": 1
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
"writeSites": {
|
|
106
|
+
"type": "integer",
|
|
107
|
+
"minimum": 0,
|
|
108
|
+
"description": "save-paths only: calls that write to storage across the whole surface, not only the ones probed."
|
|
109
|
+
},
|
|
110
|
+
"payloadFields": {
|
|
111
|
+
"type": "integer",
|
|
112
|
+
"minimum": 0,
|
|
113
|
+
"description": "save-paths only: payload keys sitting on their own line, which is what a field-dropped fault can anchor to."
|
|
114
|
+
},
|
|
115
|
+
"provability": {
|
|
116
|
+
"type": "object",
|
|
117
|
+
"required": [
|
|
118
|
+
"mocked",
|
|
119
|
+
"unmocked",
|
|
120
|
+
"none"
|
|
121
|
+
],
|
|
122
|
+
"additionalProperties": false,
|
|
123
|
+
"description": "save-paths only: what the defenders of this surface are CAPABLE of proving, before anything is probed. `mocked` — every defender replaces the persistence layer, so it can prove the call shape and never that the row landed. `unmocked` — at least one does not, so a real write is at least possible (not a promise that one happens). `none` — no defender at all. On a surface that is entirely `mocked`, a clean probe is not evidence of persistence; it is evidence that nothing could have measured it.",
|
|
124
|
+
"properties": {
|
|
125
|
+
"mocked": {
|
|
126
|
+
"type": "integer",
|
|
127
|
+
"minimum": 0
|
|
128
|
+
},
|
|
129
|
+
"unmocked": {
|
|
130
|
+
"type": "integer",
|
|
131
|
+
"minimum": 0
|
|
132
|
+
},
|
|
133
|
+
"none": {
|
|
134
|
+
"type": "integer",
|
|
135
|
+
"minimum": 0
|
|
36
136
|
}
|
|
37
137
|
}
|
|
38
138
|
}
|
|
@@ -41,74 +141,183 @@
|
|
|
41
141
|
"selection": {
|
|
42
142
|
"type": "object",
|
|
43
143
|
"description": "What was proposed and what was probed. A sweep is capped — past a few findings per file nobody reads them, and the runner is spent either way — so the count NOT probed is reported beside the count that was. A cap that hides its own remainder is a coverage claim nobody made.",
|
|
44
|
-
"required": [
|
|
144
|
+
"required": [
|
|
145
|
+
"proposed",
|
|
146
|
+
"selected",
|
|
147
|
+
"deferred",
|
|
148
|
+
"cap",
|
|
149
|
+
"byClass"
|
|
150
|
+
],
|
|
45
151
|
"additionalProperties": false,
|
|
46
152
|
"properties": {
|
|
47
|
-
"proposed": {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
153
|
+
"proposed": {
|
|
154
|
+
"type": "integer",
|
|
155
|
+
"minimum": 0
|
|
156
|
+
},
|
|
157
|
+
"selected": {
|
|
158
|
+
"type": "integer",
|
|
159
|
+
"minimum": 0
|
|
160
|
+
},
|
|
161
|
+
"deferred": {
|
|
162
|
+
"type": "integer",
|
|
163
|
+
"minimum": 0,
|
|
164
|
+
"description": "Proposed and not probed. Not a verdict of any kind."
|
|
165
|
+
},
|
|
166
|
+
"cap": {
|
|
167
|
+
"type": "integer",
|
|
168
|
+
"minimum": 1
|
|
169
|
+
},
|
|
51
170
|
"byClass": {
|
|
52
171
|
"type": "object",
|
|
53
172
|
"description": "Per fault class: how many were proposed, how many probed, and how many sat on a write payload.",
|
|
54
173
|
"additionalProperties": {
|
|
55
174
|
"type": "object",
|
|
56
|
-
"required": [
|
|
175
|
+
"required": [
|
|
176
|
+
"proposed",
|
|
177
|
+
"selected",
|
|
178
|
+
"writePath"
|
|
179
|
+
],
|
|
57
180
|
"additionalProperties": false,
|
|
58
181
|
"properties": {
|
|
59
|
-
"proposed": {
|
|
60
|
-
|
|
61
|
-
|
|
182
|
+
"proposed": {
|
|
183
|
+
"type": "integer",
|
|
184
|
+
"minimum": 0
|
|
185
|
+
},
|
|
186
|
+
"selected": {
|
|
187
|
+
"type": "integer",
|
|
188
|
+
"minimum": 0
|
|
189
|
+
},
|
|
190
|
+
"writePath": {
|
|
191
|
+
"type": "integer",
|
|
192
|
+
"minimum": 0
|
|
193
|
+
}
|
|
62
194
|
}
|
|
63
195
|
}
|
|
196
|
+
},
|
|
197
|
+
"outOfScope": {
|
|
198
|
+
"type": "integer",
|
|
199
|
+
"minimum": 0,
|
|
200
|
+
"description": "Proposals the concern's fault classes excluded. Reported so a narrow concern cannot look like a quiet one: the producers found these, and this concern is not about them."
|
|
64
201
|
}
|
|
65
202
|
}
|
|
66
203
|
},
|
|
67
204
|
"counts": {
|
|
68
205
|
"type": "object",
|
|
69
206
|
"description": "Every verdict the probe reached, including `killed`. A sweep whose faults were all caught is a real result and must be readable as one.",
|
|
70
|
-
"additionalProperties": {
|
|
71
|
-
|
|
207
|
+
"additionalProperties": {
|
|
208
|
+
"type": "integer",
|
|
209
|
+
"minimum": 0
|
|
210
|
+
},
|
|
211
|
+
"propertyNames": {
|
|
212
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
213
|
+
}
|
|
72
214
|
},
|
|
73
215
|
"findings": {
|
|
74
216
|
"type": "array",
|
|
75
217
|
"description": "Every record that was not killed, most actionable first. `survived` and `nocover` are statements about the test suite; the rest are statements about the PROPOSAL and do not gate, because a sweep's faults are machine-made and a tool that fails on its own bad guess gets switched off.",
|
|
76
218
|
"items": {
|
|
77
219
|
"type": "object",
|
|
78
|
-
"required": [
|
|
220
|
+
"required": [
|
|
221
|
+
"file",
|
|
222
|
+
"claimId",
|
|
223
|
+
"faultId",
|
|
224
|
+
"faultClass",
|
|
225
|
+
"verdict",
|
|
226
|
+
"description",
|
|
227
|
+
"writePath",
|
|
228
|
+
"hint"
|
|
229
|
+
],
|
|
79
230
|
"additionalProperties": false,
|
|
80
231
|
"properties": {
|
|
81
|
-
"file": {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
"
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
"
|
|
89
|
-
|
|
90
|
-
|
|
232
|
+
"file": {
|
|
233
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
234
|
+
},
|
|
235
|
+
"line": {
|
|
236
|
+
"type": "integer",
|
|
237
|
+
"minimum": 1
|
|
238
|
+
},
|
|
239
|
+
"claimId": {
|
|
240
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
241
|
+
},
|
|
242
|
+
"faultId": {
|
|
243
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
244
|
+
},
|
|
245
|
+
"faultClass": {
|
|
246
|
+
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
247
|
+
},
|
|
248
|
+
"verdict": {
|
|
249
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
250
|
+
},
|
|
251
|
+
"reason": {
|
|
252
|
+
"type": "string",
|
|
253
|
+
"minLength": 1
|
|
254
|
+
},
|
|
255
|
+
"description": {
|
|
256
|
+
"type": "string",
|
|
257
|
+
"minLength": 1
|
|
258
|
+
},
|
|
259
|
+
"writePath": {
|
|
260
|
+
"type": "boolean",
|
|
261
|
+
"description": "The fault sits on a database write or the payload of one. Every surviving fault in the measurement that prompted this field was such a line."
|
|
262
|
+
},
|
|
263
|
+
"hint": {
|
|
264
|
+
"type": "string",
|
|
265
|
+
"minLength": 1,
|
|
266
|
+
"description": "One line naming the mechanism, never only the verdict."
|
|
267
|
+
},
|
|
91
268
|
"signals": {
|
|
92
269
|
"type": "array",
|
|
93
270
|
"minItems": 1,
|
|
94
271
|
"description": "Static, annotation-grade signals about the defenders of a SURVIVOR on the write path. They never change a verdict. `persistence-payload-unasserted`: the defender mocks the persistence layer and the file's call assertions never name an exact payload, so a field dropped from a write cannot fail them. Descriptive, never predictive: no threshold on the assertion mix separates the files whose payload faults survive from the files whose do not.",
|
|
95
272
|
"items": {
|
|
96
273
|
"type": "object",
|
|
97
|
-
"required": [
|
|
274
|
+
"required": [
|
|
275
|
+
"file",
|
|
276
|
+
"signal",
|
|
277
|
+
"reason"
|
|
278
|
+
],
|
|
98
279
|
"additionalProperties": false,
|
|
99
280
|
"properties": {
|
|
100
|
-
"file": {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
"
|
|
281
|
+
"file": {
|
|
282
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
283
|
+
},
|
|
284
|
+
"signal": {
|
|
285
|
+
"enum": [
|
|
286
|
+
"persistence-payload-unasserted"
|
|
287
|
+
]
|
|
288
|
+
},
|
|
289
|
+
"reason": {
|
|
290
|
+
"type": "string",
|
|
291
|
+
"minLength": 1
|
|
292
|
+
},
|
|
293
|
+
"specifiers": {
|
|
294
|
+
"type": "array",
|
|
295
|
+
"items": {
|
|
296
|
+
"type": "string",
|
|
297
|
+
"minLength": 1
|
|
298
|
+
}
|
|
299
|
+
},
|
|
104
300
|
"counts": {
|
|
105
301
|
"type": "object",
|
|
106
|
-
"required": [
|
|
302
|
+
"required": [
|
|
303
|
+
"exact",
|
|
304
|
+
"partial",
|
|
305
|
+
"argless"
|
|
306
|
+
],
|
|
107
307
|
"additionalProperties": false,
|
|
108
308
|
"properties": {
|
|
109
|
-
"exact": {
|
|
110
|
-
|
|
111
|
-
|
|
309
|
+
"exact": {
|
|
310
|
+
"type": "integer",
|
|
311
|
+
"minimum": 0
|
|
312
|
+
},
|
|
313
|
+
"partial": {
|
|
314
|
+
"type": "integer",
|
|
315
|
+
"minimum": 0
|
|
316
|
+
},
|
|
317
|
+
"argless": {
|
|
318
|
+
"type": "integer",
|
|
319
|
+
"minimum": 0
|
|
320
|
+
}
|
|
112
321
|
}
|
|
113
322
|
}
|
|
114
323
|
}
|
|
@@ -117,11 +326,41 @@
|
|
|
117
326
|
}
|
|
118
327
|
}
|
|
119
328
|
},
|
|
120
|
-
"exitCode": {
|
|
329
|
+
"exitCode": {
|
|
330
|
+
"enum": [
|
|
331
|
+
0,
|
|
332
|
+
1
|
|
333
|
+
]
|
|
334
|
+
},
|
|
121
335
|
"drafts": {
|
|
122
336
|
"type": "array",
|
|
123
337
|
"description": "The draft claims documents the proposals came from, carried so a human can keep one without re-running the sweep. Drafts, never claims: every statement is a TODO nobody has written yet.",
|
|
124
|
-
"items": {
|
|
338
|
+
"items": {
|
|
339
|
+
"type": "object"
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
"ordering": {
|
|
343
|
+
"type": "object",
|
|
344
|
+
"required": [
|
|
345
|
+
"observed",
|
|
346
|
+
"sources"
|
|
347
|
+
],
|
|
348
|
+
"additionalProperties": false,
|
|
349
|
+
"description": "Where the candidate ordering came from. `observed` is how many of this project's OWN probed faults informed it; 0 means the shipped productivity prior was used unchanged. An ordering nobody can trace is a number nobody should trust.",
|
|
350
|
+
"properties": {
|
|
351
|
+
"observed": {
|
|
352
|
+
"type": "integer",
|
|
353
|
+
"minimum": 0
|
|
354
|
+
},
|
|
355
|
+
"sources": {
|
|
356
|
+
"type": "array",
|
|
357
|
+
"items": {
|
|
358
|
+
"type": "string",
|
|
359
|
+
"minLength": 1
|
|
360
|
+
},
|
|
361
|
+
"description": "The evidence documents the counts were read from."
|
|
362
|
+
}
|
|
363
|
+
}
|
|
125
364
|
}
|
|
126
365
|
}
|
|
127
366
|
}
|
package/src/cli.mjs
CHANGED
|
@@ -19,6 +19,7 @@ import { replayCommand } from './commands/replay.mjs';
|
|
|
19
19
|
import { mcpCommand } from './commands/mcp.mjs';
|
|
20
20
|
import { admitCommand } from './commands/admit.mjs';
|
|
21
21
|
import { sweepCommand } from './commands/sweep.mjs';
|
|
22
|
+
import { concernsCommand } from './commands/concerns.mjs';
|
|
22
23
|
|
|
23
24
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
24
25
|
const ISSUES = 'https://github.com/raccioly/testguard/issues';
|
|
@@ -34,6 +35,7 @@ export const USAGE = `testguard ${VERSION} — proves a test suite defends the c
|
|
|
34
35
|
testguard gate [dir] fail when a changed source file carries no claim and no excusing ignore entry
|
|
35
36
|
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
36
37
|
testguard sweep [dir] propose faults for the changed source files that carry no claim, probe a bounded selection, and report what nothing noticed
|
|
38
|
+
testguard concerns [dir] the one-sentence scopes a sweep can be aimed by: where to look, and which shapes of break are relevant
|
|
37
39
|
testguard mcp serve the read-only status/brief/claims/evidence tools over MCP on stdio, so the loop works in any agent harness
|
|
38
40
|
testguard replay [dir] would this suite have caught the bugs that already escaped? Replays real fix commits and calibrates fault classes against them
|
|
39
41
|
testguard admit <test> --claim <ID> the two-gate rule as one verb: is this test green on HEAD and does it fail on every fault of the claim?
|
|
@@ -76,7 +78,12 @@ probe
|
|
|
76
78
|
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
77
79
|
|
|
78
80
|
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
79
|
-
sweep --changed <ref> (required; CI bases and a safe local default are detected) --cap <n>
|
|
81
|
+
sweep --changed <ref> (required unless --save-paths; CI bases and a safe local default are detected) --cap <n>
|
|
82
|
+
--concern <ID> aim the sweep by one of this project's concerns: its targets and its fault classes.
|
|
83
|
+
testguard concerns lists them. --save-paths is sugar for the built-in SAVE-PERSISTS
|
|
84
|
+
--save-paths sweep every file that WRITES TO STORAGE instead of the diff, and report the surface as the
|
|
85
|
+
denominator: "120 writes across 38 files, swept 12". A clean sweep over an unstated
|
|
86
|
+
denominator is a sample of unknown size, not a guarantee that saving works
|
|
80
87
|
--include-dirty --exclude <glob> --confirm <n> --budget <ms> --max <n> --out <path> --json
|
|
81
88
|
the cold start: no claim and no concern needed. Everything it proposes is a DRAFT — it never writes testguard.claims.json,
|
|
82
89
|
and its evidence never replaces .testguard/evidence.json. exit 1 on a fault that survived, or a file no test imports
|
|
@@ -123,7 +130,7 @@ brief --evidence <path> --baseline <path> --max <n> --text (print only;
|
|
|
123
130
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or unclaimed changes) · 2 precondition failed · 3 usage
|
|
124
131
|
`;
|
|
125
132
|
|
|
126
|
-
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand };
|
|
133
|
+
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand, concerns: concernsCommand };
|
|
127
134
|
|
|
128
135
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
129
136
|
let parsed;
|
|
@@ -150,6 +157,9 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
150
157
|
since: { type: 'string' },
|
|
151
158
|
exclude: { type: 'string', multiple: true },
|
|
152
159
|
cap: { type: 'string' },
|
|
160
|
+
'save-paths': { type: 'boolean', default: false },
|
|
161
|
+
concern: { type: 'string' },
|
|
162
|
+
concerns: { type: 'string' },
|
|
153
163
|
strict: { type: 'boolean', default: false },
|
|
154
164
|
explain: { type: 'boolean', default: false },
|
|
155
165
|
ignore: { type: 'string' },
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { loadConcerns, renderConcerns } from '../supply/concerns.mjs';
|
|
3
|
+
import { validate } from '../../spec/lib/validate.mjs';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* `testguard concerns [dir]`: the one-sentence scopes a sweep can be aimed by.
|
|
7
|
+
*
|
|
8
|
+
* A concern says WHERE to look and WHICH shapes of break are relevant. It is
|
|
9
|
+
* never a promise about the code — only a claim is that — so this command
|
|
10
|
+
* validates and lists, and has no exit code that means "something is wrong
|
|
11
|
+
* with your project". Exit 2 is a malformed concerns file, which is wrong with
|
|
12
|
+
* the FILE.
|
|
13
|
+
*/
|
|
14
|
+
export async function concernsCommand({ projectDir, values, version }, io) {
|
|
15
|
+
const path = values.concerns ? resolve(values.concerns) : undefined;
|
|
16
|
+
const loaded = loadConcerns(projectDir, { path });
|
|
17
|
+
|
|
18
|
+
if (loaded.source) {
|
|
19
|
+
const result = validate('concerns', { schemaVersion: 1, concerns: loaded.raw });
|
|
20
|
+
if (!result.ok) {
|
|
21
|
+
io.err(`${loaded.source} does not conform to the spec:`);
|
|
22
|
+
for (const e of result.errors) io.err(` ${e.path}: ${e.message}`);
|
|
23
|
+
return 2;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
if (values.json) {
|
|
27
|
+
io.out(JSON.stringify({ schemaVersion: 1, tool: { name: 'testguard', version }, source: loaded.source, shadowed: loaded.shadowed, concerns: loaded.concerns }, null, 2));
|
|
28
|
+
} else {
|
|
29
|
+
io.out(renderConcerns(loaded));
|
|
30
|
+
}
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
package/src/commands/sweep.mjs
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
import { join, resolve } from 'node:path';
|
|
2
|
-
import { sweep, renderSweep } from '../sweep/sweep.mjs';
|
|
2
|
+
import { sweep, renderSweep, ConcernError } from '../sweep/sweep.mjs';
|
|
3
3
|
import { resolveChangedRef } from '../gate/changed.mjs';
|
|
4
4
|
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
5
5
|
|
|
6
6
|
export const sweepPath = (projectDir) => join(projectDir, '.testguard', 'sweep.json');
|
|
7
|
+
/**
|
|
8
|
+
* A sweep's own evidence, beside the canonical one and never replacing it.
|
|
9
|
+
* The next sweep reads it to learn which fault classes are productive HERE,
|
|
10
|
+
* which is the feedback half of the selection Google reports taking from 15%
|
|
11
|
+
* to 89% productive. Machine-proposed faults, so it is the closest observation
|
|
12
|
+
* of the distribution the ranker actually orders.
|
|
13
|
+
*/
|
|
14
|
+
export const sweepEvidencePath = (projectDir) => join(projectDir, '.testguard', 'sweep-evidence.json');
|
|
7
15
|
|
|
8
16
|
/**
|
|
9
17
|
* `testguard sweep [dir] --changed <ref>`: propose faults for the changed
|
|
@@ -20,9 +28,17 @@ export const sweepPath = (projectDir) => join(projectDir, '.testguard', 'sweep.j
|
|
|
20
28
|
* read.
|
|
21
29
|
*/
|
|
22
30
|
export async function sweepCommand({ projectDir, values, version }, io) {
|
|
23
|
-
const
|
|
31
|
+
const mode = values['save-paths'] ? 'save-paths' : 'changed';
|
|
32
|
+
// A concern that aims at the write surface or at a glob needs no diff, the
|
|
33
|
+
// same as --save-paths. Only a concern that defers to the gate does.
|
|
34
|
+
const aimed = Boolean(values.concern) || mode === 'save-paths';
|
|
35
|
+
// `--save-paths` scans the whole write surface, so it needs no diff. The gate
|
|
36
|
+
// is still computed underneath (the document reports `changed`), and HEAD is
|
|
37
|
+
// a reference every repository has.
|
|
38
|
+
const resolved = resolveChangedRef({ explicit: values.changed, projectDir })
|
|
39
|
+
?? (aimed ? { ref: 'HEAD', from: values.concern ? `concern ${values.concern}` : 'save-paths', required: false } : null);
|
|
24
40
|
if (!resolved) {
|
|
25
|
-
io.err('sweep needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. CI bases and a safe local remote default or differently named upstream are detected automatically.');
|
|
41
|
+
io.err('sweep needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. CI bases and a safe local remote default or differently named upstream are detected automatically. Or sweep the write surface instead with --save-paths.');
|
|
26
42
|
return 3;
|
|
27
43
|
}
|
|
28
44
|
const confirmRuns = Number(values.confirm);
|
|
@@ -39,35 +55,54 @@ export async function sweepCommand({ projectDir, values, version }, io) {
|
|
|
39
55
|
return 3;
|
|
40
56
|
}
|
|
41
57
|
}
|
|
42
|
-
if (!resolved.required && !values.json && !values.quiet) io.err(`sweep: comparing against ${resolved.ref} (${resolved.from})`);
|
|
58
|
+
if (mode === 'changed' && !resolved.required && !values.json && !values.quiet) io.err(`sweep: comparing against ${resolved.ref} (${resolved.from})`);
|
|
43
59
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
60
|
+
let doc;
|
|
61
|
+
try {
|
|
62
|
+
doc = await sweep({
|
|
63
|
+
projectDir,
|
|
64
|
+
ref: resolved.ref,
|
|
65
|
+
mode,
|
|
66
|
+
concern: values.concern,
|
|
67
|
+
concernsPath: values.concerns ? resolve(values.concerns) : undefined,
|
|
68
|
+
includeDirty: values['include-dirty'],
|
|
69
|
+
exclude: values.exclude ?? [],
|
|
70
|
+
cap,
|
|
71
|
+
confirmRuns,
|
|
72
|
+
budgetMs,
|
|
73
|
+
runnerCommand: values['runner-cmd'],
|
|
74
|
+
runnerName: values.runner,
|
|
75
|
+
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
76
|
+
toolVersion: version,
|
|
77
|
+
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
78
|
+
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
|
79
|
+
onStage: !values.quiet && !values.json && process.stderr.isTTY
|
|
59
80
|
? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`)
|
|
60
81
|
: undefined,
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
onWarn: (m) => { if (!values.quiet) io.err(`warning: ${m}`); },
|
|
83
|
+
});
|
|
84
|
+
} catch (e) {
|
|
85
|
+
// A concern named on the command line that this project does not declare
|
|
86
|
+
// is a usage error, not a precondition failure: nothing about the
|
|
87
|
+
// repository is wrong.
|
|
88
|
+
if (e instanceof ConcernError) {
|
|
89
|
+
io.err(e.message);
|
|
90
|
+
return 3;
|
|
91
|
+
}
|
|
92
|
+
throw e;
|
|
93
|
+
}
|
|
63
94
|
if (process.stderr.isTTY && !values.quiet && !values.json) process.stderr.write('\r\x1b[K');
|
|
64
95
|
|
|
96
|
+
const { evidence, ...document } = doc;
|
|
65
97
|
const outPath = values.out ? resolve(values.out) : sweepPath(projectDir);
|
|
66
|
-
writeSpecDoc('sweep', outPath,
|
|
98
|
+
writeSpecDoc('sweep', outPath, document);
|
|
99
|
+
// Written only when something was actually probed: an empty document would
|
|
100
|
+
// teach the next run that every class is unproductive.
|
|
101
|
+
if (evidence?.records?.length) writeSpecDoc('evidence', sweepEvidencePath(projectDir), evidence);
|
|
67
102
|
if (values.json) {
|
|
68
|
-
io.out(JSON.stringify(
|
|
103
|
+
io.out(JSON.stringify(document, null, 2));
|
|
69
104
|
} else {
|
|
70
|
-
io.out(renderSweep(
|
|
105
|
+
io.out(renderSweep(document, { limit: Number(values.max) || 20 }));
|
|
71
106
|
if (!values.quiet) io.out(`\nsweep: ${outPath}`);
|
|
72
107
|
}
|
|
73
108
|
return doc.exitCode;
|