testguard-cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +489 -0
- package/README.md +287 -34
- package/package.json +2 -2
- package/spec/GATE-SEMANTICS.md +210 -8
- package/spec/README.md +32 -11
- package/spec/lib/validate.mjs +125 -14
- package/spec/lib/wilson.mjs +112 -0
- package/spec/schemas/baseline.schema.json +7 -7
- package/spec/schemas/brief.schema.json +12 -12
- package/spec/schemas/calibration.schema.json +96 -26
- package/spec/schemas/claims.schema.json +129 -25
- package/spec/schemas/common.schema.json +127 -29
- package/spec/schemas/evidence.schema.json +175 -73
- package/spec/schemas/gate.schema.json +12 -12
- package/spec/schemas/ignore.schema.json +4 -4
- package/spec/schemas/replay.schema.json +10 -10
- package/spec/schemas/status.schema.json +25 -25
- package/src/cli.mjs +20 -5
- package/src/commands/claims.mjs +22 -2
- package/src/commands/probe.mjs +24 -4
- package/src/commands/replay.mjs +1 -1
- package/src/gate/changed.mjs +9 -2
- package/src/probe/attribution.mjs +123 -0
- package/src/probe/classify.mjs +18 -2
- package/src/probe/cost.mjs +205 -0
- package/src/probe/discover.mjs +14 -6
- package/src/probe/inject.mjs +41 -9
- package/src/probe/mocks.mjs +2 -0
- package/src/probe/probe.mjs +235 -52
- package/src/probe/progress.mjs +87 -0
- package/src/probe/pyimports.mjs +287 -0
- package/src/probe/rank.mjs +2 -0
- package/src/probe/runners/index.mjs +38 -11
- package/src/probe/runners/jest.mjs +13 -1
- package/src/probe/runners/playwright.mjs +9 -0
- package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
- package/src/probe/runners/python/_testguard_report.py +141 -0
- package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
- package/src/probe/runners/python.mjs +316 -0
- package/src/probe/runners/shared.mjs +10 -1
- package/src/probe/runners/vitest.mjs +13 -1
- package/src/replay/replay.mjs +50 -22
- package/src/scaffold/producers.python.mjs +209 -0
- package/src/scaffold/scaffold.mjs +86 -32
- package/src/util/glob.mjs +7 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:evidence",
|
|
4
|
+
"title": "claimspec v1 \u2014 evidence file",
|
|
5
5
|
"description": "One run's findings, on disk, in a shape an auditor can read without the tool. Each record binds a claim to a subject (for TestGuard, an injected fault) and a verdict, with every run that produced the verdict.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": [
|
|
@@ -18,26 +18,24 @@
|
|
|
18
18
|
"const": 1
|
|
19
19
|
},
|
|
20
20
|
"tool": {
|
|
21
|
-
"$ref": "urn:
|
|
21
|
+
"$ref": "urn:claimspec:v1:common#/$defs/tool"
|
|
22
22
|
},
|
|
23
23
|
"run": {
|
|
24
24
|
"type": "object",
|
|
25
25
|
"required": [
|
|
26
26
|
"id",
|
|
27
27
|
"startedAt",
|
|
28
|
-
"repo"
|
|
29
|
-
"confirmRuns",
|
|
30
|
-
"mode"
|
|
28
|
+
"repo"
|
|
31
29
|
],
|
|
32
30
|
"properties": {
|
|
33
31
|
"id": {
|
|
34
|
-
"$ref": "urn:
|
|
32
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
35
33
|
},
|
|
36
34
|
"startedAt": {
|
|
37
|
-
"$ref": "urn:
|
|
35
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
38
36
|
},
|
|
39
37
|
"finishedAt": {
|
|
40
|
-
"$ref": "urn:
|
|
38
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
41
39
|
},
|
|
42
40
|
"repo": {
|
|
43
41
|
"type": "object",
|
|
@@ -47,21 +45,21 @@
|
|
|
47
45
|
],
|
|
48
46
|
"properties": {
|
|
49
47
|
"head": {
|
|
50
|
-
"$ref": "urn:
|
|
48
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
51
49
|
},
|
|
52
50
|
"dirty": {
|
|
53
51
|
"type": "boolean"
|
|
54
52
|
},
|
|
55
53
|
"snapshot": {
|
|
56
54
|
"description": "When the working tree was probed instead of `head`: the dangling commit that captured it. Not a ref; may be garbage-collected.",
|
|
57
|
-
"$ref": "urn:
|
|
55
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
58
56
|
},
|
|
59
57
|
"ignoredDirty": {
|
|
60
58
|
"description": "Defender/target files that had uncommitted changes when `head` was probed as committed (an explicit --ref, or --ignore-dirty). The working-tree versions of these files are NOT what the verdicts describe.",
|
|
61
59
|
"type": "array",
|
|
62
60
|
"minItems": 1,
|
|
63
61
|
"items": {
|
|
64
|
-
"$ref": "urn:
|
|
62
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
65
63
|
}
|
|
66
64
|
}
|
|
67
65
|
},
|
|
@@ -74,7 +72,7 @@
|
|
|
74
72
|
],
|
|
75
73
|
"properties": {
|
|
76
74
|
"name": {
|
|
77
|
-
"$ref": "urn:
|
|
75
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
78
76
|
},
|
|
79
77
|
"version": {
|
|
80
78
|
"type": "string",
|
|
@@ -97,18 +95,33 @@
|
|
|
97
95
|
"contention": {
|
|
98
96
|
"description": "Other test runners that were already running when this probe started. Best effort, never a verdict: it exists so a reader of a TIMEOUT or FLAKY-DEFENDER can tell the machine was loaded.",
|
|
99
97
|
"type": "object",
|
|
100
|
-
"required": [
|
|
98
|
+
"required": [
|
|
99
|
+
"detected",
|
|
100
|
+
"runners"
|
|
101
|
+
],
|
|
101
102
|
"properties": {
|
|
102
|
-
"detected": {
|
|
103
|
+
"detected": {
|
|
104
|
+
"const": true
|
|
105
|
+
},
|
|
103
106
|
"runners": {
|
|
104
107
|
"type": "array",
|
|
105
108
|
"minItems": 1,
|
|
106
109
|
"items": {
|
|
107
110
|
"type": "object",
|
|
108
|
-
"required": [
|
|
111
|
+
"required": [
|
|
112
|
+
"pid",
|
|
113
|
+
"command"
|
|
114
|
+
],
|
|
109
115
|
"properties": {
|
|
110
|
-
"pid": {
|
|
111
|
-
|
|
116
|
+
"pid": {
|
|
117
|
+
"type": "integer",
|
|
118
|
+
"minimum": 1
|
|
119
|
+
},
|
|
120
|
+
"command": {
|
|
121
|
+
"type": "string",
|
|
122
|
+
"minLength": 1,
|
|
123
|
+
"maxLength": 200
|
|
124
|
+
}
|
|
112
125
|
},
|
|
113
126
|
"additionalProperties": false
|
|
114
127
|
}
|
|
@@ -143,7 +156,7 @@
|
|
|
143
156
|
],
|
|
144
157
|
"properties": {
|
|
145
158
|
"name": {
|
|
146
|
-
"$ref": "urn:
|
|
159
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
147
160
|
},
|
|
148
161
|
"version": {
|
|
149
162
|
"type": "string",
|
|
@@ -152,9 +165,48 @@
|
|
|
152
165
|
},
|
|
153
166
|
"additionalProperties": false
|
|
154
167
|
}
|
|
168
|
+
},
|
|
169
|
+
"method": {
|
|
170
|
+
"$ref": "urn:claimspec:v1:common#/$defs/method"
|
|
155
171
|
}
|
|
156
172
|
},
|
|
157
|
-
"additionalProperties": false
|
|
173
|
+
"additionalProperties": false,
|
|
174
|
+
"allOf": [
|
|
175
|
+
{
|
|
176
|
+
"if": {
|
|
177
|
+
"anyOf": [
|
|
178
|
+
{
|
|
179
|
+
"properties": {
|
|
180
|
+
"method": {
|
|
181
|
+
"const": "fault-injection"
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
"required": [
|
|
185
|
+
"method"
|
|
186
|
+
]
|
|
187
|
+
},
|
|
188
|
+
{
|
|
189
|
+
"not": {
|
|
190
|
+
"required": [
|
|
191
|
+
"method"
|
|
192
|
+
]
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
]
|
|
196
|
+
},
|
|
197
|
+
"then": {
|
|
198
|
+
"$comment": "Absent `method` means fault-injection, so every evidence file written before the field existed is held to exactly the same requirements as before. `confirmRuns` and `mode` are what injection means by a confirmed run and by where the mutation happened; a method that mutates nothing has neither.",
|
|
199
|
+
"properties": {
|
|
200
|
+
"confirmRuns": true,
|
|
201
|
+
"mode": true
|
|
202
|
+
},
|
|
203
|
+
"required": [
|
|
204
|
+
"confirmRuns",
|
|
205
|
+
"mode"
|
|
206
|
+
]
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
]
|
|
158
210
|
},
|
|
159
211
|
"records": {
|
|
160
212
|
"type": "array",
|
|
@@ -223,14 +275,12 @@
|
|
|
223
275
|
"claim",
|
|
224
276
|
"subject",
|
|
225
277
|
"verdict",
|
|
226
|
-
"detail"
|
|
227
|
-
"defenders",
|
|
228
|
-
"inputs"
|
|
278
|
+
"detail"
|
|
229
279
|
],
|
|
230
280
|
"properties": {
|
|
231
281
|
"fingerprint": {
|
|
232
282
|
"description": "sha256 of `<claim.id>\\n<subject.id>\\n<subject.file>\\n<verdict>`. Derived from identities and outcome, never from `find`/`replace`, so repairing a rotted anchor does not churn the baseline.",
|
|
233
|
-
"$ref": "urn:
|
|
283
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
234
284
|
},
|
|
235
285
|
"claim": {
|
|
236
286
|
"type": "object",
|
|
@@ -242,7 +292,7 @@
|
|
|
242
292
|
],
|
|
243
293
|
"properties": {
|
|
244
294
|
"id": {
|
|
245
|
-
"$ref": "urn:
|
|
295
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
246
296
|
},
|
|
247
297
|
"statement": {
|
|
248
298
|
"type": "string",
|
|
@@ -250,7 +300,7 @@
|
|
|
250
300
|
"maxLength": 2000
|
|
251
301
|
},
|
|
252
302
|
"severity": {
|
|
253
|
-
"$ref": "urn:
|
|
303
|
+
"$ref": "urn:claimspec:v1:common#/$defs/severity"
|
|
254
304
|
},
|
|
255
305
|
"source": {
|
|
256
306
|
"type": "object",
|
|
@@ -275,7 +325,7 @@
|
|
|
275
325
|
"additionalProperties": false
|
|
276
326
|
},
|
|
277
327
|
"producedBy": {
|
|
278
|
-
"$ref": "urn:
|
|
328
|
+
"$ref": "urn:claimspec:v1:common#/$defs/provenance"
|
|
279
329
|
}
|
|
280
330
|
},
|
|
281
331
|
"additionalProperties": false
|
|
@@ -289,40 +339,37 @@
|
|
|
289
339
|
],
|
|
290
340
|
"properties": {
|
|
291
341
|
"kind": {
|
|
292
|
-
"$ref": "urn:
|
|
342
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
293
343
|
},
|
|
294
344
|
"id": {
|
|
295
|
-
"$ref": "urn:
|
|
345
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
296
346
|
},
|
|
297
347
|
"description": {
|
|
298
348
|
"type": "string",
|
|
299
349
|
"maxLength": 2000
|
|
300
350
|
},
|
|
301
351
|
"file": {
|
|
302
|
-
"$ref": "urn:
|
|
352
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
303
353
|
},
|
|
304
354
|
"faultClass": {
|
|
305
|
-
"$ref": "urn:
|
|
355
|
+
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
306
356
|
},
|
|
307
357
|
"producedBy": {
|
|
308
|
-
"$ref": "urn:
|
|
358
|
+
"$ref": "urn:claimspec:v1:common#/$defs/provenance"
|
|
309
359
|
},
|
|
310
360
|
"contentHash": {
|
|
311
361
|
"description": "sha256 of the subject's content (for a fault: find + newline + replace). Excluded from the fingerprint on purpose, so repairing an anchor does not churn a baseline \u2014 and recorded here on purpose, so weakening a fault to make a finding disappear is visible.",
|
|
312
|
-
"$ref": "urn:
|
|
362
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
313
363
|
}
|
|
314
364
|
},
|
|
315
365
|
"additionalProperties": true
|
|
316
366
|
},
|
|
317
367
|
"verdict": {
|
|
318
|
-
"$ref": "urn:
|
|
368
|
+
"$ref": "urn:claimspec:v1:common#/$defs/verdict"
|
|
319
369
|
},
|
|
320
370
|
"detail": {
|
|
321
371
|
"type": "object",
|
|
322
|
-
"required": [
|
|
323
|
-
"baselineRuns",
|
|
324
|
-
"probeRuns"
|
|
325
|
-
],
|
|
372
|
+
"required": [],
|
|
326
373
|
"properties": {
|
|
327
374
|
"reason": {
|
|
328
375
|
"description": "Machine-readable qualifier, e.g. `anchor-missing`, `anchor-ambiguous`, `defenders-not-green`, `replacement-does-not-compile`.",
|
|
@@ -349,7 +396,9 @@
|
|
|
349
396
|
},
|
|
350
397
|
"escalationStoppedEarly": {
|
|
351
398
|
"description": "Why the escalation loop ran fewer than N times. `no-common-failure`: no test failed in every run so far, so no further run could name an undeclared killer. Stopping can only fail to NAME a killer, never upgrade a verdict.",
|
|
352
|
-
"enum": [
|
|
399
|
+
"enum": [
|
|
400
|
+
"no-common-failure"
|
|
401
|
+
]
|
|
353
402
|
},
|
|
354
403
|
"escalationRuns": {
|
|
355
404
|
"type": "array",
|
|
@@ -368,28 +417,73 @@
|
|
|
368
417
|
}
|
|
369
418
|
},
|
|
370
419
|
"flakeRate": {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
420
|
+
"description": "Observed instability of the defenders on UNMODIFIED source: `failures` of `runs` baseline runs failed with the source untouched. Only runs that actually ran are counted (`pass`/`fail`); a load error or a timeout measures nothing about stability and is excluded from both numbers. `failures > 0` means the verdict is `flaky-defender`; the degree is for whoever fixes the defender, and it is the first input to calibration.",
|
|
421
|
+
"type": "object",
|
|
422
|
+
"required": [
|
|
423
|
+
"runs",
|
|
424
|
+
"failures"
|
|
425
|
+
],
|
|
426
|
+
"properties": {
|
|
427
|
+
"runs": {
|
|
428
|
+
"type": "integer",
|
|
429
|
+
"minimum": 1
|
|
430
|
+
},
|
|
431
|
+
"failures": {
|
|
432
|
+
"type": "integer",
|
|
433
|
+
"minimum": 0
|
|
434
|
+
}
|
|
435
|
+
},
|
|
436
|
+
"additionalProperties": false
|
|
437
|
+
},
|
|
438
|
+
"independence": {
|
|
439
|
+
"description": "L3 signal, on `killed` records only: was the killing defender last touched by the same change (or the same author) as the fault's target? A signal for ranking and reporting; it never changes a verdict.",
|
|
440
|
+
"type": "object",
|
|
441
|
+
"required": [
|
|
442
|
+
"class"
|
|
443
|
+
],
|
|
444
|
+
"properties": {
|
|
445
|
+
"class": {
|
|
446
|
+
"enum": [
|
|
447
|
+
"co-authored",
|
|
448
|
+
"separate-change",
|
|
449
|
+
"unknown"
|
|
450
|
+
]
|
|
451
|
+
},
|
|
452
|
+
"defenderCommit": {
|
|
453
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
454
|
+
},
|
|
455
|
+
"targetCommit": {
|
|
456
|
+
"$ref": "urn:claimspec:v1:common#/$defs/gitSha"
|
|
457
|
+
},
|
|
458
|
+
"sameAuthor": {
|
|
459
|
+
"type": "boolean"
|
|
460
|
+
}
|
|
461
|
+
},
|
|
462
|
+
"additionalProperties": false
|
|
463
|
+
},
|
|
464
|
+
"negativeControl": {
|
|
465
|
+
"description": "The result of replacing the subject with content its loader cannot parse, and running the defenders once. `reached`: they went red, so they do execute this file and a `survived` verdict is about their assertions. `not-reached`: they stayed green, so they never execute it and the verdict becomes `unverifiable` with reason `subject-not-executed`. Charged only where a false answer is expensive \u2014 on a would-be survivor \u2014 because a kill already proves the defenders reached the code. Absent when the control was not run or the runner cannot say what \"cannot compile\" means for its language.",
|
|
466
|
+
"enum": [
|
|
467
|
+
"reached",
|
|
468
|
+
"not-reached"
|
|
469
|
+
]
|
|
470
|
+
},
|
|
471
|
+
"targetNotImported": {
|
|
472
|
+
"description": "True when, with the fault applied, no defender caused the subject's module to be imported at all. Reported by runners that can tell (Python), never inferred. It does not change the verdict, and it is the one thing that makes a `survived` unreadable: the tests were not blind to the fault, they never reached it. A module imported from OUTSIDE the tree being probed is not recorded here \u2014 that run is refused outright, because every verdict in it would be false.",
|
|
473
|
+
"const": true
|
|
474
|
+
},
|
|
475
|
+
"message": {
|
|
476
|
+
"description": "Human-readable detail for a reason that carries one \u2014 today `probe-error`, whose first line names what threw. Never parsed; a reader should not have to reproduce the run to learn what happened.",
|
|
477
|
+
"type": "string",
|
|
478
|
+
"maxLength": 1024
|
|
479
|
+
},
|
|
480
|
+
"restoreSkipped": {
|
|
481
|
+
"description": "The fault was applied but the restore found nothing to put back: the target file no longer existed. In worktree mode that is a disturbed run rather than a failed one \u2014 the mutation only ever existed inside a scratch worktree, and something outside the process deleted it \u2014 but the verdicts in that run were measured against a tree that was being removed underneath them, and a reader has to be able to see that. In `--in-place` mode it never appears: there a missing target is the user's own file gone, and the restore throws.",
|
|
482
|
+
"enum": [
|
|
483
|
+
"target-missing"
|
|
484
|
+
]
|
|
485
|
+
},
|
|
486
|
+
"anchor": {
|
|
393
487
|
"description": "With an anchor-related reason: how many times `find` occurred versus how many were expected.",
|
|
394
488
|
"type": "object",
|
|
395
489
|
"required": [
|
|
@@ -407,9 +501,16 @@
|
|
|
407
501
|
}
|
|
408
502
|
},
|
|
409
503
|
"additionalProperties": false
|
|
504
|
+
},
|
|
505
|
+
"methodDetail": {
|
|
506
|
+
"description": "Data belonging to a method the spec has not specified yet. `assertion` and `scan` are reserved names with no required fields, so a tool adopting one needs somewhere to put the rule it checked and the place it checked \u2014 without which it can declare that a probe exists and nothing about it. Deliberately unconstrained, deliberately namespaced, and **forbidden under `fault-injection`**, whose shape IS specified and must not acquire a junk drawer. What the first real consumer puts here is the evidence for what the specified shape should become; it is a staging area, not a permanent home.",
|
|
507
|
+
"type": "object",
|
|
508
|
+
"minProperties": 1,
|
|
509
|
+
"additionalProperties": true
|
|
410
510
|
}
|
|
411
511
|
},
|
|
412
|
-
"additionalProperties": false
|
|
512
|
+
"additionalProperties": false,
|
|
513
|
+
"description": "Everything behind the verdict. Under `fault-injection` the runs are mandatory and the validator enforces them; a method that runs no tests carries whatever its own evidence is."
|
|
413
514
|
},
|
|
414
515
|
"defenders": {
|
|
415
516
|
"type": "object",
|
|
@@ -428,7 +529,7 @@
|
|
|
428
529
|
"resolved": {
|
|
429
530
|
"type": "array",
|
|
430
531
|
"items": {
|
|
431
|
-
"$ref": "urn:
|
|
532
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
432
533
|
}
|
|
433
534
|
},
|
|
434
535
|
"nocover": {
|
|
@@ -442,11 +543,11 @@
|
|
|
442
543
|
"description": "Candidate test files that import the subject's file but mock it (`vi.mock`/`jest.mock`). Discovered candidates that mock are excluded from `resolved`; a declared defender that mocks stays in `resolved` and is listed here, because a declared defender that mocks the subject is a broken evidence chain.",
|
|
443
544
|
"type": "array",
|
|
444
545
|
"items": {
|
|
445
|
-
"$ref": "urn:
|
|
546
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
446
547
|
}
|
|
447
548
|
},
|
|
448
549
|
"signals": {
|
|
449
|
-
"description": "Static, annotation-grade signals about the defenders. They never change a verdict. `mocked-never-asserted`: the file mocks the subject and never asserts on anything imported from it. `unasserted-annotated`: the same, but the author annotated the mock with a reason (`// unasserted: <why>`), recorded here \u2014 silenced, never hidden.",
|
|
550
|
+
"description": "Static, annotation-grade signals about the defenders. They never change a verdict. `mocked-never-asserted`: the file mocks the subject and never asserts on anything imported from it. `unasserted-annotated`: the same, but the author annotated the mock with a reason (`// unasserted: <why>`), recorded here \u2014 silenced, never hidden. `target-attribute-patched`: a Python defender that patches individual attributes of the subject (`patch(\"pkg.mod.fn\")`) rather than the module. Unlike a JavaScript module mock it does NOT disqualify the file \u2014 a fault anywhere else in the module is still detectable \u2014 so the file stays in `resolved`; `reason` names the patched attributes, which are where that test is least likely to notice anything.",
|
|
450
551
|
"type": "array",
|
|
451
552
|
"items": {
|
|
452
553
|
"type": "object",
|
|
@@ -456,12 +557,13 @@
|
|
|
456
557
|
],
|
|
457
558
|
"properties": {
|
|
458
559
|
"file": {
|
|
459
|
-
"$ref": "urn:
|
|
560
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
460
561
|
},
|
|
461
562
|
"signal": {
|
|
462
563
|
"enum": [
|
|
463
564
|
"mocked-never-asserted",
|
|
464
|
-
"unasserted-annotated"
|
|
565
|
+
"unasserted-annotated",
|
|
566
|
+
"target-attribute-patched"
|
|
465
567
|
]
|
|
466
568
|
},
|
|
467
569
|
"reason": {
|
|
@@ -477,13 +579,13 @@
|
|
|
477
579
|
"description": "When the resolved defenders ran under more than one runner: runner name \u2192 the files it ran. Every file appears in `resolved`. Absent when the project runner ran them all.",
|
|
478
580
|
"type": "object",
|
|
479
581
|
"propertyNames": {
|
|
480
|
-
"$ref": "urn:
|
|
582
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
481
583
|
},
|
|
482
584
|
"additionalProperties": {
|
|
483
585
|
"type": "array",
|
|
484
586
|
"minItems": 1,
|
|
485
587
|
"items": {
|
|
486
|
-
"$ref": "urn:
|
|
588
|
+
"$ref": "urn:claimspec:v1:common#/$defs/repoPath"
|
|
487
589
|
}
|
|
488
590
|
}
|
|
489
591
|
}
|
|
@@ -499,12 +601,12 @@
|
|
|
499
601
|
],
|
|
500
602
|
"properties": {
|
|
501
603
|
"targetHash": {
|
|
502
|
-
"$ref": "urn:
|
|
604
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
503
605
|
},
|
|
504
606
|
"defenderHashes": {
|
|
505
607
|
"type": "object",
|
|
506
608
|
"additionalProperties": {
|
|
507
|
-
"$ref": "urn:
|
|
609
|
+
"$ref": "urn:claimspec:v1:common#/$defs/sha256"
|
|
508
610
|
}
|
|
509
611
|
}
|
|
510
612
|
},
|
|
@@ -512,7 +614,7 @@
|
|
|
512
614
|
},
|
|
513
615
|
"reusedFrom": {
|
|
514
616
|
"description": "Run id whose verdict was reused because `inputs` were unchanged.",
|
|
515
|
-
"$ref": "urn:
|
|
617
|
+
"$ref": "urn:claimspec:v1:common#/$defs/identifier"
|
|
516
618
|
},
|
|
517
619
|
"rank": {
|
|
518
620
|
"description": "Additive ordering signal. Never alters `verdict`.",
|
|
@@ -539,7 +641,7 @@
|
|
|
539
641
|
"annotations": {
|
|
540
642
|
"type": "array",
|
|
541
643
|
"items": {
|
|
542
|
-
"$ref": "urn:
|
|
644
|
+
"$ref": "urn:claimspec:v1:common#/$defs/annotation"
|
|
543
645
|
}
|
|
544
646
|
}
|
|
545
647
|
},
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:gate",
|
|
4
|
+
"title": "claimspec v1 — change gate",
|
|
5
5
|
"description": "Claim coverage of one change. `probe` asks whether the tests defend the claims that exist; `gate` asks whether the files a change touched carry any claim at all. Measured per file against the merge-base of a reference and HEAD (or the working tree). A file is covered when a fault anchors to it, when it is a test file that defends a claim, or when an unexpired ignore entry excuses it — and every reliance on an ignore entry is listed, so a reviewer sees why the gate passed.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": ["schemaVersion", "tool", "generatedAt", "ref", "base", "head", "includeDirty", "strict", "changed", "evaluated", "excluded", "covered", "uncovered", "reliedOn", "expired", "exitCode"],
|
|
8
8
|
"properties": {
|
|
9
9
|
"$schema": { "type": "string" },
|
|
10
10
|
"schemaVersion": { "const": 1 },
|
|
11
|
-
"tool": { "$ref": "urn:
|
|
12
|
-
"generatedAt": { "$ref": "urn:
|
|
11
|
+
"tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
|
|
12
|
+
"generatedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
13
13
|
"ref": { "description": "The reference the change was measured against, as given (a branch, a tag, a sha).", "type": "string", "minLength": 1, "maxLength": 256 },
|
|
14
|
-
"base": { "description": "merge-base(ref, HEAD): the commit the change is relative to.", "$ref": "urn:
|
|
15
|
-
"head": { "$ref": "urn:
|
|
14
|
+
"base": { "description": "merge-base(ref, HEAD): the commit the change is relative to.", "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
15
|
+
"head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
16
16
|
"includeDirty": { "description": "True when the working tree was compared instead of HEAD.", "type": "boolean" },
|
|
17
17
|
"strict": { "description": "True when a non-empty change with zero evaluated files fails instead of passing with a note.", "type": "boolean" },
|
|
18
18
|
"changed": { "description": "Files changed inside the project directory, deletions excluded.", "type": "integer", "minimum": 0 },
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
"type": "object",
|
|
24
24
|
"required": ["file", "by"],
|
|
25
25
|
"properties": {
|
|
26
|
-
"file": { "$ref": "urn:
|
|
26
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
27
27
|
"by": { "description": "`non-source`, `default:<glob>` or `exclude:<glob>`.", "type": "string", "minLength": 1, "maxLength": 600 }
|
|
28
28
|
},
|
|
29
29
|
"additionalProperties": false
|
|
@@ -35,17 +35,17 @@
|
|
|
35
35
|
"type": "object",
|
|
36
36
|
"required": ["file", "by"],
|
|
37
37
|
"properties": {
|
|
38
|
-
"file": { "$ref": "urn:
|
|
38
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
39
39
|
"by": { "enum": ["fault", "defender", "ignore"] },
|
|
40
|
-
"claimIds": { "type": "array", "minItems": 1, "items": { "$ref": "urn:
|
|
40
|
+
"claimIds": { "type": "array", "minItems": 1, "items": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" } },
|
|
41
41
|
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 }
|
|
42
42
|
},
|
|
43
43
|
"additionalProperties": false
|
|
44
44
|
}
|
|
45
45
|
},
|
|
46
|
-
"uncovered": { "type": "array", "items": { "$ref": "urn:
|
|
47
|
-
"reliedOn": { "description": "Unexpired ignore entries that excused at least one changed file. The gate passed *because of* these; a reviewer reads them.", "type": "array", "items": { "$ref": "urn:
|
|
48
|
-
"expired": { "description": "Expired ignore entries that would have matched a changed file. They excuse nothing; the files are reported as uncovered.", "type": "array", "items": { "$ref": "urn:
|
|
46
|
+
"uncovered": { "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/uncoveredFile" } },
|
|
47
|
+
"reliedOn": { "description": "Unexpired ignore entries that excused at least one changed file. The gate passed *because of* these; a reviewer reads them.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance" } },
|
|
48
|
+
"expired": { "description": "Expired ignore entries that would have matched a changed file. They excuse nothing; the files are reported as uncovered.", "type": "array", "items": { "$ref": "urn:claimspec:v1:common#/$defs/ignoreReliance" } },
|
|
49
49
|
"exitCode": { "enum": [0, 1] }
|
|
50
50
|
},
|
|
51
51
|
"additionalProperties": false
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:ignore",
|
|
4
|
+
"title": "claimspec v1 — structured ignore",
|
|
5
5
|
"description": "Reviewable scoping. Every entry carries a reason. Tools may also accept a plain gitignore-syntax text file as shorthand for path entries without reasons; this structured form is what an auditor reads.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": ["schemaVersion", "entries"],
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
|
|
22
22
|
"reason": { "type": "string", "minLength": 8, "maxLength": 2000 },
|
|
23
23
|
"by": { "type": "string", "maxLength": 256 },
|
|
24
|
-
"at": { "$ref": "urn:
|
|
24
|
+
"at": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
25
25
|
"expires": {
|
|
26
26
|
"description": "After this instant the entry no longer applies. Tools warn on expired entries.",
|
|
27
|
-
"$ref": "urn:
|
|
27
|
+
"$ref": "urn:claimspec:v1:common#/$defs/isoDateTime"
|
|
28
28
|
}
|
|
29
29
|
},
|
|
30
30
|
"additionalProperties": false
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "urn:
|
|
4
|
-
"title": "
|
|
3
|
+
"$id": "urn:claimspec:v1:replay",
|
|
4
|
+
"title": "claimspec v1 — bug replay",
|
|
5
5
|
"description": "Would this suite have caught the bugs that actually escaped? For each fix commit in a range: revert only the source to its parent, remove the test the fix shipped, run what remains. `blind` means the suite was blind to a bug that really shipped. Unlike an injected fault, a replayed bug is ground truth — a human already confirmed it was a defect — which is what makes this the oracle a fault model is calibrated against.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"required": ["schemaVersion", "tool", "run", "records"],
|
|
8
8
|
"properties": {
|
|
9
9
|
"$schema": { "type": "string" },
|
|
10
10
|
"schemaVersion": { "const": 1 },
|
|
11
|
-
"tool": { "$ref": "urn:
|
|
11
|
+
"tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
|
|
12
12
|
"run": {
|
|
13
13
|
"type": "object",
|
|
14
14
|
"required": ["id", "startedAt", "range", "confirmRuns"],
|
|
15
15
|
"properties": {
|
|
16
16
|
"id": { "type": "string", "minLength": 1, "maxLength": 128 },
|
|
17
|
-
"startedAt": { "$ref": "urn:
|
|
18
|
-
"finishedAt": { "$ref": "urn:
|
|
17
|
+
"startedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
18
|
+
"finishedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
19
19
|
"range": {
|
|
20
20
|
"description": "The commit range searched for fix commits, as given.",
|
|
21
21
|
"type": "string",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"type": "object",
|
|
44
44
|
"required": ["commit", "patchId", "verdict", "source", "removedTests", "runs"],
|
|
45
45
|
"properties": {
|
|
46
|
-
"commit": { "$ref": "urn:
|
|
46
|
+
"commit": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
47
47
|
"patchId": {
|
|
48
48
|
"description": "`git patch-id --stable` of the fix: the identity of the CHANGE rather than of the commit. A dual-branch topology carries the same fix under several shas, and counting it twice corrupts the corpus. Hex, 40 characters for a SHA-1 repository and 64 for a SHA-256 one; the commit sha when git could not compute one.",
|
|
49
49
|
"type": "string",
|
|
@@ -59,22 +59,22 @@
|
|
|
59
59
|
"description": "Source files reverted to the fix's parent.",
|
|
60
60
|
"type": "array",
|
|
61
61
|
"minItems": 1,
|
|
62
|
-
"items": { "$ref": "urn:
|
|
62
|
+
"items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" }
|
|
63
63
|
},
|
|
64
64
|
"removedTests": {
|
|
65
65
|
"description": "Test files the fix shipped, removed before the run: the fix's own test proves nothing about what the suite knew before it.",
|
|
66
66
|
"type": "array",
|
|
67
|
-
"items": { "$ref": "urn:
|
|
67
|
+
"items": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" }
|
|
68
68
|
},
|
|
69
69
|
"ranTests": { "type": "integer", "minimum": 0 },
|
|
70
70
|
"faultClass": {
|
|
71
71
|
"description": "The injected-fault class this real diff most resembles, derived mechanically from the diff shape. This is the join key between ground truth and a fault model: it is what lets P(this fault class is a real bug) be computed.",
|
|
72
|
-
"$ref": "urn:
|
|
72
|
+
"$ref": "urn:claimspec:v1:common#/$defs/faultClass"
|
|
73
73
|
},
|
|
74
74
|
"runs": {
|
|
75
75
|
"type": "array",
|
|
76
76
|
"minItems": 1,
|
|
77
|
-
"items": { "$ref": "urn:
|
|
77
|
+
"items": { "$ref": "urn:claimspec:v1:evidence#/$defs/testRun" }
|
|
78
78
|
}
|
|
79
79
|
},
|
|
80
80
|
"additionalProperties": false
|