testguard-cli 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +255 -0
- package/README.md +275 -31
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +58 -1
- package/spec/README.md +1 -0
- package/spec/lib/validate.mjs +57 -1
- package/spec/schemas/baseline.schema.json +8 -0
- package/spec/schemas/common.schema.json +5 -1
- package/spec/schemas/evidence.schema.json +374 -63
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +256 -54
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +71 -7
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +53 -10
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +13 -7
- package/src/commands/claims.mjs +35 -6
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +11 -3
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +3 -1
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +3 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/contention.mjs +60 -0
- package/src/probe/discover.mjs +12 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/mocks.mjs +120 -0
- package/src/probe/probe.mjs +126 -18
- package/src/probe/rank.mjs +73 -15
- package/src/probe/runners/index.mjs +49 -5
- package/src/probe/runners/jest.mjs +3 -3
- package/src/probe/runners/playwright.mjs +133 -0
- package/src/probe/runners/shared.mjs +70 -13
- package/src/probe/runners/vitest.mjs +4 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +309 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/scaffold.mjs +9 -3
- package/src/status/status.mjs +47 -6
|
@@ -1,10 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Mechanical fault producers. Deterministic, line-oriented, no AST, no LLM.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Field reports found that most hand-written faults are one of a few shapes.
|
|
5
|
+
* Each producer looks at one line (plus a little context) and proposes a
|
|
6
|
+
* fault whose `find` is the exact line, so the anchor hits by construction.
|
|
7
|
+
* A human keeps or drops every proposal.
|
|
8
|
+
*
|
|
9
|
+
* Shapes:
|
|
10
|
+
* condition-forced `if (<guard>) {` → `if (false) {`
|
|
11
|
+
* statement-deleted a single-line guard or a state change removed
|
|
12
|
+
* return-altered `return <check>;` → `return true;`
|
|
13
|
+
* literal-changed a security flag / window / cost literal weakened
|
|
14
|
+
* call-removed a bare verify/validate/check call removed
|
|
15
|
+
* field-dropped a field removed from a persisted or transmitted object,
|
|
16
|
+
* an allow-list, a schema, or a spread merge (issue #14)
|
|
17
|
+
* argument-swapped a call kept, its parameter-derived argument swapped for
|
|
18
|
+
* a trivial value (issue #22)
|
|
19
|
+
* element-removed a one-line JSX element (self-closing or paired) removed
|
|
20
|
+
* (issue #15)
|
|
21
|
+
* handler-dropped an on<Event>={…} prop removed from a JSX element
|
|
22
|
+
* (issue #15)
|
|
8
23
|
*/
|
|
9
24
|
|
|
10
25
|
const COMMENT = /^\s*(\/\/|\*|\/\*)/;
|
|
@@ -26,6 +41,36 @@ const FUNCTION_HEAD = [
|
|
|
26
41
|
];
|
|
27
42
|
const NOT_A_FUNCTION = new Set(['if', 'for', 'while', 'switch', 'catch', 'else', 'return', 'function', 'constructor']);
|
|
28
43
|
|
|
44
|
+
// ── field-dropped ──
|
|
45
|
+
// An object literal counts as a payload when its opening line says so: it is
|
|
46
|
+
// returned, built by an arrow, assigned to a payload-ish name, passed to a
|
|
47
|
+
// persistence/transport call, or is a validation schema. Type-level braces
|
|
48
|
+
// (`interface`, `type X = {`) are never payloads: dropping a key there is a
|
|
49
|
+
// compile error, not a behaviour.
|
|
50
|
+
const PAYLOAD_OPENER = /(\breturn\s*\{\s*$|=>\s*\(?\s*\{\s*$|\b(?:body|payload|input|item|params|data|record|row|attrs|attributes|values|fields|patch|update|updates|changes|doc|document|entity|dto|request|response|options|args|props|config|settings|message|event|entry|audit)\w*\s*[:=]\s*\{\s*$|\.(?:save|update|put|patch|post|send|write|insert|upsert|create|set|publish|emit|dispatch|persist|store|log|fetch|request|audit|record|track|notify)\w*\s*\(\s*(?:[^,{]*,\s*)*\{\s*$|\b(?:z|yup|joi|v|t)\.(?:object|shape|strictObject|looseObject)\s*\(\s*\{\s*$|\.(?:object|shape|extend|merge|pick|omit)\s*\(\s*\{\s*$|\bobject\s*\(\s*\{\s*$)/i;
|
|
51
|
+
const TYPE_OPENER = /\b(?:interface|type|enum|declare)\b/;
|
|
52
|
+
const ALLOWLIST_OPENER = /\b\w*(?:fields|keys|allowed|writable|columns|attributes|props|whitelist|allowlist|editable|mutable|updatable|permitted|selectable|sortable|filterable|required)\w*\b[^[]*\[\s*$/i;
|
|
53
|
+
const PROP_LINE = /^\s*(?:[\w$]+|'[^']*'|"[^"]*"|\[[^\]]+\])\s*:\s*(?!\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[\w$]+\s*=>))(.+?)(,?)\s*$/;
|
|
54
|
+
const SHORTHAND_LINE = /^\s*[\w$]+\s*,\s*$/;
|
|
55
|
+
const METHOD_LINE = /^\s*(?:async\s+)?(?:get\s+|set\s+)?[\w$]+\s*\([^)]*\)\s*\{\s*$/;
|
|
56
|
+
const SPREAD_LINE = /^\s*\.\.\.[\w$.()]+\s*,?\s*$/;
|
|
57
|
+
const STRING_ENTRY = /^\s*(['"])[^'"]+\1\s*,?\s*$/;
|
|
58
|
+
const INLINE_SPREAD_MERGE = /\{\s*\.\.\.([\w$.()]+)\s*,\s*(?=[^}]*\S)/;
|
|
59
|
+
|
|
60
|
+
// ── element-removed / handler-dropped (JSX) ──
|
|
61
|
+
// One-line elements only, so the removal is syntactically safe by construction.
|
|
62
|
+
const JSX_SELF_CLOSING = /^\s*<[A-Za-z][\w.-]*\b[^<>]*\/>\s*$/;
|
|
63
|
+
const JSX_PAIRED_LINE = /^\s*<([A-Za-z][\w.-]*)\b[^<>]*>[^<>]*<\/\1>\s*$/;
|
|
64
|
+
const JSX_HANDLER_PROP_LINE = /^\s*on[A-Z]\w*=\{(?:[^{}]|\{[^{}]*\})*\}\s*$/;
|
|
65
|
+
const JSX_INLINE_HANDLER = /\s+on[A-Z]\w*=\{(?:[^{}]|\{[^{}]*\})*\}/;
|
|
66
|
+
|
|
67
|
+
// ── argument-swapped ──
|
|
68
|
+
// A call whose first argument is derived from a parameter or a request-like
|
|
69
|
+
// value. `callee(` must not be a keyword, a constructor, an import, or a test
|
|
70
|
+
// helper; the argument must not be a literal.
|
|
71
|
+
const CALL_WITH_ARG = /(?<![\w$.])(?<!\bnew\s+)(?!(?:if|for|while|switch|catch|return|function|typeof|await|new|require|import|expect|describe|it|test|vi|jest|console|logger)\b)([\w$]+(?:\.[\w$]+)*)\(\s*((?:[\w$]+(?:\?\.|\.)[\w$?.]+)|(?:[\w$]+\([^()]*\))|(?:[\w$]+))\s*(?=,|\))/g;
|
|
72
|
+
const REQUEST_LIKE = /^(?:req|request|res|ctx|context|event|input|args|params|options|opts|props|state|session|user|payload|body|query|headers|msg|message|job|task)$/;
|
|
73
|
+
|
|
29
74
|
/** Name of the function whose head is on this line, or null. */
|
|
30
75
|
export function functionHead(line) {
|
|
31
76
|
for (const re of FUNCTION_HEAD) {
|
|
@@ -35,17 +80,177 @@ export function functionHead(line) {
|
|
|
35
80
|
return null;
|
|
36
81
|
}
|
|
37
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Parameter names declared on a function-head line: plain, defaulted,
|
|
85
|
+
* typed, rest, and the leaves of one level of destructuring. Best effort.
|
|
86
|
+
*/
|
|
87
|
+
export function functionParams(line) {
|
|
88
|
+
const open = line.indexOf('(');
|
|
89
|
+
const arrow = line.indexOf('=>');
|
|
90
|
+
if (open === -1 || (arrow !== -1 && arrow < open)) {
|
|
91
|
+
// `const f = x => …` style: a bare identifier before `=>`
|
|
92
|
+
const bare = /=\s*(?:async\s+)?([\w$]+)\s*=>/.exec(line);
|
|
93
|
+
return bare ? [bare[1]] : [];
|
|
94
|
+
}
|
|
95
|
+
let depth = 0;
|
|
96
|
+
let close = -1;
|
|
97
|
+
for (let i = open; i < line.length; i++) {
|
|
98
|
+
if (line[i] === '(') depth++;
|
|
99
|
+
else if (line[i] === ')' && --depth === 0) { close = i; break; }
|
|
100
|
+
}
|
|
101
|
+
if (close === -1) return [];
|
|
102
|
+
const inner = line.slice(open + 1, close);
|
|
103
|
+
const names = [];
|
|
104
|
+
for (const raw of inner.split(/,(?![^{[]*[}\]])/)) {
|
|
105
|
+
let p = raw.trim();
|
|
106
|
+
if (!p) continue;
|
|
107
|
+
p = p.replace(/^\.\.\./, '').replace(/\s*=[\s\S]*$/, '').replace(/\s*:[^,{}]*$/, '').replace(/\?$/, '');
|
|
108
|
+
const destructured = /^[{[]([\s\S]*)[}\]]$/.exec(p);
|
|
109
|
+
if (destructured) {
|
|
110
|
+
for (const part of destructured[1].split(',')) {
|
|
111
|
+
const leaf = part.trim().replace(/^\.\.\./, '').split(/\s*[:=]\s*/).pop().trim();
|
|
112
|
+
if (/^[\w$]+$/.test(leaf)) names.push(leaf);
|
|
113
|
+
}
|
|
114
|
+
} else if (/^[\w$]+$/.test(p)) {
|
|
115
|
+
names.push(p);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return names;
|
|
119
|
+
}
|
|
120
|
+
|
|
38
121
|
function isGuard(cond, lines, i) {
|
|
39
122
|
if (/^\s*!/.test(cond)) return true;
|
|
40
123
|
const tail = lines.slice(i, i + 4).join('\n');
|
|
41
124
|
return GUARD_BODY.test(tail);
|
|
42
125
|
}
|
|
43
126
|
|
|
44
|
-
|
|
45
|
-
|
|
127
|
+
const openerCache = new WeakMap();
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* For every line, the innermost bracket left open before that line starts:
|
|
131
|
+
* `{ line, char }` or null. One forward pass with a stack; strings and
|
|
132
|
+
* comments are not parsed (deterministic heuristic, like everything here).
|
|
133
|
+
*/
|
|
134
|
+
export function openerIndex(lines) {
|
|
135
|
+
if (openerCache.has(lines)) return openerCache.get(lines);
|
|
136
|
+
const out = new Array(lines.length);
|
|
137
|
+
const stack = [];
|
|
138
|
+
for (let i = 0; i < lines.length; i++) {
|
|
139
|
+
out[i] = stack.length ? stack[stack.length - 1] : null;
|
|
140
|
+
const line = lines[i];
|
|
141
|
+
if (COMMENT.test(line)) continue;
|
|
142
|
+
for (const ch of line) {
|
|
143
|
+
if (ch === '{' || ch === '[' || ch === '(') stack.push({ line: i, char: ch });
|
|
144
|
+
else if (ch === '}' || ch === ']' || ch === ')') stack.pop();
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
openerCache.set(lines, out);
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const balanced = (line) => {
|
|
152
|
+
const n = (re) => (line.match(re) ?? []).length;
|
|
153
|
+
return n(/\{/g) === n(/\}/g) && n(/\[/g) === n(/\]/g) && n(/\(/g) === n(/\)/g);
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
/** A line that can be removed on its own: balanced, and either comma-terminated or followed by the closer. */
|
|
157
|
+
function removableAlone(lines, i) {
|
|
158
|
+
const line = lines[i];
|
|
159
|
+
if (!balanced(line)) return false;
|
|
160
|
+
if (/,\s*$/.test(line)) return true;
|
|
161
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
162
|
+
if (!lines[j].trim()) continue;
|
|
163
|
+
return /^\s*[}\])]/.test(lines[j]);
|
|
164
|
+
}
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function fieldDropProposals(lines, i) {
|
|
169
|
+
const line = lines[i];
|
|
170
|
+
const out = [];
|
|
171
|
+
const opener = openerIndex(lines)[i];
|
|
172
|
+
const openerLine = opener ? lines[opener.line] : '';
|
|
173
|
+
const inPayload = opener?.char === '{' && PAYLOAD_OPENER.test(openerLine) && !TYPE_OPENER.test(openerLine);
|
|
174
|
+
const inAllowList = opener?.char === '[' && ALLOWLIST_OPENER.test(openerLine);
|
|
175
|
+
|
|
176
|
+
if (inPayload && !METHOD_LINE.test(line) && removableAlone(lines, i)) {
|
|
177
|
+
const schema = /\b(?:z|yup|joi|v|t)\.|\.(?:object|shape|extend|merge|pick|omit)\s*\(|\bobject\s*\(/i.test(openerLine);
|
|
178
|
+
if (SPREAD_LINE.test(line)) {
|
|
179
|
+
out.push({ faultClass: 'field-dropped', description: `Base object dropped from the merge: \`${line.trim()}\` no longer contributes its fields.`, replace: '' });
|
|
180
|
+
} else if (PROP_LINE.test(line) || SHORTHAND_LINE.test(line)) {
|
|
181
|
+
const name = line.trim().replace(/[,:].*$/, '').replace(/^['"[]|['"\]]$/g, '');
|
|
182
|
+
out.push({ faultClass: 'field-dropped', description: schema ? `Field dropped from the schema: \`${name}\` is no longer validated or serialised.` : `Field dropped: \`${name}\` is no longer written.`, replace: '' });
|
|
183
|
+
}
|
|
184
|
+
} else if (inAllowList && STRING_ENTRY.test(line) && removableAlone(lines, i)) {
|
|
185
|
+
out.push({ faultClass: 'field-dropped', description: `Field dropped from the allow-list: ${line.trim().replace(/,\s*$/, '')} is no longer permitted.`, replace: '' });
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const merge = INLINE_SPREAD_MERGE.exec(line);
|
|
189
|
+
if (merge) {
|
|
190
|
+
out.push({ faultClass: 'field-dropped', description: `Base object dropped from the inline merge: \`...${merge[1]}\` no longer contributes its fields.`, replace: line.replace(merge[0], '{ ') });
|
|
191
|
+
}
|
|
192
|
+
return out;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function jsxProposals(lines, i) {
|
|
196
|
+
const line = lines[i];
|
|
197
|
+
const out = [];
|
|
198
|
+
if (JSX_SELF_CLOSING.test(line) || JSX_PAIRED_LINE.test(line)) {
|
|
199
|
+
const tag = /<([A-Za-z][\w.-]*)/.exec(line)[1];
|
|
200
|
+
out.push({ faultClass: 'element-removed', description: `Element removed: \`<${tag}>\` is no longer rendered.`, replace: '' });
|
|
201
|
+
}
|
|
202
|
+
if (JSX_HANDLER_PROP_LINE.test(line)) {
|
|
203
|
+
const name = /on[A-Z]\w*/.exec(line)[0];
|
|
204
|
+
out.push({ faultClass: 'handler-dropped', description: `Handler dropped: \`${name}\` is no longer wired.`, replace: '' });
|
|
205
|
+
} else {
|
|
206
|
+
const m = JSX_INLINE_HANDLER.exec(line);
|
|
207
|
+
if (m && /<[A-Za-z]/.test(line)) {
|
|
208
|
+
const name = /on[A-Z]\w*/.exec(m[0])[0];
|
|
209
|
+
out.push({ faultClass: 'handler-dropped', description: `Handler dropped: \`${name}\` is no longer wired.`, replace: line.replace(m[0], '') });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function isDerived(arg, params) {
|
|
216
|
+
const idents = arg.match(/[\w$]+/g) ?? [];
|
|
217
|
+
if (idents.some((id) => params.has(id))) return true;
|
|
218
|
+
const head = /^([\w$]+)/.exec(arg)?.[1] ?? '';
|
|
219
|
+
return REQUEST_LIKE.test(head) || arg === 'this' || /^this[.?]/.test(arg);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function argumentSwapProposals(lines, i, params) {
|
|
223
|
+
const line = lines[i];
|
|
224
|
+
const out = [];
|
|
225
|
+
const head = functionHead(line);
|
|
226
|
+
const seen = new Set();
|
|
227
|
+
for (const m of line.matchAll(CALL_WITH_ARG)) {
|
|
228
|
+
const [whole, callee, arg] = m;
|
|
229
|
+
if (head && callee.split('.').pop() === head) continue; // the head of a declaration, not a call
|
|
230
|
+
if (/^(?:'|"|`|\d|true$|false$|null$|undefined$)/.test(arg)) continue;
|
|
231
|
+
if (!isDerived(arg, params)) continue;
|
|
232
|
+
if (seen.has(whole)) continue;
|
|
233
|
+
seen.add(whole);
|
|
234
|
+
const swap = (value) => line.replace(whole, `${callee}(${value}`);
|
|
235
|
+
out.push({ faultClass: 'argument-swapped', description: `Argument swapped: \`${callee}(${arg}, …)\` is called with \`undefined\` instead of the derived value.`, replace: swap('undefined') });
|
|
236
|
+
if (/\(/.test(arg)) out.push({ faultClass: 'argument-swapped', description: `Argument swapped: \`${callee}(${arg}, …)\` is called with an empty object instead of the derived value.`, replace: swap('{}') });
|
|
237
|
+
}
|
|
238
|
+
return out;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Proposals for one line. Each: { faultClass, description, replace } where
|
|
243
|
+
* find is the line itself. `ctx.params` is the enclosing function's parameter
|
|
244
|
+
* set (for argument-swapped); `ctx.fieldDrops = false` disables the
|
|
245
|
+
* data/UI shapes — field-dropped, element-removed, handler-dropped — for
|
|
246
|
+
* test files, fixtures and migrations, where an object or element is data,
|
|
247
|
+
* not the product's output.
|
|
248
|
+
*/
|
|
249
|
+
export function proposalsForLine(lines, i, ctx = {}) {
|
|
46
250
|
const line = lines[i];
|
|
47
251
|
const out = [];
|
|
48
252
|
if (!line.trim() || COMMENT.test(line)) return out;
|
|
253
|
+
const params = ctx.params ?? new Set();
|
|
49
254
|
|
|
50
255
|
const ifm = IF_LINE.exec(line);
|
|
51
256
|
if (ifm && isGuard(ifm[2], lines, i)) {
|
|
@@ -81,5 +286,11 @@ export function proposalsForLine(lines, i) {
|
|
|
81
286
|
} else if ((m = WINDOW_NUM.exec(line))) {
|
|
82
287
|
out.push({ faultClass: 'literal-changed', description: `Window widened ×1000: \`${m[1]}\` ${m[3]} becomes ${Number(m[3]) * 1000}.`, replace: line.replace(m[0], `${m[1]}${m[2] === ':' ? ': ' : ' = '}${Number(m[3]) * 1000}`) });
|
|
83
288
|
}
|
|
289
|
+
|
|
290
|
+
if (ctx.fieldDrops !== false) {
|
|
291
|
+
out.push(...fieldDropProposals(lines, i));
|
|
292
|
+
out.push(...jsxProposals(lines, i));
|
|
293
|
+
}
|
|
294
|
+
out.push(...argumentSwapProposals(lines, i, params));
|
|
84
295
|
return out;
|
|
85
296
|
}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { basename, extname, join } from 'node:path';
|
|
3
|
-
import { proposalsForLine, functionHead } from './producers.mjs';
|
|
3
|
+
import { proposalsForLine, functionHead, functionParams } from './producers.mjs';
|
|
4
4
|
import { locate } from '../probe/inject.mjs';
|
|
5
5
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
6
6
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
7
7
|
|
|
8
8
|
const ANNOTATION = /@claim\s+([A-Za-z0-9]+(?:[._]?[A-Za-z0-9]+)*-[A-Za-z0-9._-]*[A-Za-z0-9])\b/;
|
|
9
|
+
// field-dropped is never proposed inside tests, fixtures or migrations: an
|
|
10
|
+
// object literal there is data, not a payload the product writes.
|
|
11
|
+
const NO_FIELD_DROPS = /\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)(__tests__|__fixtures__|__mocks__|fixtures|migrations)\//;
|
|
9
12
|
|
|
10
13
|
const idPart = (s) => s.replace(/[^A-Za-z0-9]+/g, '-').replace(/^-+|-+$/g, '').toUpperCase();
|
|
11
14
|
|
|
@@ -43,6 +46,8 @@ export function scaffoldFile({ projectDir, file, claimId, existingClaims, toolVe
|
|
|
43
46
|
let offset = 0;
|
|
44
47
|
let fn = null;
|
|
45
48
|
let fnDepth = 0;
|
|
49
|
+
let params = new Set();
|
|
50
|
+
const fieldDrops = !NO_FIELD_DROPS.test(file);
|
|
46
51
|
let depth = 0;
|
|
47
52
|
let pendingAnnotation = null;
|
|
48
53
|
const proposals = [];
|
|
@@ -54,9 +59,10 @@ export function scaffoldFile({ projectDir, file, claimId, existingClaims, toolVe
|
|
|
54
59
|
if (head) {
|
|
55
60
|
fn = head;
|
|
56
61
|
fnDepth = depth;
|
|
62
|
+
params = new Set(functionParams(line));
|
|
57
63
|
}
|
|
58
64
|
|
|
59
|
-
for (const p of proposalsForLine(lines, i)) {
|
|
65
|
+
for (const p of proposalsForLine(lines, i, { params, fieldDrops })) {
|
|
60
66
|
const find = line;
|
|
61
67
|
const { hits, occurrence } = anchorFor(source, find, offset);
|
|
62
68
|
const fault = { ...p, find, replace: p.replace, expectHits: hits, occurrence, line: i + 1, fn, annotation: pendingAnnotation };
|
|
@@ -65,7 +71,7 @@ export function scaffoldFile({ projectDir, file, claimId, existingClaims, toolVe
|
|
|
65
71
|
}
|
|
66
72
|
|
|
67
73
|
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
68
|
-
if (fn && depth <= fnDepth && !head) fn = null;
|
|
74
|
+
if (fn && depth <= fnDepth && !head) { fn = null; params = new Set(); }
|
|
69
75
|
if (pendingAnnotation && !ann && !head && line.trim() && !/^\s*(\/\/|\*|\/\*)/.test(line)) pendingAnnotation = null;
|
|
70
76
|
offset += line.length + 1;
|
|
71
77
|
});
|
package/src/status/status.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
|
8
8
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
9
9
|
import { sortForReport } from '../render.mjs';
|
|
10
10
|
import { computeChangedGate, defaultIgnorePath } from '../gate/changed.mjs';
|
|
11
|
+
import { headSha, isAncestor } from '../git.mjs';
|
|
11
12
|
|
|
12
13
|
export const faultContentHash = (fault) => sha256(`${fault.find}\n${fault.replace}`);
|
|
13
14
|
|
|
@@ -23,7 +24,13 @@ const P = (projectDir) => ({
|
|
|
23
24
|
* Reads the claims file, evidence, baseline and the working tree; never
|
|
24
25
|
* trusts a cached verdict whose inputs have changed.
|
|
25
26
|
*/
|
|
26
|
-
|
|
27
|
+
/**
|
|
28
|
+
* `evidence` overrides the project's own evidence file: a document taken on
|
|
29
|
+
* ANOTHER commit (CI's, fetched as an artifact) is legitimate input. Staleness
|
|
30
|
+
* is computed from the recorded input hashes exactly as for a local run, so a
|
|
31
|
+
* foreign document is trusted only for the faults whose inputs still match.
|
|
32
|
+
*/
|
|
33
|
+
export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt = new Date().toISOString(), paths = P(projectDir), max = 20, changedRef, includeDirty = false, evidence: evidenceOverride }) {
|
|
27
34
|
const rel = (p) => relative(projectDir, p) || '.';
|
|
28
35
|
const doc = {
|
|
29
36
|
schemaVersion: 1,
|
|
@@ -36,6 +43,7 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
36
43
|
stale: [],
|
|
37
44
|
changedFaults: [],
|
|
38
45
|
findings: [],
|
|
46
|
+
notes: [],
|
|
39
47
|
paths: {},
|
|
40
48
|
};
|
|
41
49
|
// ── Claim coverage of the change, when a reference is known ──
|
|
@@ -79,7 +87,8 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
79
87
|
return doc;
|
|
80
88
|
}
|
|
81
89
|
|
|
82
|
-
const
|
|
90
|
+
const evidenceFile = evidenceOverride ?? paths.evidence;
|
|
91
|
+
const hasEvidence = existsSync(evidenceFile);
|
|
83
92
|
const hasProvisional = existsSync(paths.provisional);
|
|
84
93
|
if (hasProvisional) doc.paths.provisionalEvidence = rel(paths.provisional);
|
|
85
94
|
if (!hasEvidence) {
|
|
@@ -93,11 +102,29 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
93
102
|
}
|
|
94
103
|
return doc;
|
|
95
104
|
}
|
|
96
|
-
doc.paths.evidence = rel(
|
|
97
|
-
|
|
105
|
+
doc.paths.evidence = rel(evidenceFile);
|
|
106
|
+
doc.evidenceSource = evidenceOverride ? 'provided' : 'local';
|
|
107
|
+
const evidence = readSpecDoc('evidence', evidenceFile);
|
|
108
|
+
// A verdict is about a commit. When the evidence describes a different one,
|
|
109
|
+
// say both: "looks right and is not" is the failure mode this prevents.
|
|
110
|
+
doc.evidenceHead = evidence.run.repo.snapshot ?? evidence.run.repo.head;
|
|
111
|
+
const localHead = (() => { try { return headSha(projectDir); } catch { return null; } })();
|
|
112
|
+
if (localHead) doc.head = localHead;
|
|
113
|
+
|
|
98
114
|
const baseline = existsSync(paths.baseline) ? readSpecDoc('baseline', paths.baseline) : undefined;
|
|
99
115
|
if (baseline) doc.paths.baseline = rel(paths.baseline);
|
|
100
116
|
|
|
117
|
+
// ── Baseline provenance: informational, never a state. A baseline frozen
|
|
118
|
+
// from a snapshot or a dirty tree names the PARENT of the commit that carries
|
|
119
|
+
// its tests; once HEAD moved on, say so and name the way to re-stamp. ──
|
|
120
|
+
if (baseline?.head) {
|
|
121
|
+
const head = headSha(projectDir);
|
|
122
|
+
if (head && head !== baseline.head) {
|
|
123
|
+
if (baseline.snapshot || baseline.dirty) doc.notes.push(`baseline was frozen from a ${baseline.snapshot ? 'working-tree snapshot' : 'dirty tree'} at ${baseline.head.slice(0, 7)}; HEAD is ${head.slice(0, 7)} — it predates the commit that carries its tests. After a clean probe of HEAD reproduces the same fingerprints: testguard baseline --restamp`);
|
|
124
|
+
else if (!isAncestor(projectDir, baseline.head, head)) doc.notes.push(`baseline head ${baseline.head.slice(0, 7)} is not an ancestor of HEAD ${head.slice(0, 7)} (rewritten or foreign history); re-probe and re-baseline if the fingerprints changed`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
101
128
|
// ── Staleness: does the evidence still describe this claims file and this tree? ──
|
|
102
129
|
const probed = new Map(evidence.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]));
|
|
103
130
|
for (const key of faultIndex.keys()) if (!probed.has(key)) doc.stale.push(`fault ${key} has never been probed`);
|
|
@@ -124,6 +151,14 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
124
151
|
const counts = {};
|
|
125
152
|
for (const r of evidence.records) counts[r.verdict] = (counts[r.verdict] ?? 0) + 1;
|
|
126
153
|
doc.counts.byVerdict = counts;
|
|
154
|
+
// L3 independence and observed flakiness, as numbers rather than adjectives.
|
|
155
|
+
const kills = evidence.records.filter((r) => r.verdict === 'killed');
|
|
156
|
+
if (kills.length) doc.counts.killedCoAuthored = kills.filter((r) => r.detail.independence?.class === 'co-authored').length;
|
|
157
|
+
const flaky = evidence.records.filter((r) => r.detail.flakeRate?.failures > 0);
|
|
158
|
+
if (flaky.length) {
|
|
159
|
+
doc.counts.flakyDefenderSets = new Set(flaky.map((r) => r.defenders.resolved.join('\n'))).size;
|
|
160
|
+
doc.counts.worstFlakeRate = Math.max(...flaky.map((r) => r.detail.flakeRate.failures / r.detail.flakeRate.runs));
|
|
161
|
+
}
|
|
127
162
|
doc.counts.new = g.new.length + g.belowFloor.length;
|
|
128
163
|
doc.counts.baselined = g.baselined.length;
|
|
129
164
|
const ranked = sortForReport([...g.new, ...g.belowFloor, ...g.baselined]);
|
|
@@ -170,7 +205,7 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
170
205
|
action: fixClaim ? 'probe' : 'write-test',
|
|
171
206
|
command: fixClaim
|
|
172
207
|
? `edit ${top.claimId}/${top.subjectId} in testguard.claims.json, then: testguard probe --claim ${top.claimId} --include-dirty`
|
|
173
|
-
: `write a test in ${where} that fails on ${top.claimId}/${top.subjectId} and passes on HEAD, then: testguard
|
|
208
|
+
: `write a test in ${where} that fails on ${top.claimId}/${top.subjectId} and passes on HEAD, then: testguard admit ${defenders[0] ?? '<that test file>'} --claim ${top.claimId}`,
|
|
174
209
|
why: whyByVerdict[top.verdict] ?? `${top.claimId}/${top.subjectId} is ${top.verdict}.`,
|
|
175
210
|
target: { claimId: top.claimId, subjectId: top.subjectId, ...(top.file ? { file: top.file } : {}), verdict: top.verdict },
|
|
176
211
|
};
|
|
@@ -187,7 +222,12 @@ export function computeStatus({ projectDir, toolVersion = '0.0.0', generatedAt =
|
|
|
187
222
|
}
|
|
188
223
|
|
|
189
224
|
export function renderStatus(doc) {
|
|
190
|
-
const lines = [
|
|
225
|
+
const lines = [];
|
|
226
|
+
if (doc.evidenceSource === 'provided') {
|
|
227
|
+
lines.push(`evidence: ${doc.paths.evidence} (provided)` + (doc.evidenceHead ? ` describes ${doc.evidenceHead.slice(0, 7)}` : '') + (doc.head && doc.evidenceHead && doc.head !== doc.evidenceHead ? `; this tree is ${doc.head.slice(0, 7)}` : ''));
|
|
228
|
+
}
|
|
229
|
+
lines.push(`state: ${doc.state}${doc.provisional ? ' (provisional)' : ''} — ${doc.counts.claims} claims / ${doc.counts.faults} faults` + (doc.counts.byVerdict ? `; ${Object.entries(doc.counts.byVerdict).map(([k, v]) => `${v} ${k}`).join(', ')}; ${doc.counts.new ?? 0} new, ${doc.counts.baselined ?? 0} baselined` : '')
|
|
230
|
+
);
|
|
191
231
|
if (doc.changes) {
|
|
192
232
|
const c = doc.changes;
|
|
193
233
|
lines.push(`changes: ${c.changed} file${c.changed === 1 ? '' : 's'} since ${c.ref}; ${c.evaluated} evaluated, ${c.excluded} excluded, ${c.uncovered.length} unclaimed`);
|
|
@@ -197,6 +237,7 @@ export function renderStatus(doc) {
|
|
|
197
237
|
}
|
|
198
238
|
for (const c of doc.changedFaults) lines.push(`CHANGED ${c.claimId}/${c.subjectId} edited since it ${c.previousVerdict} (${c.file})`);
|
|
199
239
|
for (const s of doc.stale.slice(0, 8)) lines.push(`stale ${s}`);
|
|
240
|
+
for (const n of doc.notes ?? []) lines.push(`note ${n}`);
|
|
200
241
|
lines.push(`next: [${doc.next.action}] ${doc.next.command}`);
|
|
201
242
|
lines.push(`why: ${doc.next.why}`);
|
|
202
243
|
return lines.join('\n');
|