tripwire-e2e 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +201 -3
- package/bin/tripwire.js +8 -0
- package/operators/_util.js +127 -0
- package/operators/agent-proposed.js +41 -0
- package/operators/auth-drop.js +30 -0
- package/operators/dead-control.js +23 -0
- package/operators/double-apply.js +20 -0
- package/operators/fake-success.js +21 -0
- package/operators/field-swap.js +42 -0
- package/operators/hide-control.js +30 -0
- package/operators/index.js +81 -0
- package/operators/item-drop.js +36 -0
- package/operators/item-dup.js +20 -0
- package/operators/label-swap.js +30 -0
- package/operators/missing-row-action.js +24 -0
- package/operators/number-drift.js +40 -0
- package/operators/rendered-item-drop.js +31 -0
- package/operators/rendered-item-dup.js +20 -0
- package/operators/rendered-number-drift.js +24 -0
- package/operators/silent-fail.js +19 -0
- package/operators/stale-read.js +34 -0
- package/operators/text-change.js +43 -0
- package/operators/validation-off.js +21 -0
- package/operators/wrong-link.js +28 -0
- package/operators/wrong-owner.js +41 -0
- package/package.json +41 -4
- package/policies/tripwire.clean.policy.yaml +23 -0
- package/policies/tripwire.mutant.policy.yaml +13 -0
- package/src/actions.js +59 -0
- package/src/agent.js +30 -0
- package/src/browser/mutate.js +156 -0
- package/src/browser/recorder.js +102 -0
- package/src/cli.js +41 -0
- package/src/commands/check.js +45 -0
- package/src/commands/coverage.js +110 -0
- package/src/commands/doctor.js +28 -0
- package/src/commands/fix.js +208 -0
- package/src/commands/list.js +24 -0
- package/src/commands/proxy.js +28 -0
- package/src/commands/report.js +11 -0
- package/src/commands/run.js +370 -0
- package/src/commands/sabotage.js +85 -0
- package/src/config.js +52 -0
- package/src/jsonpath.js +58 -0
- package/src/kane.js +158 -0
- package/src/ledger.js +53 -0
- package/src/mutant.js +26 -0
- package/src/orphans.js +40 -0
- package/src/profile.js +46 -0
- package/src/proxy.js +292 -0
- package/src/report.js +183 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// tripwire fix: the closed loop. Each surviving (or heal-masked) mutant goes to the coding agent,
|
|
2
|
+
// which appends one assertion step to one test. Kane then proves it twice:
|
|
3
|
+
// app clean -> the test must pass
|
|
4
|
+
// app broken -> the test must fail
|
|
5
|
+
// Both hold: the edit is kept and the proof (two Kane sessions and packs) is recorded.
|
|
6
|
+
// Otherwise: the test and its recording are restored and the agent tries again with the reason.
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { loadConfig } from '../config.js';
|
|
10
|
+
import { readLedger, latest, lastRunId, appendLedger } from '../ledger.js';
|
|
11
|
+
import { runTest } from '../kane.js';
|
|
12
|
+
import { runAgent } from '../agent.js';
|
|
13
|
+
import { lastScreenshot, imageExt } from '../report.js';
|
|
14
|
+
import { withProxy, resetApp, snapshotRecording } from './run.js';
|
|
15
|
+
import { generateAll } from '../../operators/index.js';
|
|
16
|
+
import { get } from '../jsonpath.js';
|
|
17
|
+
|
|
18
|
+
const HELP = `tripwire fix [options]
|
|
19
|
+
|
|
20
|
+
--agent claude|codex which coding agent strengthens the tests (default from config: claude)
|
|
21
|
+
--max-attempts <n> attempts per survivor (default 3)
|
|
22
|
+
--only <id,...> fix only these mutant ids (default: every survivor and masked mutant)
|
|
23
|
+
--limit <n> fix at most n mutants this time
|
|
24
|
+
--dir <path> state directory (default .tripwire)`;
|
|
25
|
+
|
|
26
|
+
const slug = (s) => s.replace(/[^\w.-]+/g, '_').slice(0, 80);
|
|
27
|
+
|
|
28
|
+
// What exactly changes on the page/response: the "before and after" for the agent.
|
|
29
|
+
export function beforeAfter(m, profile) {
|
|
30
|
+
const a = m.action;
|
|
31
|
+
const entry = profile?.tests.flatMap((t) => t.traffic).reverse().find((e) => e.method === m.match.method && e.path === m.match.path && e.json !== undefined);
|
|
32
|
+
if (a.type === 'json' && entry) {
|
|
33
|
+
return a.ops.map((op) => {
|
|
34
|
+
const before = get(entry.json, op.path);
|
|
35
|
+
const after = op.op === 'add' ? before + op.value : op.op === 'scale' ? Math.round(before * op.value * 100) / 100 : op.op === 'remove-index' ? `(list without item ${op.index}: ${before?.length - 1} items)` : op.op === 'dup-index' ? `(item ${op.index} repeated: ${before?.length + 1} items)` : op.op === 'swap' ? `(${op.a} and ${op.b} exchanged)` : op.op === 'push' ? '(an extra record owned by another user)' : JSON.stringify(op.value);
|
|
36
|
+
return `${m.match.method} ${m.match.path} ${op.path}: real ${JSON.stringify(before)} -> broken ${typeof after === 'string' ? after : JSON.stringify(after)}`;
|
|
37
|
+
}).join('\n');
|
|
38
|
+
}
|
|
39
|
+
if (a.type === 'respond') return `${m.match.method} ${m.match.path} answers ${a.status} without reaching the app${a.status < 400 ? ' (it looks successful, but nothing is saved)' : ''}`;
|
|
40
|
+
if (a.type === 'forward-twice') return `${m.match.method} ${m.match.path} reaches the app twice`;
|
|
41
|
+
if (a.type === 'status-map') return `${m.match.method} ${m.match.path}: a ${a.from.join('/')} validation error is turned into ${a.to}`;
|
|
42
|
+
if (a.type === 'stale-read') return `GET ${m.match.path} returns what it returned before the latest change`;
|
|
43
|
+
if (a.type === 'dom') return `On the page: ${JSON.stringify(a.dom)}`;
|
|
44
|
+
return JSON.stringify(a);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function buildPrompt({ m, test, testText, change, shotPath, lessons, previousFailure, masked }) {
|
|
48
|
+
return `You are strengthening an end-to-end Kane CLI test (a *_test.md file). Do not change any application code.
|
|
49
|
+
|
|
50
|
+
This bug was live in the app and the test below still PASSED:
|
|
51
|
+
${m.describe}
|
|
52
|
+
What changed: ${change}
|
|
53
|
+
${masked ? `\nWith strict replay the test failed, but with Kane's default adaptive healing it re-authored its steps and passed on the broken app. The new assertion must state the expected outcome so plainly that no re-authored path can satisfy it on the broken app.\n` : ''}
|
|
54
|
+
A good test would: ${m.expect ? m.expect.charAt(0).toLowerCase() + m.expect.slice(1) : 'check the outcome a user relies on'}.
|
|
55
|
+
${shotPath ? `Screenshot of the broken page at the end of the test: ${shotPath}\n` : ''}
|
|
56
|
+
Test file: ${test}
|
|
57
|
+
----- current contents -----
|
|
58
|
+
${testText}
|
|
59
|
+
----- end -----
|
|
60
|
+
|
|
61
|
+
Edit ${test} by APPENDING exactly one new step at the very end of the file:
|
|
62
|
+
## <short heading>
|
|
63
|
+
<one or two plain-English sentences: navigate if needed, then "Verify ..." with the exact expected value>
|
|
64
|
+
|
|
65
|
+
Rules:
|
|
66
|
+
- Only append. Do not modify, reorder, weaken or delete any existing line (Kane re-records every step after an edit, so editing earlier steps is expensive and is rejected).
|
|
67
|
+
- The new step must PASS on the real app and FAIL on the broken app described above.
|
|
68
|
+
- Use exact expected values a user would see (amounts with currency symbol, item names, counts, status words).
|
|
69
|
+
- Do not mention bugs, mutants or Tripwire in the test, and never name the broken value
|
|
70
|
+
(the wrong number, the extra record): assert the correct state, so the check stays general.
|
|
71
|
+
${lessons ? `\nAssertion patterns that worked before in this repo:\n${lessons}\n` : ''}${previousFailure ? `\nYour previous attempt was rejected: ${previousFailure}\nTry a different assertion.\n` : ''}
|
|
72
|
+
Reply with one line describing the assertion you added.`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// The agent may only append. Returns null when fine, or the reason it is not.
|
|
76
|
+
export function appendOnly(before, after) {
|
|
77
|
+
const b = before.replace(/\r\n/g, '\n').trimEnd();
|
|
78
|
+
const a = after.replace(/\r\n/g, '\n');
|
|
79
|
+
if (!a.startsWith(b)) return 'existing lines were changed; only appending a new final step is allowed';
|
|
80
|
+
const added = a.slice(b.length).trim();
|
|
81
|
+
if (!added) return 'nothing was added to the test';
|
|
82
|
+
if (!/^##\s+\S/m.test(added)) return 'the addition must be a new "## " step';
|
|
83
|
+
if ((added.match(/^##\s/gm) || []).length > 1) return 'add exactly one step';
|
|
84
|
+
if (/@import/.test(added)) return 'the new step must be prose, not an @import';
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// The step body (without its heading) that an agent appended.
|
|
89
|
+
export function addedStep(original, edited) {
|
|
90
|
+
const added = edited.replace(/\r\n/g, '\n').slice(original.replace(/\r\n/g, '\n').trimEnd().length).trim();
|
|
91
|
+
return added.replace(/^##[^\n]*\n?/, '').replace(/\s+/g, ' ').trim();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// A lesson is what a good test checks plus the assertion that proved it, never agent chatter.
|
|
95
|
+
export function appendLesson(dir, m, step) {
|
|
96
|
+
const f = path.join(dir, 'lessons.md');
|
|
97
|
+
const head = fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : '# Assertion patterns that caught real bugs in this repo\n\n';
|
|
98
|
+
const lesson = `- ${m.expect || 'Check the visible outcome'} (${m.operator}). Proven step: "${step.slice(0, 220)}"\n`;
|
|
99
|
+
if (!head.includes(lesson)) fs.writeFileSync(f, head + lesson);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export default async function fix(argv, parseArgs) {
|
|
103
|
+
const { values } = parseArgs({
|
|
104
|
+
args: argv,
|
|
105
|
+
options: {
|
|
106
|
+
agent: { type: 'string' }, 'max-attempts': { type: 'string' }, only: { type: 'string' },
|
|
107
|
+
limit: { type: 'string' }, dir: { type: 'string' }, help: { type: 'boolean', short: 'h' }
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
if (values.help) { console.log(HELP); return 0; }
|
|
111
|
+
const cfg = loadConfig();
|
|
112
|
+
const dir = path.resolve(values.dir || cfg.dir);
|
|
113
|
+
const agent = values.agent || cfg.agent || 'claude';
|
|
114
|
+
const maxAttempts = Number(values['max-attempts'] || 3);
|
|
115
|
+
const rows = readLedger(dir);
|
|
116
|
+
if (!rows.length) throw new Error('no ledger yet: run `tripwire run` first');
|
|
117
|
+
const profile = JSON.parse(fs.readFileSync(path.join(dir, 'traffic.json'), 'utf8'));
|
|
118
|
+
const defs = new Map(generateAll(profile, { dir }).candidates.map((m) => [m.id, m]));
|
|
119
|
+
const only = values.only ? new Set(values.only.split(',')) : null;
|
|
120
|
+
let todo = latest(rows, lastRunId(rows)).filter((r) => (r.status === 'survived' || r.status === 'masked') && (!only || only.has(r.id)));
|
|
121
|
+
if (values.limit) todo = todo.slice(0, Number(values.limit));
|
|
122
|
+
if (!todo.length) { console.log('Nothing to fix: no survivors in the latest run.'); return 0; }
|
|
123
|
+
const fixDir = path.join(dir, 'fix');
|
|
124
|
+
fs.mkdirSync(fixDir, { recursive: true });
|
|
125
|
+
|
|
126
|
+
return withProxy(cfg, async (tw, base) => {
|
|
127
|
+
let fixed = 0;
|
|
128
|
+
for (const row of todo) {
|
|
129
|
+
const m = defs.get(row.id);
|
|
130
|
+
if (!m) { console.log(`skip ${row.id}: no longer generated from the current baseline`); continue; }
|
|
131
|
+
const masked = row.status === 'masked';
|
|
132
|
+
// strengthen the test that touched the mutant and passed anyway (the killing test for masked ones)
|
|
133
|
+
const test = masked ? row.killed_by : (row.runs.find((x) => x.code === 0 && x.fired) || row.runs[0]).test;
|
|
134
|
+
const passedRun = row.runs.find((x) => x.test === test && x.code === 0) || row.runs[0];
|
|
135
|
+
const shot = lastScreenshot(passedRun?.pack);
|
|
136
|
+
const shotPath = shot ? path.join(fixDir, `${slug(m.id)}.${imageExt(shot)}`) : null;
|
|
137
|
+
if (shot) fs.writeFileSync(shotPath, shot);
|
|
138
|
+
const lessonsFile = path.join(dir, 'lessons.md');
|
|
139
|
+
const lessons = fs.existsSync(lessonsFile) ? fs.readFileSync(lessonsFile, 'utf8').split('\n').filter((l) => l.startsWith('- ')).slice(-12).join('\n') : '';
|
|
140
|
+
|
|
141
|
+
console.log(`\n${masked ? 'MASKED ' : 'SURVIVED'} ${m.describe}\n strengthening ${test} with ${agent}`);
|
|
142
|
+
let previousFailure = null;
|
|
143
|
+
let result = 'gave-up';
|
|
144
|
+
let proof = null;
|
|
145
|
+
let note = '';
|
|
146
|
+
let attempt = 0;
|
|
147
|
+
let added = '';
|
|
148
|
+
for (attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
149
|
+
const original = fs.readFileSync(test, 'utf8');
|
|
150
|
+
const restore = snapshotRecording(test);
|
|
151
|
+
const prompt = buildPrompt({ m, test, testText: original, change: beforeAfter(m, profile), shotPath, lessons, previousFailure, masked });
|
|
152
|
+
fs.writeFileSync(path.join(fixDir, `${slug(m.id)}_prompt_${attempt}.txt`), prompt);
|
|
153
|
+
const a = await runAgent(agent, prompt);
|
|
154
|
+
if (a.code === 127) throw new Error(`could not start the ${agent} CLI: ${a.stderr.trim()}`);
|
|
155
|
+
note = a.stdout.trim().split('\n').slice(-1)[0] || '';
|
|
156
|
+
const edited = fs.readFileSync(test, 'utf8');
|
|
157
|
+
const bad = appendOnly(original, edited);
|
|
158
|
+
if (bad) { restore(); previousFailure = bad; console.log(` attempt ${attempt}: rejected (${bad})`); continue; }
|
|
159
|
+
|
|
160
|
+
// proof 1: clean app, the new step is authored once and everything must pass
|
|
161
|
+
await tw.disarm(); await resetApp(cfg, base); await tw.reset(test, false);
|
|
162
|
+
const clean = await runTest(test, { timeout: cfg.timeout, logFile: path.join(fixDir, `${slug(m.id)}_clean_${attempt}.ndjson`) });
|
|
163
|
+
if (clean.code !== 0) {
|
|
164
|
+
restore();
|
|
165
|
+
previousFailure = `on the REAL app the strengthened test failed (exit ${clean.code}${clean.failureReason ? `: ${clean.failureReason}` : ''}). The assertion must hold on the working app.`;
|
|
166
|
+
console.log(` attempt ${attempt}: clean run failed, reverted`);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
// proof 2: broken app, replay must fail (healing on for masked mutants: it must survive a heal too)
|
|
170
|
+
await resetApp(cfg, base); await tw.reset(test, false); await tw.arm(m);
|
|
171
|
+
// keep the recording the clean run just committed; whatever Kane writes while the app is
|
|
172
|
+
// broken must not become the test's recording
|
|
173
|
+
const keep = snapshotRecording(test);
|
|
174
|
+
let broken;
|
|
175
|
+
try {
|
|
176
|
+
broken = await runTest(test, { heal: masked, timeout: cfg.timeout, logFile: path.join(fixDir, `${slug(m.id)}_mutant_${attempt}.ndjson`) });
|
|
177
|
+
} finally { keep(); }
|
|
178
|
+
const hits = (await tw.hit()).hits;
|
|
179
|
+
await tw.disarm();
|
|
180
|
+
if (broken.code === 1 && hits > 0) {
|
|
181
|
+
result = 'killed';
|
|
182
|
+
proof = { clean: { code: clean.code, session: clean.sessionId, pack: clean.pack }, mutant: { code: broken.code, session: broken.sessionId, pack: broken.pack, failure: broken.failureReason } };
|
|
183
|
+
added = addedStep(original, edited);
|
|
184
|
+
appendLesson(dir, m, added);
|
|
185
|
+
console.log(` attempt ${attempt}: proven. Real app: pass. Broken app: fail. ${note}`);
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
restore();
|
|
189
|
+
previousFailure = broken.code === 0
|
|
190
|
+
? 'the strengthened test still PASSED on the broken app, so the assertion does not observe the bug. Assert the exact value or state that the bug changes.'
|
|
191
|
+
: `the run on the broken app ended with exit ${broken.code} (${hits ? 'mutant fired' : 'mutant never fired'}), not a clean assertion failure.`;
|
|
192
|
+
console.log(` attempt ${attempt}: not proven (${previousFailure.split('.')[0]}), reverted`);
|
|
193
|
+
}
|
|
194
|
+
const rec = { at: new Date().toISOString(), id: m.id, describe: m.describe, operator: m.operator, status_before: row.status, test, agent, attempts: Math.min(attempt, maxAttempts), result, proof, note, added: added || undefined };
|
|
195
|
+
fs.appendFileSync(path.join(dir, 'fixes.jsonl'), JSON.stringify(rec) + '\n');
|
|
196
|
+
// The proof's mutant run is a Kane run in which the strengthened suite caught this bug.
|
|
197
|
+
// Appending a step cannot un-catch any other mutant (earlier failures still fail first),
|
|
198
|
+
// so the run's score is updated by a superseding ledger row, marked as a fix.
|
|
199
|
+
if (result === 'killed') appendLedger(dir, { ...row, status: 'killed', killed_by: test, fixed: true, fix: { at: rec.at, attempts: rec.attempts, proof } });
|
|
200
|
+
if (result === 'killed') fixed++;
|
|
201
|
+
}
|
|
202
|
+
console.log(`\n${fixed} of ${todo.length} survivors now caught, each proven by a passing clean run and a failing mutant run.`);
|
|
203
|
+
console.log('Re-score with: tripwire run');
|
|
204
|
+
const { writeReport } = await import('../report.js');
|
|
205
|
+
writeReport(dir);
|
|
206
|
+
return fixed === todo.length ? 0 : 1;
|
|
207
|
+
});
|
|
208
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// tripwire list: the operator catalogue, and the mutants the last baseline would produce.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { loadConfig } from '../config.js';
|
|
5
|
+
import { OPERATORS, generateAll } from '../../operators/index.js';
|
|
6
|
+
import { loadSaboteur } from '../../operators/agent-proposed.js';
|
|
7
|
+
|
|
8
|
+
export default async function list(argv, parseArgs) {
|
|
9
|
+
const { values } = parseArgs({ args: argv, options: { dir: { type: 'string' }, json: { type: 'boolean' }, help: { type: 'boolean', short: 'h' } } });
|
|
10
|
+
if (values.help) { console.log('tripwire list [--json] operators, then candidate mutants from .tripwire/traffic.json'); return 0; }
|
|
11
|
+
const dir = path.resolve(values.dir || loadConfig().dir);
|
|
12
|
+
const f = path.join(dir, 'traffic.json');
|
|
13
|
+
const profile = fs.existsSync(f) ? JSON.parse(fs.readFileSync(f, 'utf8')) : null;
|
|
14
|
+
const gen = profile ? generateAll(profile, { dir }) : null;
|
|
15
|
+
if (values.json) { console.log(JSON.stringify(gen ? gen.candidates : [], null, 2)); return 0; }
|
|
16
|
+
console.log('Operators');
|
|
17
|
+
for (const op of OPERATORS) console.log(` ${op.family.padEnd(10)} ${op.name.padEnd(19)} ${op.summary}${gen?.counts[op.name] ? ` (${gen.counts[op.name].generated ?? 0})` : ''}`);
|
|
18
|
+
const sab = loadSaboteur(dir);
|
|
19
|
+
if (sab.rejected.length) { console.log('\nRejected saboteur mutants'); for (const r of sab.rejected) console.log(` ${r.id}: ${r.errors.join('; ')}`); }
|
|
20
|
+
if (!gen) { console.log('\nNo baseline yet: run `tripwire run --baseline-only` to see concrete mutants.'); return 0; }
|
|
21
|
+
console.log(`\n${gen.candidates.length} candidate mutants from the last baseline`);
|
|
22
|
+
for (const m of gen.candidates) console.log(` ${m.operator.padEnd(19)} ${m.describe}`);
|
|
23
|
+
return 0;
|
|
24
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { createProxy } from '../proxy.js';
|
|
2
|
+
|
|
3
|
+
export default async function proxy(argv, parseArgs) {
|
|
4
|
+
const { values } = parseArgs({
|
|
5
|
+
args: argv,
|
|
6
|
+
options: {
|
|
7
|
+
target: { type: 'string', default: process.env.TRIPWIRE_TARGET || 'http://localhost:3000' },
|
|
8
|
+
port: { type: 'string', default: process.env.TRIPWIRE_PORT || '8080' },
|
|
9
|
+
record: { type: 'boolean', default: false },
|
|
10
|
+
quiet: { type: 'boolean', default: false },
|
|
11
|
+
help: { type: 'boolean', short: 'h' }
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
if (values.help) {
|
|
15
|
+
console.log('tripwire proxy --target <app url> [--port 8080] [--record] [--quiet]\n\nControl API: POST /_tw/arm, POST /_tw/disarm, POST /_tw/reset, GET /_tw/traffic, GET /_tw/dom, GET /_tw/hit');
|
|
16
|
+
return 0;
|
|
17
|
+
}
|
|
18
|
+
const p = createProxy({ target: values.target, port: Number(values.port), log: values.quiet ? () => {} : (m) => console.log(`[tripwire] ${m}`) });
|
|
19
|
+
p.state.record = values.record;
|
|
20
|
+
const port = await p.listen();
|
|
21
|
+
console.log(`tripwire proxy on http://localhost:${port} -> ${values.target}${values.record ? ' (recording)' : ''}`);
|
|
22
|
+
await new Promise((resolve) => {
|
|
23
|
+
process.on('SIGINT', resolve);
|
|
24
|
+
process.on('SIGTERM', resolve);
|
|
25
|
+
});
|
|
26
|
+
await p.close();
|
|
27
|
+
return 0;
|
|
28
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { loadConfig } from '../config.js';
|
|
3
|
+
import { writeReport } from '../report.js';
|
|
4
|
+
|
|
5
|
+
export default async function report(argv, parseArgs) {
|
|
6
|
+
const { values } = parseArgs({ args: argv, options: { dir: { type: 'string' }, help: { type: 'boolean', short: 'h' } } });
|
|
7
|
+
if (values.help) { console.log('tripwire report [--dir .tripwire] rebuild .tripwire/report/index.html and badge.svg from the ledger'); return 0; }
|
|
8
|
+
const dir = path.resolve(values.dir || loadConfig().dir);
|
|
9
|
+
console.log(writeReport(dir));
|
|
10
|
+
return 0;
|
|
11
|
+
}
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
// tripwire run: baseline -> generate -> replay each mutant -> classify -> ledger -> report.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { execSync, spawnSync } from 'node:child_process';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { createProxy, proxyClient, PROXY_CODE } from '../proxy.js';
|
|
7
|
+
import { loadConfig, findTests } from '../config.js';
|
|
8
|
+
import { runTest, verdict, metricFromVerdict, balance } from '../kane.js';
|
|
9
|
+
import { profileEntry, summarize, lineCounts } from '../profile.js';
|
|
10
|
+
import { generateAll, select } from '../../operators/index.js';
|
|
11
|
+
import { appendLedger, readLedger, latest, lastRunId, score } from '../ledger.js';
|
|
12
|
+
import { writeReport } from '../report.js';
|
|
13
|
+
import { findOrphanKaneBrowsers } from '../orphans.js';
|
|
14
|
+
|
|
15
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
16
|
+
export const POLICY_DIR = path.resolve(here, '../../policies');
|
|
17
|
+
|
|
18
|
+
const HELP = `tripwire run [options]
|
|
19
|
+
|
|
20
|
+
--tests <glob> test files (default from tripwire.config.json, else **/*_test.md)
|
|
21
|
+
--app <url> the real app behind the proxy (default http://localhost:3000)
|
|
22
|
+
--port <n> proxy port your tests point at (default 8080)
|
|
23
|
+
--max-mutants <n> cap on mutants replayed (default 40)
|
|
24
|
+
--only <ops> comma-separated operators or families (data, behaviour, interface, saboteur)
|
|
25
|
+
--since <commit> incremental: re-run only mutants whose tests changed; trust earlier kills
|
|
26
|
+
--no-heal-check skip the healing-on rerun that detects heal-masked mutants
|
|
27
|
+
--heal-budget <n> at most n heal checks per run (default 10)
|
|
28
|
+
--timeout <s> per-step kane timeout (default 180)
|
|
29
|
+
--baseline-only record the baseline and list mutants, replay nothing
|
|
30
|
+
--dry-run generate from the last baseline and print the plan
|
|
31
|
+
--resume continue the last run: reuse its baseline, skip mutants already in the ledger
|
|
32
|
+
--retry-errors re-run only the mutants that hit a Kane error in the last run
|
|
33
|
+
--redo <id,...> re-run these mutants of the last run (e.g. after fixing an operator)
|
|
34
|
+
-q, --quiet one line per mutant, no per-replay progress
|
|
35
|
+
--dir <path> state directory (default .tripwire)`;
|
|
36
|
+
|
|
37
|
+
export const normalizePath = (p) => p.replace(/\\/g, '/').replace(/^\.\//, '');
|
|
38
|
+
// Test files changed according to `git diff --name-only --relative` output.
|
|
39
|
+
export function changedTests(diffOutput) {
|
|
40
|
+
return new Set(String(diffOutput || '').split(/\r?\n/).map((s) => normalizePath(s.trim())).filter((s) => s.endsWith('_test.md')));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const readJson = (f) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch { return null; } };
|
|
44
|
+
const sh = (cmd) => { try { return execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim(); } catch { return null; } };
|
|
45
|
+
const stamp = () => new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
46
|
+
const slug = (s) => s.replace(/[^\w.-]+/g, '_').slice(0, 80);
|
|
47
|
+
|
|
48
|
+
export async function resetApp(cfg, base) {
|
|
49
|
+
if (!cfg.reset) return;
|
|
50
|
+
if (cfg.reset.cmd) {
|
|
51
|
+
const r = spawnSync(cfg.reset.cmd, { shell: true, stdio: 'ignore' });
|
|
52
|
+
if (r.status !== 0) throw new Error(`reset command failed: ${cfg.reset.cmd}`);
|
|
53
|
+
} else if (cfg.reset.url) {
|
|
54
|
+
// reset through the app directly, never through the proxy, so it is not recorded or mutated
|
|
55
|
+
const url = cfg.reset.url.startsWith('http') ? cfg.reset.url : new URL(cfg.reset.url, base).toString();
|
|
56
|
+
const r = await fetch(url, { method: cfg.reset.method || 'POST' });
|
|
57
|
+
if (!r.ok) throw new Error(`reset ${url} returned ${r.status}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function withProxy(cfg, fn) {
|
|
62
|
+
// Reuse a proxy that is already running on the port (e.g. `tripwire proxy` in another terminal).
|
|
63
|
+
const base = `http://localhost:${cfg.port}`;
|
|
64
|
+
let h = null;
|
|
65
|
+
try { h = await (await fetch(base + '/_tw/health')).json(); } catch { /* nothing tripwire-like on the port */ }
|
|
66
|
+
if (h?.ok) {
|
|
67
|
+
if (h.code !== PROXY_CODE) throw new Error(`a tripwire proxy from different code is already running on port ${cfg.port} (its code ${h.code || 'unknown'}, this one ${PROXY_CODE}). Stop it, or let tripwire start its own.`);
|
|
68
|
+
if (h.target && h.target.replace(/\/$/, '') !== String(cfg.app).replace(/\/$/, '')) throw new Error(`the proxy on port ${cfg.port} points at ${h.target}, not ${cfg.app}`);
|
|
69
|
+
return fn(proxyClient(base), base, false);
|
|
70
|
+
}
|
|
71
|
+
const p = createProxy({ target: cfg.app, port: Number(cfg.port), log: () => {} });
|
|
72
|
+
await new Promise((resolve, reject) => {
|
|
73
|
+
p.server.once('error', (e) => reject(e.code === 'EADDRINUSE' ? new Error(`port ${cfg.port} is in use by something that is not a tripwire proxy; set "port" in tripwire.config.json`) : e));
|
|
74
|
+
p.listen().then(resolve, reject);
|
|
75
|
+
});
|
|
76
|
+
try { return await fn(proxyClient(base), base, true); } finally { await p.close(); }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export default async function run(argv, parseArgs) {
|
|
80
|
+
const { values } = parseArgs({
|
|
81
|
+
args: argv,
|
|
82
|
+
options: {
|
|
83
|
+
tests: { type: 'string' }, app: { type: 'string' }, port: { type: 'string' },
|
|
84
|
+
'max-mutants': { type: 'string' }, only: { type: 'string' }, since: { type: 'string' },
|
|
85
|
+
'no-heal-check': { type: 'boolean' }, 'heal-budget': { type: 'string' }, timeout: { type: 'string' },
|
|
86
|
+
'baseline-only': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, resume: { type: 'boolean' }, 'retry-errors': { type: 'boolean' }, redo: { type: 'string' }, quiet: { type: 'boolean', short: 'q' }, dir: { type: 'string' },
|
|
87
|
+
help: { type: 'boolean', short: 'h' }
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
if (values.help) { console.log(HELP); return 0; }
|
|
91
|
+
const cfg = loadConfig();
|
|
92
|
+
for (const [k, v] of Object.entries({ tests: values.tests, app: values.app, port: values.port, dir: values.dir })) if (v) cfg[k] = v;
|
|
93
|
+
if (values['max-mutants']) cfg.maxMutants = Number(values['max-mutants']);
|
|
94
|
+
if (values.timeout) cfg.timeout = Number(values.timeout);
|
|
95
|
+
if (values['heal-budget']) cfg.healBudget = Number(values['heal-budget']);
|
|
96
|
+
if (values['no-heal-check']) cfg.healCheck = false;
|
|
97
|
+
const dir = path.resolve(cfg.dir);
|
|
98
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
99
|
+
// --retry-errors: re-run only the mutants that ended in a Kane error (exit 2/3) in the last run,
|
|
100
|
+
// on that run's baseline. Their error rows are superseded by the new rows (latest row wins).
|
|
101
|
+
// --redo <id,...>: the same, for named mutants (for example after fixing an operator).
|
|
102
|
+
if (values['retry-errors'] || values.redo) {
|
|
103
|
+
const rows = readLedger(dir);
|
|
104
|
+
const last = lastRunId(rows);
|
|
105
|
+
const redo = new Set((values.redo || '').split(',').map((s) => s.trim()).filter(Boolean));
|
|
106
|
+
const again = (r) => (values['retry-errors'] && r.status === 'error') || redo.has(r.id);
|
|
107
|
+
const errored = latest(rows, last).filter(again);
|
|
108
|
+
const pending = readJson(path.join(dir, 'current-run.json'));
|
|
109
|
+
if (redo.size && errored.length < redo.size) throw new Error(`not in the last run: ${[...redo].filter((id) => !errored.some((r) => r.id === id)).join(', ')}`);
|
|
110
|
+
if (!errored.length && !(pending?.run === last)) { console.log('No errored mutants in the last run.'); return 0; }
|
|
111
|
+
const kept = rows.filter((r) => !(r.run === last && again(r)));
|
|
112
|
+
fs.writeFileSync(path.join(dir, 'ledger.jsonl'), kept.map((r) => JSON.stringify(r)).join('\n') + (kept.length ? '\n' : ''));
|
|
113
|
+
// keep every mutant the run picked, including ones an interrupted run never reached
|
|
114
|
+
const prev = readJson(path.join(dir, 'current-run.json'));
|
|
115
|
+
const picked = [...new Set([...(prev?.run === last ? prev.picked : []), ...latest(rows, last).map((r) => r.id)])];
|
|
116
|
+
fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: last, picked }));
|
|
117
|
+
console.log(`Re-running ${errored.length} mutant(s) of run ${last}`);
|
|
118
|
+
values.resume = true;
|
|
119
|
+
}
|
|
120
|
+
const resumeFrom = values.resume ? readJson(path.join(dir, 'current-run.json')) : null;
|
|
121
|
+
if (values.resume && !resumeFrom) throw new Error('nothing to resume: no interrupted run found in ' + dir);
|
|
122
|
+
const runId = resumeFrom?.run || stamp();
|
|
123
|
+
const runDir = path.join(dir, 'runs', runId);
|
|
124
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
125
|
+
const commit = sh('git rev-parse --short HEAD');
|
|
126
|
+
|
|
127
|
+
const tests = findTests(cfg.tests);
|
|
128
|
+
if (!tests.length) throw new Error(`no *_test.md files match ${cfg.tests}`);
|
|
129
|
+
|
|
130
|
+
if (values['dry-run']) {
|
|
131
|
+
const profile = JSON.parse(fs.readFileSync(path.join(dir, 'traffic.json'), 'utf8'));
|
|
132
|
+
const plan = plan_(profile, cfg, dir, values.only);
|
|
133
|
+
for (const m of plan.picked) console.log(`${m.family.padEnd(10)} ${m.operator.padEnd(19)} ${m.describe} [${m.touches_tests.join(', ')}]`);
|
|
134
|
+
console.log(`\n${plan.picked.length} of ${plan.candidates.length} candidates selected (${plan.excluded.length} excluded as equivalent)`);
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const orphans = findOrphanKaneBrowsers(120);
|
|
139
|
+
if (orphans.length) console.log(`warning: ${orphans.length} Kane browser(s) from earlier runs are still open (oldest ${Math.round(Math.max(...orphans.map((o) => o.ageMinutes)) / 60)} h). They can change a shared app's state mid-run. Stop them: tripwire doctor --kill`);
|
|
140
|
+
const creditsBefore = await balance();
|
|
141
|
+
return withProxy(cfg, async (tw, base) => {
|
|
142
|
+
// ---------- 1. baseline (reused when resuming) ----------
|
|
143
|
+
let profile = { tests: [], created: new Date().toISOString(), commit };
|
|
144
|
+
let baseline = [];
|
|
145
|
+
if (resumeFrom) {
|
|
146
|
+
profile = readJson(path.join(dir, 'traffic.json'));
|
|
147
|
+
baseline = readJson(path.join(dir, 'baseline.json'))?.tests || [];
|
|
148
|
+
console.log(`\nResuming run ${runId}: baseline from ${profile.created}`);
|
|
149
|
+
} else console.log(`\nBaseline: ${tests.length} test(s) through the proxy at ${base}, healing off`);
|
|
150
|
+
for (const t of resumeFrom ? [] : tests) {
|
|
151
|
+
await tw.disarm();
|
|
152
|
+
await resetApp(cfg, base);
|
|
153
|
+
await tw.reset(t, true);
|
|
154
|
+
const r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(runDir, `baseline_${slug(t)}.ndjson`), onRetry: hiccup });
|
|
155
|
+
const v = await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml'));
|
|
156
|
+
const withoutChecks = metricFromVerdict(v.raw, 'metrics.tests_without_checks');
|
|
157
|
+
baseline.push({ test: t, code: r.code, durationMs: r.durationMs, pack: r.pack, verdict: v.verdict, testsWithoutChecks: withoutChecks, session: r.sessionId });
|
|
158
|
+
console.log(` ${r.code === 0 ? 'pass' : 'FAIL'} ${t} ${(r.durationMs / 1000).toFixed(1)}s${withoutChecks ? ' (no assertions!)' : ''}`);
|
|
159
|
+
if (r.code !== 0) {
|
|
160
|
+
console.error(`\nThe baseline must be green: ${t} exited ${r.code}. ${r.failureReason || ''}\nLog: ${path.join(runDir, `baseline_${slug(t)}.ndjson`)}`);
|
|
161
|
+
if (r.spawnError) console.error('Is kane-cli installed and on PATH? (npm install -g @testmuai/kane-cli)');
|
|
162
|
+
return 2;
|
|
163
|
+
}
|
|
164
|
+
profile.tests.push(profileEntry(t, await tw.traffic(), await tw.dom()));
|
|
165
|
+
}
|
|
166
|
+
await tw.reset(null, false);
|
|
167
|
+
if (!resumeFrom) {
|
|
168
|
+
fs.writeFileSync(path.join(dir, 'traffic.json'), JSON.stringify(profile, null, 1));
|
|
169
|
+
fs.writeFileSync(path.join(dir, 'baseline.json'), JSON.stringify({ run: runId, commit, tests: baseline, routes: summarize(profile) }, null, 2));
|
|
170
|
+
}
|
|
171
|
+
const noChecks = baseline.filter((b) => b.testsWithoutChecks > 0);
|
|
172
|
+
if (noChecks.length) console.log(`\n${noChecks.length} test(s) assert nothing at all: ${noChecks.map((b) => b.test).join(', ')}`);
|
|
173
|
+
|
|
174
|
+
// ---------- 2. generate ----------
|
|
175
|
+
const plan = plan_(profile, cfg, dir, values.only);
|
|
176
|
+
const { candidates, excluded, counts } = plan;
|
|
177
|
+
// when resuming, replay exactly the mutants the interrupted run had picked
|
|
178
|
+
const picked = resumeFrom ? resumeFrom.picked.map((id) => candidates.find((m) => m.id === id)).filter(Boolean) : plan.picked;
|
|
179
|
+
if (!values['baseline-only']) fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: runId, picked: picked.map((m) => m.id), since: values.since || null }, null, 2));
|
|
180
|
+
fs.writeFileSync(path.join(runDir, 'mutants.json'), JSON.stringify({ counts, picked, excluded: excluded.map((m) => m.id) }, null, 2));
|
|
181
|
+
console.log(`\nGenerated ${candidates.length} mutants; replaying ${picked.length}${excluded.length ? ` (${excluded.length} excluded as equivalent)` : ''}`);
|
|
182
|
+
if (values['baseline-only']) return 0;
|
|
183
|
+
|
|
184
|
+
// ---------- incremental ----------
|
|
185
|
+
let carried = [];
|
|
186
|
+
let todo = picked;
|
|
187
|
+
if (values.since) {
|
|
188
|
+
// --relative: paths relative to this folder, the same form touches_tests uses
|
|
189
|
+
const diff = sh(`git diff --name-only --relative ${values.since} --`);
|
|
190
|
+
if (diff === null) throw new Error(`--since ${values.since}: git diff failed (not a git repository, or unknown commit)`);
|
|
191
|
+
const changed = changedTests(diff);
|
|
192
|
+
const prev = new Map(latest(readLedger(dir)).map((r) => [r.id, r]));
|
|
193
|
+
todo = [];
|
|
194
|
+
for (const m of picked) {
|
|
195
|
+
const p = prev.get(m.id);
|
|
196
|
+
const touchedChanged = m.touches_tests.some((t) => changed.has(normalizePath(t)));
|
|
197
|
+
if (p && p.status === 'killed' && !touchedChanged) carried.push({ ...p, run: runId, carried: true });
|
|
198
|
+
else todo.push(m);
|
|
199
|
+
}
|
|
200
|
+
console.log(`Incremental since ${values.since}: ${changed.size} changed test(s); ${carried.length} earlier kills trusted, ${todo.length} to run`);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ---------- 3. replay each mutant ----------
|
|
204
|
+
const rows = [...carried];
|
|
205
|
+
for (const r of carried) appendLedger(dir, r);
|
|
206
|
+
let healChecks = 0;
|
|
207
|
+
if (resumeFrom) {
|
|
208
|
+
const done = latest(readLedger(dir), runId);
|
|
209
|
+
const doneIds = new Set(done.map((r) => r.id));
|
|
210
|
+
rows.push(...done);
|
|
211
|
+
healChecks = done.filter((r) => r.healChecked).length;
|
|
212
|
+
todo = todo.filter((m) => !doneIds.has(m.id));
|
|
213
|
+
console.log(`${done.length} mutants already done, ${todo.length} to go`);
|
|
214
|
+
}
|
|
215
|
+
for (const [i, m] of todo.entries()) {
|
|
216
|
+
if (!values.quiet) console.log(` [${String(i + 1).padStart(2)}/${todo.length}] ${m.family} · ${m.operator}: ${m.describe}`);
|
|
217
|
+
const row = await runMutant(m, { tw, cfg, base, runDir, profile, healAllowed: cfg.healCheck && healChecks < cfg.healBudget, progress: values.quiet ? undefined : (msg) => console.log(` ${msg}`) });
|
|
218
|
+
if (row.healChecked) healChecks++;
|
|
219
|
+
Object.assign(row, { run: runId, commit, at: new Date().toISOString() });
|
|
220
|
+
rows.push(row);
|
|
221
|
+
appendLedger(dir, row);
|
|
222
|
+
console.log(` ${values.quiet ? `[${String(i + 1).padStart(2)}/${todo.length}] ` : ' → '}${values.quiet ? label(row.status) : label(row.status).trim()} ${values.quiet ? m.describe : ''}${row.killed_by ? `${values.quiet ? ' ' : ''}(by ${row.killed_by})` : ''}`);
|
|
223
|
+
}
|
|
224
|
+
await tw.disarm();
|
|
225
|
+
fs.rmSync(path.join(dir, 'current-run.json'), { force: true });
|
|
226
|
+
|
|
227
|
+
// ---------- 4. score + report ----------
|
|
228
|
+
const creditsAfter = await balance();
|
|
229
|
+
const s = score(rows);
|
|
230
|
+
const meta = { run: runId, commit, creditsUsed: creditsBefore !== undefined && creditsAfter !== undefined ? Math.round((creditsBefore - creditsAfter) * 100) / 100 : null, baseline };
|
|
231
|
+
fs.writeFileSync(path.join(runDir, 'summary.json'), JSON.stringify({ ...meta, score: s }, null, 2));
|
|
232
|
+
const report = writeReport(dir);
|
|
233
|
+
console.log(`\nScore ${s.score ?? '-'}% killed ${s.killed} · survived ${s.survived} · masked by heal ${s.masked} · not reached ${s['not-reached']}${s.error ? ` · errors ${s.error}` : ''}`);
|
|
234
|
+
if (meta.creditsUsed !== null) console.log(`Credits used: ${meta.creditsUsed}`);
|
|
235
|
+
console.log(`Report: ${report}`);
|
|
236
|
+
if (s.survived + s.masked > 0) console.log('Next: tripwire fix (hands each survivor to your coding agent and proves the new assertion)');
|
|
237
|
+
return 0;
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function plan_(profile, cfg, dir, only) {
|
|
242
|
+
const eqFile = path.join(dir, 'equivalents.json');
|
|
243
|
+
const excludedIds = new Set(fs.existsSync(eqFile) ? JSON.parse(fs.readFileSync(eqFile, 'utf8')).map((e) => e.id) : []);
|
|
244
|
+
const { candidates, excluded, counts } = generateAll(profile, { dir, only: only ? only.split(',') : undefined, excluded: excludedIds });
|
|
245
|
+
return { candidates, excluded, counts, picked: select(candidates, cfg.maxMutants) };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// output-<stem>/ next to foo_test.md holds the recordings kane replays.
|
|
249
|
+
export function recordingDir(testFile) {
|
|
250
|
+
return path.join(path.dirname(testFile), 'output-' + path.basename(testFile).replace(/_test\.md$/, ''));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export function snapshotRecording(testFile) {
|
|
254
|
+
const src = recordingDir(testFile);
|
|
255
|
+
const md = fs.existsSync(testFile) ? fs.readFileSync(testFile) : null;
|
|
256
|
+
if (!fs.existsSync(src)) return () => { if (md) fs.writeFileSync(testFile, md); };
|
|
257
|
+
const tmp = fs.mkdtempSync(path.join(path.dirname(src), '.tw-snap-'));
|
|
258
|
+
fs.cpSync(src, path.join(tmp, 'rec'), { recursive: true });
|
|
259
|
+
return () => {
|
|
260
|
+
fs.rmSync(src, { recursive: true, force: true });
|
|
261
|
+
fs.cpSync(path.join(tmp, 'rec'), src, { recursive: true });
|
|
262
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
263
|
+
if (md) fs.writeFileSync(testFile, md);
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const hiccup = (n, msg) => console.log(` kane service hiccup (${msg}); waiting and retrying (${n}/3)`);
|
|
268
|
+
|
|
269
|
+
const label = (s) => ({ killed: 'killed ', survived: 'SURVIVED ', masked: 'MASKED ', 'not-reached': 'not reached', error: 'error ' }[s] || s);
|
|
270
|
+
|
|
271
|
+
// Lines of text the pages of one test showed in the clean baseline.
|
|
272
|
+
// line -> the most times the clean run showed it in one page snapshot. `counted` is false for
|
|
273
|
+
// baselines recorded before per-snapshot counts existed (then only new lines can be detected).
|
|
274
|
+
export function baselineLines(profile, test) {
|
|
275
|
+
const t = profile?.tests?.find((x) => x.test === test);
|
|
276
|
+
const max = new Map();
|
|
277
|
+
let counted = true;
|
|
278
|
+
for (const p of t?.pages || []) {
|
|
279
|
+
if (!p.maxCounts) counted = false;
|
|
280
|
+
const counts = p.maxCounts ? Object.entries(p.maxCounts) : [...lineCounts(p.text)].map(([l]) => [l, Infinity]);
|
|
281
|
+
for (const [l, n] of counts) if (!(max.get(l) >= n)) max.set(l, n);
|
|
282
|
+
}
|
|
283
|
+
max.counted = counted;
|
|
284
|
+
return max;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Did any page of this mutant run show a line the clean run never showed, or show a line more
|
|
288
|
+
// often than any clean snapshot did (a duplicated row)?
|
|
289
|
+
export function showedSomethingNew(dom, lines) {
|
|
290
|
+
for (const ev of dom) if (ev.kind === 'page') for (const [l, n] of lineCounts(ev.text)) {
|
|
291
|
+
if (!lines.has(l) || n > lines.get(l)) return true;
|
|
292
|
+
}
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const DUPLICATING = new Set(['item-dup', 'rendered-item-dup', 'double-apply']);
|
|
297
|
+
|
|
298
|
+
// Arm one mutant, replay the tests that touch it (healing off) until one fails.
|
|
299
|
+
export async function runMutant(m, { tw, cfg, base, runDir, healAllowed, progress = () => {}, profile }) {
|
|
300
|
+
const row = { id: m.id, family: m.family, operator: m.operator, describe: m.describe, expect: m.expect, touches_tests: m.touches_tests, status: null, fired: false, runs: [] };
|
|
301
|
+
const mdir = path.join(runDir, slug(m.id));
|
|
302
|
+
fs.mkdirSync(mdir, { recursive: true });
|
|
303
|
+
let infra = 0;
|
|
304
|
+
for (const t of m.touches_tests) {
|
|
305
|
+
let attempt = 0;
|
|
306
|
+
let r;
|
|
307
|
+
let hits;
|
|
308
|
+
do {
|
|
309
|
+
await tw.disarm();
|
|
310
|
+
await resetApp(cfg, base);
|
|
311
|
+
// the recorder stays on so we can tell whether the bug ever changed what the user saw
|
|
312
|
+
await tw.reset(t, true);
|
|
313
|
+
await tw.arm(m);
|
|
314
|
+
// Kane can rewrite a recording during any run against the broken app (observed even on
|
|
315
|
+
// failed runs), so every mutant run gets the real recording back afterwards.
|
|
316
|
+
const restore = snapshotRecording(t);
|
|
317
|
+
progress(`replaying ${t}${attempt ? ' (retry)' : ''}`);
|
|
318
|
+
try {
|
|
319
|
+
r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(mdir, `${slug(t)}_${attempt}.ndjson`), onRetry: hiccup });
|
|
320
|
+
} finally { restore(); }
|
|
321
|
+
hits = (await tw.hit()).hits;
|
|
322
|
+
r.visibleChange = profile ? showedSomethingNew(await tw.dom(), baselineLines(profile, t)) : undefined;
|
|
323
|
+
attempt++;
|
|
324
|
+
} while ((r.code === 2 || r.code === 3) && attempt < 2); // infrastructure problem: retry once, never a kill
|
|
325
|
+
const fired = hits > 0;
|
|
326
|
+
row.fired = row.fired || fired;
|
|
327
|
+
const v = r.code === 0 || r.code === 1 ? await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.mutant.policy.yaml')) : { verdict: 'unknown' };
|
|
328
|
+
row.runs.push({ test: t, code: r.code, fired, hits, durationMs: r.durationMs, pack: r.pack, session: r.sessionId, policy: v.verdict, visibleChange: r.visibleChange, failure: r.failureReason, failedStep: r.failedStep, kaneTriage: r.bugVerdict, credits: r.credits || undefined });
|
|
329
|
+
if (r.code === 2 || r.code === 3) { infra++; continue; }
|
|
330
|
+
if (r.code === 1 && fired) {
|
|
331
|
+
row.status = 'killed';
|
|
332
|
+
row.killed_by = t;
|
|
333
|
+
break;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
if (!row.status) {
|
|
337
|
+
if (!row.fired) row.status = infra === m.touches_tests.length ? 'error' : 'not-reached';
|
|
338
|
+
else row.status = infra && !row.runs.some((x) => x.code === 0) ? 'error' : 'survived';
|
|
339
|
+
}
|
|
340
|
+
// A survivor that fired but never changed a single line on any page is probably equivalent
|
|
341
|
+
// (for example a write response the page ignores because it re-reads the data). Flag it for a
|
|
342
|
+
// human; it still counts until someone adds it to equivalents.json with a reason.
|
|
343
|
+
// Without per-snapshot counts a duplicate cannot be seen, so duplicating mutants are not flagged.
|
|
344
|
+
const canTell = !DUPLICATING.has(m.operator) || profile?.tests?.every((t) => t.pages.every((p) => p.maxCounts));
|
|
345
|
+
if (row.status === 'survived' && canTell && row.runs.length && row.runs.every((x) => x.visibleChange === false)) {
|
|
346
|
+
row.suspectedEquivalent = 'The mutant fired, but no page showed anything the clean run did not show.';
|
|
347
|
+
}
|
|
348
|
+
// Heal check: would Kane's default healing have re-authored its way past this bug?
|
|
349
|
+
if (row.status === 'killed' && healAllowed) {
|
|
350
|
+
row.healChecked = true;
|
|
351
|
+
progress(`caught by ${row.killed_by}; heal check: same bug with Kane's default healing on`);
|
|
352
|
+
await tw.disarm();
|
|
353
|
+
await resetApp(cfg, base);
|
|
354
|
+
await tw.reset(row.killed_by, false);
|
|
355
|
+
await tw.arm(m);
|
|
356
|
+
// A run that passes after a heal commits the re-authored steps. Those steps were recorded
|
|
357
|
+
// against the broken app, so keep a copy of the real recording and put it back afterwards.
|
|
358
|
+
const restore = snapshotRecording(row.killed_by);
|
|
359
|
+
let h;
|
|
360
|
+
try {
|
|
361
|
+
h = await runTest(row.killed_by, { heal: true, timeout: cfg.timeout, logFile: path.join(mdir, `${slug(row.killed_by)}_heal.ndjson`) });
|
|
362
|
+
} finally { restore(); }
|
|
363
|
+
const hv = h.pack ? await verdict(h.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml')) : { raw: null };
|
|
364
|
+
const healed = metricFromVerdict(hv.raw, 'metrics.heal.healed_tests');
|
|
365
|
+
row.heal = { code: h.code, healedTests: healed, healEvents: h.healEvents, pack: h.pack, session: h.sessionId };
|
|
366
|
+
if (h.code === 0 && (healed > 0 || h.healEvents > 0 || h.mentionsHeal)) row.status = 'masked';
|
|
367
|
+
}
|
|
368
|
+
await tw.disarm();
|
|
369
|
+
return row;
|
|
370
|
+
}
|