tripwire-e2e 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,310 +1,370 @@
1
- // tripwire run: baseline -> generate -> replay each mutant -> classify -> ledger -> report.
2
- import fs from 'node:fs';
3
- import path from 'node:path';
4
- import { execSync, spawnSync } from 'node:child_process';
5
- import { fileURLToPath } from 'node:url';
6
- import { createProxy, proxyClient } from '../proxy.js';
7
- import { loadConfig, findTests } from '../config.js';
8
- import { runTest, verdict, metricFromVerdict, balance } from '../kane.js';
9
- import { profileEntry, summarize } from '../profile.js';
10
- import { generateAll, select } from '../../operators/index.js';
11
- import { appendLedger, readLedger, latest, lastRunId, score } from '../ledger.js';
12
- import { writeReport } from '../report.js';
13
-
14
- const here = path.dirname(fileURLToPath(import.meta.url));
15
- export const POLICY_DIR = path.resolve(here, '../../policies');
16
-
17
- const HELP = `tripwire run [options]
18
-
19
- --tests <glob> test files (default from tripwire.config.json, else **/*_test.md)
20
- --app <url> the real app behind the proxy (default http://localhost:3000)
21
- --port <n> proxy port your tests point at (default 8080)
22
- --max-mutants <n> cap on mutants replayed (default 40)
23
- --only <ops> comma-separated operators or families (data, behaviour, interface, saboteur)
24
- --since <commit> incremental: re-run only mutants whose tests changed; trust earlier kills
25
- --no-heal-check skip the healing-on rerun that detects heal-masked mutants
26
- --heal-budget <n> at most n heal checks per run (default 10)
27
- --timeout <s> per-step kane timeout (default 180)
28
- --baseline-only record the baseline and list mutants, replay nothing
29
- --dry-run generate from the last baseline and print the plan
30
- --resume continue the last run: reuse its baseline, skip mutants already in the ledger
31
- --retry-errors re-run only the mutants that hit a Kane error in the last run
32
- -q, --quiet one line per mutant, no per-replay progress
33
- --dir <path> state directory (default .tripwire)`;
34
-
35
- const readJson = (f) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch { return null; } };
36
- const sh = (cmd) => { try { return execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim(); } catch { return null; } };
37
- const stamp = () => new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
38
- const slug = (s) => s.replace(/[^\w.-]+/g, '_').slice(0, 80);
39
-
40
- export async function resetApp(cfg, base) {
41
- if (!cfg.reset) return;
42
- if (cfg.reset.cmd) {
43
- const r = spawnSync(cfg.reset.cmd, { shell: true, stdio: 'ignore' });
44
- if (r.status !== 0) throw new Error(`reset command failed: ${cfg.reset.cmd}`);
45
- } else if (cfg.reset.url) {
46
- // reset through the app directly, never through the proxy, so it is not recorded or mutated
47
- const url = cfg.reset.url.startsWith('http') ? cfg.reset.url : new URL(cfg.reset.url, base).toString();
48
- const r = await fetch(url, { method: cfg.reset.method || 'POST' });
49
- if (!r.ok) throw new Error(`reset ${url} returned ${r.status}`);
50
- }
51
- }
52
-
53
- export async function withProxy(cfg, fn) {
54
- // Reuse a proxy that is already running on the port (e.g. `tripwire proxy` in another terminal).
55
- const base = `http://localhost:${cfg.port}`;
56
- try {
57
- const h = await (await fetch(base + '/_tw/health')).json();
58
- if (h.ok) return fn(proxyClient(base), base, false);
59
- } catch { /* none running */ }
60
- const p = createProxy({ target: cfg.app, port: Number(cfg.port), log: () => {} });
61
- await p.listen();
62
- try { return await fn(proxyClient(base), base, true); } finally { await p.close(); }
63
- }
64
-
65
- export default async function run(argv, parseArgs) {
66
- const { values } = parseArgs({
67
- args: argv,
68
- options: {
69
- tests: { type: 'string' }, app: { type: 'string' }, port: { type: 'string' },
70
- 'max-mutants': { type: 'string' }, only: { type: 'string' }, since: { type: 'string' },
71
- 'no-heal-check': { type: 'boolean' }, 'heal-budget': { type: 'string' }, timeout: { type: 'string' },
72
- 'baseline-only': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, resume: { type: 'boolean' }, 'retry-errors': { type: 'boolean' }, quiet: { type: 'boolean', short: 'q' }, dir: { type: 'string' },
73
- help: { type: 'boolean', short: 'h' }
74
- }
75
- });
76
- if (values.help) { console.log(HELP); return 0; }
77
- const cfg = loadConfig();
78
- for (const [k, v] of Object.entries({ tests: values.tests, app: values.app, port: values.port, dir: values.dir })) if (v) cfg[k] = v;
79
- if (values['max-mutants']) cfg.maxMutants = Number(values['max-mutants']);
80
- if (values.timeout) cfg.timeout = Number(values.timeout);
81
- if (values['heal-budget']) cfg.healBudget = Number(values['heal-budget']);
82
- if (values['no-heal-check']) cfg.healCheck = false;
83
- const dir = path.resolve(cfg.dir);
84
- fs.mkdirSync(dir, { recursive: true });
85
- // --retry-errors: re-run only the mutants that ended in a Kane error (exit 2/3) in the last run,
86
- // on that run's baseline. Their error rows are superseded by the new rows (latest row wins).
87
- if (values['retry-errors']) {
88
- const rows = readLedger(dir);
89
- const last = lastRunId(rows);
90
- const errored = latest(rows, last).filter((r) => r.status === 'error');
91
- const pending = readJson(path.join(dir, 'current-run.json'));
92
- if (!errored.length && !(pending?.run === last)) { console.log('No errored mutants in the last run.'); return 0; }
93
- const kept = rows.filter((r) => !(r.run === last && r.status === 'error'));
94
- fs.writeFileSync(path.join(dir, 'ledger.jsonl'), kept.map((r) => JSON.stringify(r)).join('\n') + (kept.length ? '\n' : ''));
95
- // keep every mutant the run picked, including ones an interrupted run never reached
96
- const prev = readJson(path.join(dir, 'current-run.json'));
97
- const picked = [...new Set([...(prev?.run === last ? prev.picked : []), ...latest(rows, last).map((r) => r.id)])];
98
- fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: last, picked }));
99
- console.log(`Retrying ${errored.length} errored mutants of run ${last}`);
100
- values.resume = true;
101
- }
102
- const resumeFrom = values.resume ? readJson(path.join(dir, 'current-run.json')) : null;
103
- if (values.resume && !resumeFrom) throw new Error('nothing to resume: no interrupted run found in ' + dir);
104
- const runId = resumeFrom?.run || stamp();
105
- const runDir = path.join(dir, 'runs', runId);
106
- fs.mkdirSync(runDir, { recursive: true });
107
- const commit = sh('git rev-parse --short HEAD');
108
-
109
- const tests = findTests(cfg.tests);
110
- if (!tests.length) throw new Error(`no *_test.md files match ${cfg.tests}`);
111
-
112
- if (values['dry-run']) {
113
- const profile = JSON.parse(fs.readFileSync(path.join(dir, 'traffic.json'), 'utf8'));
114
- const plan = plan_(profile, cfg, dir, values.only);
115
- for (const m of plan.picked) console.log(`${m.family.padEnd(10)} ${m.operator.padEnd(19)} ${m.describe} [${m.touches_tests.join(', ')}]`);
116
- console.log(`\n${plan.picked.length} of ${plan.candidates.length} candidates selected (${plan.excluded.length} excluded as equivalent)`);
117
- return 0;
118
- }
119
-
120
- const creditsBefore = await balance();
121
- return withProxy(cfg, async (tw, base) => {
122
- // ---------- 1. baseline (reused when resuming) ----------
123
- let profile = { tests: [], created: new Date().toISOString(), commit };
124
- let baseline = [];
125
- if (resumeFrom) {
126
- profile = readJson(path.join(dir, 'traffic.json'));
127
- baseline = readJson(path.join(dir, 'baseline.json'))?.tests || [];
128
- console.log(`\nResuming run ${runId}: baseline from ${profile.created}`);
129
- } else console.log(`\nBaseline: ${tests.length} test(s) through the proxy at ${base}, healing off`);
130
- for (const t of resumeFrom ? [] : tests) {
131
- await tw.disarm();
132
- await resetApp(cfg, base);
133
- await tw.reset(t, true);
134
- const r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(runDir, `baseline_${slug(t)}.ndjson`), onRetry: hiccup });
135
- const v = await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml'));
136
- const withoutChecks = metricFromVerdict(v.raw, 'metrics.tests_without_checks');
137
- baseline.push({ test: t, code: r.code, durationMs: r.durationMs, pack: r.pack, verdict: v.verdict, testsWithoutChecks: withoutChecks, session: r.sessionId });
138
- console.log(` ${r.code === 0 ? 'pass' : 'FAIL'} ${t} ${(r.durationMs / 1000).toFixed(1)}s${withoutChecks ? ' (no assertions!)' : ''}`);
139
- if (r.code !== 0) {
140
- console.error(`\nThe baseline must be green: ${t} exited ${r.code}. ${r.failureReason || ''}\nLog: ${path.join(runDir, `baseline_${slug(t)}.ndjson`)}`);
141
- if (r.spawnError) console.error('Is kane-cli installed and on PATH? (npm install -g @testmuai/kane-cli)');
142
- return 2;
143
- }
144
- profile.tests.push(profileEntry(t, await tw.traffic(), await tw.dom()));
145
- }
146
- await tw.reset(null, false);
147
- if (!resumeFrom) {
148
- fs.writeFileSync(path.join(dir, 'traffic.json'), JSON.stringify(profile, null, 1));
149
- fs.writeFileSync(path.join(dir, 'baseline.json'), JSON.stringify({ run: runId, commit, tests: baseline, routes: summarize(profile) }, null, 2));
150
- }
151
- const noChecks = baseline.filter((b) => b.testsWithoutChecks > 0);
152
- if (noChecks.length) console.log(`\n${noChecks.length} test(s) assert nothing at all: ${noChecks.map((b) => b.test).join(', ')}`);
153
-
154
- // ---------- 2. generate ----------
155
- const plan = plan_(profile, cfg, dir, values.only);
156
- const { candidates, excluded, counts } = plan;
157
- // when resuming, replay exactly the mutants the interrupted run had picked
158
- const picked = resumeFrom ? resumeFrom.picked.map((id) => candidates.find((m) => m.id === id)).filter(Boolean) : plan.picked;
159
- if (!values['baseline-only']) fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: runId, picked: picked.map((m) => m.id), since: values.since || null }, null, 2));
160
- fs.writeFileSync(path.join(runDir, 'mutants.json'), JSON.stringify({ counts, picked, excluded: excluded.map((m) => m.id) }, null, 2));
161
- console.log(`\nGenerated ${candidates.length} mutants; replaying ${picked.length}${excluded.length ? ` (${excluded.length} excluded as equivalent)` : ''}`);
162
- if (values['baseline-only']) return 0;
163
-
164
- // ---------- incremental ----------
165
- let carried = [];
166
- let todo = picked;
167
- if (values.since) {
168
- const changed = new Set((sh(`git diff --name-only ${values.since} --`) || '').split('\n').filter(Boolean));
169
- const prev = new Map(latest(readLedger(dir)).map((r) => [r.id, r]));
170
- todo = [];
171
- for (const m of picked) {
172
- const p = prev.get(m.id);
173
- const touchedChanged = m.touches_tests.some((t) => changed.has(t));
174
- if (p && p.status === 'killed' && !touchedChanged) carried.push({ ...p, run: runId, carried: true });
175
- else todo.push(m);
176
- }
177
- console.log(`Incremental since ${values.since}: ${carried.length} earlier kills trusted, ${todo.length} to run`);
178
- }
179
-
180
- // ---------- 3. replay each mutant ----------
181
- const rows = [...carried];
182
- for (const r of carried) appendLedger(dir, r);
183
- let healChecks = 0;
184
- if (resumeFrom) {
185
- const done = latest(readLedger(dir), runId);
186
- const doneIds = new Set(done.map((r) => r.id));
187
- rows.push(...done);
188
- healChecks = done.filter((r) => r.healChecked).length;
189
- todo = todo.filter((m) => !doneIds.has(m.id));
190
- console.log(`${done.length} mutants already done, ${todo.length} to go`);
191
- }
192
- for (const [i, m] of todo.entries()) {
193
- if (!values.quiet) console.log(` [${String(i + 1).padStart(2)}/${todo.length}] ${m.family} · ${m.operator}: ${m.describe}`);
194
- const row = await runMutant(m, { tw, cfg, base, runDir, healAllowed: cfg.healCheck && healChecks < cfg.healBudget, progress: values.quiet ? undefined : (msg) => console.log(` ${msg}`) });
195
- if (row.healChecked) healChecks++;
196
- Object.assign(row, { run: runId, commit, at: new Date().toISOString() });
197
- rows.push(row);
198
- appendLedger(dir, row);
199
- console.log(` ${values.quiet ? `[${String(i + 1).padStart(2)}/${todo.length}] ` : ' → '}${values.quiet ? label(row.status) : label(row.status).trim()} ${values.quiet ? m.describe : ''}${row.killed_by ? `${values.quiet ? ' ' : ''}(by ${row.killed_by})` : ''}`);
200
- }
201
- await tw.disarm();
202
- fs.rmSync(path.join(dir, 'current-run.json'), { force: true });
203
-
204
- // ---------- 4. score + report ----------
205
- const creditsAfter = await balance();
206
- const s = score(rows);
207
- const meta = { run: runId, commit, creditsUsed: creditsBefore !== undefined && creditsAfter !== undefined ? Math.round((creditsBefore - creditsAfter) * 100) / 100 : null, baseline };
208
- fs.writeFileSync(path.join(runDir, 'summary.json'), JSON.stringify({ ...meta, score: s }, null, 2));
209
- const report = writeReport(dir);
210
- console.log(`\nScore ${s.score ?? '-'}% killed ${s.killed} · survived ${s.survived} · masked by heal ${s.masked} · not reached ${s['not-reached']}${s.error ? ` · errors ${s.error}` : ''}`);
211
- if (meta.creditsUsed !== null) console.log(`Credits used: ${meta.creditsUsed}`);
212
- console.log(`Report: ${report}`);
213
- if (s.survived + s.masked > 0) console.log('Next: tripwire fix (hands each survivor to your coding agent and proves the new assertion)');
214
- return 0;
215
- });
216
- }
217
-
218
- function plan_(profile, cfg, dir, only) {
219
- const eqFile = path.join(dir, 'equivalents.json');
220
- const excludedIds = new Set(fs.existsSync(eqFile) ? JSON.parse(fs.readFileSync(eqFile, 'utf8')).map((e) => e.id) : []);
221
- const { candidates, excluded, counts } = generateAll(profile, { dir, only: only ? only.split(',') : undefined, excluded: excludedIds });
222
- return { candidates, excluded, counts, picked: select(candidates, cfg.maxMutants) };
223
- }
224
-
225
- // output-<stem>/ next to foo_test.md holds the recordings kane replays.
226
- export function recordingDir(testFile) {
227
- return path.join(path.dirname(testFile), 'output-' + path.basename(testFile).replace(/_test\.md$/, ''));
228
- }
229
-
230
- export function snapshotRecording(testFile) {
231
- const src = recordingDir(testFile);
232
- const md = fs.existsSync(testFile) ? fs.readFileSync(testFile) : null;
233
- if (!fs.existsSync(src)) return () => { if (md) fs.writeFileSync(testFile, md); };
234
- const tmp = fs.mkdtempSync(path.join(path.dirname(src), '.tw-snap-'));
235
- fs.cpSync(src, path.join(tmp, 'rec'), { recursive: true });
236
- return () => {
237
- fs.rmSync(src, { recursive: true, force: true });
238
- fs.cpSync(path.join(tmp, 'rec'), src, { recursive: true });
239
- fs.rmSync(tmp, { recursive: true, force: true });
240
- if (md) fs.writeFileSync(testFile, md);
241
- };
242
- }
243
-
244
- const hiccup = (n, msg) => console.log(` kane service hiccup (${msg}); waiting and retrying (${n}/3)`);
245
-
246
- const label = (s) => ({ killed: 'killed ', survived: 'SURVIVED ', masked: 'MASKED ', 'not-reached': 'not reached', error: 'error ' }[s] || s);
247
-
248
- // Arm one mutant, replay the tests that touch it (healing off) until one fails.
249
- export async function runMutant(m, { tw, cfg, base, runDir, healAllowed, progress = () => {} }) {
250
- const row = { id: m.id, family: m.family, operator: m.operator, describe: m.describe, expect: m.expect, touches_tests: m.touches_tests, status: null, fired: false, runs: [] };
251
- const mdir = path.join(runDir, slug(m.id));
252
- fs.mkdirSync(mdir, { recursive: true });
253
- let infra = 0;
254
- for (const t of m.touches_tests) {
255
- let attempt = 0;
256
- let r;
257
- let hits;
258
- do {
259
- await tw.disarm();
260
- await resetApp(cfg, base);
261
- await tw.reset(t, false);
262
- await tw.arm(m);
263
- // Kane can rewrite a recording during any run against the broken app (observed even on
264
- // failed runs), so every mutant run gets the real recording back afterwards.
265
- const restore = snapshotRecording(t);
266
- progress(`replaying ${t}${attempt ? ' (retry)' : ''}`);
267
- try {
268
- r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(mdir, `${slug(t)}_${attempt}.ndjson`), onRetry: hiccup });
269
- } finally { restore(); }
270
- hits = (await tw.hit()).hits;
271
- attempt++;
272
- } while ((r.code === 2 || r.code === 3) && attempt < 2); // infrastructure problem: retry once, never a kill
273
- const fired = hits > 0;
274
- row.fired = row.fired || fired;
275
- const v = r.code === 0 || r.code === 1 ? await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.mutant.policy.yaml')) : { verdict: 'unknown' };
276
- row.runs.push({ test: t, code: r.code, fired, hits, durationMs: r.durationMs, pack: r.pack, session: r.sessionId, policy: v.verdict, failure: r.failureReason, failedStep: r.failedStep, kaneTriage: r.bugVerdict, credits: r.credits || undefined });
277
- if (r.code === 2 || r.code === 3) { infra++; continue; }
278
- if (r.code === 1 && fired) {
279
- row.status = 'killed';
280
- row.killed_by = t;
281
- break;
282
- }
283
- }
284
- if (!row.status) {
285
- if (!row.fired) row.status = infra === m.touches_tests.length ? 'error' : 'not-reached';
286
- else row.status = infra && !row.runs.some((x) => x.code === 0) ? 'error' : 'survived';
287
- }
288
- // Heal check: would Kane's default healing have re-authored its way past this bug?
289
- if (row.status === 'killed' && healAllowed) {
290
- row.healChecked = true;
291
- progress(`caught by ${row.killed_by}; heal check: same bug with Kane's default healing on`);
292
- await tw.disarm();
293
- await resetApp(cfg, base);
294
- await tw.reset(row.killed_by, false);
295
- await tw.arm(m);
296
- // A run that passes after a heal commits the re-authored steps. Those steps were recorded
297
- // against the broken app, so keep a copy of the real recording and put it back afterwards.
298
- const restore = snapshotRecording(row.killed_by);
299
- let h;
300
- try {
301
- h = await runTest(row.killed_by, { heal: true, timeout: cfg.timeout, logFile: path.join(mdir, `${slug(row.killed_by)}_heal.ndjson`) });
302
- } finally { restore(); }
303
- const hv = h.pack ? await verdict(h.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml')) : { raw: null };
304
- const healed = metricFromVerdict(hv.raw, 'metrics.heal.healed_tests');
305
- row.heal = { code: h.code, healedTests: healed, healEvents: h.healEvents, pack: h.pack, session: h.sessionId };
306
- if (h.code === 0 && (healed > 0 || h.healEvents > 0 || h.mentionsHeal)) row.status = 'masked';
307
- }
308
- await tw.disarm();
309
- return row;
310
- }
1
+ // tripwire run: baseline -> generate -> replay each mutant -> classify -> ledger -> report.
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ import { execSync, spawnSync } from 'node:child_process';
5
+ import { fileURLToPath } from 'node:url';
6
+ import { createProxy, proxyClient, PROXY_CODE } from '../proxy.js';
7
+ import { loadConfig, findTests } from '../config.js';
8
+ import { runTest, verdict, metricFromVerdict, balance } from '../kane.js';
9
+ import { profileEntry, summarize, lineCounts } from '../profile.js';
10
+ import { generateAll, select } from '../../operators/index.js';
11
+ import { appendLedger, readLedger, latest, lastRunId, score } from '../ledger.js';
12
+ import { writeReport } from '../report.js';
13
+ import { findOrphanKaneBrowsers } from '../orphans.js';
14
+
15
+ const here = path.dirname(fileURLToPath(import.meta.url));
16
+ export const POLICY_DIR = path.resolve(here, '../../policies');
17
+
18
+ const HELP = `tripwire run [options]
19
+
20
+ --tests <glob> test files (default from tripwire.config.json, else **/*_test.md)
21
+ --app <url> the real app behind the proxy (default http://localhost:3000)
22
+ --port <n> proxy port your tests point at (default 8080)
23
+ --max-mutants <n> cap on mutants replayed (default 40)
24
+ --only <ops> comma-separated operators or families (data, behaviour, interface, saboteur)
25
+ --since <commit> incremental: re-run only mutants whose tests changed; trust earlier kills
26
+ --no-heal-check skip the healing-on rerun that detects heal-masked mutants
27
+ --heal-budget <n> at most n heal checks per run (default 10)
28
+ --timeout <s> per-step kane timeout (default 180)
29
+ --baseline-only record the baseline and list mutants, replay nothing
30
+ --dry-run generate from the last baseline and print the plan
31
+ --resume continue the last run: reuse its baseline, skip mutants already in the ledger
32
+ --retry-errors re-run only the mutants that hit a Kane error in the last run
33
+ --redo <id,...> re-run these mutants of the last run (e.g. after fixing an operator)
34
+ -q, --quiet one line per mutant, no per-replay progress
35
+ --dir <path> state directory (default .tripwire)`;
36
+
37
+ export const normalizePath = (p) => p.replace(/\\/g, '/').replace(/^\.\//, '');
38
+ // Test files changed according to `git diff --name-only --relative` output.
39
+ export function changedTests(diffOutput) {
40
+ return new Set(String(diffOutput || '').split(/\r?\n/).map((s) => normalizePath(s.trim())).filter((s) => s.endsWith('_test.md')));
41
+ }
42
+
43
+ const readJson = (f) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch { return null; } };
44
+ const sh = (cmd) => { try { return execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim(); } catch { return null; } };
45
+ const stamp = () => new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
46
+ const slug = (s) => s.replace(/[^\w.-]+/g, '_').slice(0, 80);
47
+
48
+ export async function resetApp(cfg, base) {
49
+ if (!cfg.reset) return;
50
+ if (cfg.reset.cmd) {
51
+ const r = spawnSync(cfg.reset.cmd, { shell: true, stdio: 'ignore' });
52
+ if (r.status !== 0) throw new Error(`reset command failed: ${cfg.reset.cmd}`);
53
+ } else if (cfg.reset.url) {
54
+ // reset through the app directly, never through the proxy, so it is not recorded or mutated
55
+ const url = cfg.reset.url.startsWith('http') ? cfg.reset.url : new URL(cfg.reset.url, base).toString();
56
+ const r = await fetch(url, { method: cfg.reset.method || 'POST' });
57
+ if (!r.ok) throw new Error(`reset ${url} returned ${r.status}`);
58
+ }
59
+ }
60
+
61
+ export async function withProxy(cfg, fn) {
62
+ // Reuse a proxy that is already running on the port (e.g. `tripwire proxy` in another terminal).
63
+ const base = `http://localhost:${cfg.port}`;
64
+ let h = null;
65
+ try { h = await (await fetch(base + '/_tw/health')).json(); } catch { /* nothing tripwire-like on the port */ }
66
+ if (h?.ok) {
67
+ if (h.code !== PROXY_CODE) throw new Error(`a tripwire proxy from different code is already running on port ${cfg.port} (its code ${h.code || 'unknown'}, this one ${PROXY_CODE}). Stop it, or let tripwire start its own.`);
68
+ if (h.target && h.target.replace(/\/$/, '') !== String(cfg.app).replace(/\/$/, '')) throw new Error(`the proxy on port ${cfg.port} points at ${h.target}, not ${cfg.app}`);
69
+ return fn(proxyClient(base), base, false);
70
+ }
71
+ const p = createProxy({ target: cfg.app, port: Number(cfg.port), log: () => {} });
72
+ await new Promise((resolve, reject) => {
73
+ p.server.once('error', (e) => reject(e.code === 'EADDRINUSE' ? new Error(`port ${cfg.port} is in use by something that is not a tripwire proxy; set "port" in tripwire.config.json`) : e));
74
+ p.listen().then(resolve, reject);
75
+ });
76
+ try { return await fn(proxyClient(base), base, true); } finally { await p.close(); }
77
+ }
78
+
79
+ export default async function run(argv, parseArgs) {
80
+ const { values } = parseArgs({
81
+ args: argv,
82
+ options: {
83
+ tests: { type: 'string' }, app: { type: 'string' }, port: { type: 'string' },
84
+ 'max-mutants': { type: 'string' }, only: { type: 'string' }, since: { type: 'string' },
85
+ 'no-heal-check': { type: 'boolean' }, 'heal-budget': { type: 'string' }, timeout: { type: 'string' },
86
+ 'baseline-only': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, resume: { type: 'boolean' }, 'retry-errors': { type: 'boolean' }, redo: { type: 'string' }, quiet: { type: 'boolean', short: 'q' }, dir: { type: 'string' },
87
+ help: { type: 'boolean', short: 'h' }
88
+ }
89
+ });
90
+ if (values.help) { console.log(HELP); return 0; }
91
+ const cfg = loadConfig();
92
+ for (const [k, v] of Object.entries({ tests: values.tests, app: values.app, port: values.port, dir: values.dir })) if (v) cfg[k] = v;
93
+ if (values['max-mutants']) cfg.maxMutants = Number(values['max-mutants']);
94
+ if (values.timeout) cfg.timeout = Number(values.timeout);
95
+ if (values['heal-budget']) cfg.healBudget = Number(values['heal-budget']);
96
+ if (values['no-heal-check']) cfg.healCheck = false;
97
+ const dir = path.resolve(cfg.dir);
98
+ fs.mkdirSync(dir, { recursive: true });
99
+ // --retry-errors: re-run only the mutants that ended in a Kane error (exit 2/3) in the last run,
100
+ // on that run's baseline. Their error rows are superseded by the new rows (latest row wins).
101
+ // --redo <id,...>: the same, for named mutants (for example after fixing an operator).
102
+ if (values['retry-errors'] || values.redo) {
103
+ const rows = readLedger(dir);
104
+ const last = lastRunId(rows);
105
+ const redo = new Set((values.redo || '').split(',').map((s) => s.trim()).filter(Boolean));
106
+ const again = (r) => (values['retry-errors'] && r.status === 'error') || redo.has(r.id);
107
+ const errored = latest(rows, last).filter(again);
108
+ const pending = readJson(path.join(dir, 'current-run.json'));
109
+ if (redo.size && errored.length < redo.size) throw new Error(`not in the last run: ${[...redo].filter((id) => !errored.some((r) => r.id === id)).join(', ')}`);
110
+ if (!errored.length && !(pending?.run === last)) { console.log('No errored mutants in the last run.'); return 0; }
111
+ const kept = rows.filter((r) => !(r.run === last && again(r)));
112
+ fs.writeFileSync(path.join(dir, 'ledger.jsonl'), kept.map((r) => JSON.stringify(r)).join('\n') + (kept.length ? '\n' : ''));
113
+ // keep every mutant the run picked, including ones an interrupted run never reached
114
+ const prev = readJson(path.join(dir, 'current-run.json'));
115
+ const picked = [...new Set([...(prev?.run === last ? prev.picked : []), ...latest(rows, last).map((r) => r.id)])];
116
+ fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: last, picked }));
117
+ console.log(`Re-running ${errored.length} mutant(s) of run ${last}`);
118
+ values.resume = true;
119
+ }
120
+ const resumeFrom = values.resume ? readJson(path.join(dir, 'current-run.json')) : null;
121
+ if (values.resume && !resumeFrom) throw new Error('nothing to resume: no interrupted run found in ' + dir);
122
+ const runId = resumeFrom?.run || stamp();
123
+ const runDir = path.join(dir, 'runs', runId);
124
+ fs.mkdirSync(runDir, { recursive: true });
125
+ const commit = sh('git rev-parse --short HEAD');
126
+
127
+ const tests = findTests(cfg.tests);
128
+ if (!tests.length) throw new Error(`no *_test.md files match ${cfg.tests}`);
129
+
130
+ if (values['dry-run']) {
131
+ const profile = JSON.parse(fs.readFileSync(path.join(dir, 'traffic.json'), 'utf8'));
132
+ const plan = plan_(profile, cfg, dir, values.only);
133
+ for (const m of plan.picked) console.log(`${m.family.padEnd(10)} ${m.operator.padEnd(19)} ${m.describe} [${m.touches_tests.join(', ')}]`);
134
+ console.log(`\n${plan.picked.length} of ${plan.candidates.length} candidates selected (${plan.excluded.length} excluded as equivalent)`);
135
+ return 0;
136
+ }
137
+
138
+ const orphans = findOrphanKaneBrowsers(120);
139
+ if (orphans.length) console.log(`warning: ${orphans.length} Kane browser(s) from earlier runs are still open (oldest ${Math.round(Math.max(...orphans.map((o) => o.ageMinutes)) / 60)} h). They can change a shared app's state mid-run. Stop them: tripwire doctor --kill`);
140
+ const creditsBefore = await balance();
141
+ return withProxy(cfg, async (tw, base) => {
142
+ // ---------- 1. baseline (reused when resuming) ----------
143
+ let profile = { tests: [], created: new Date().toISOString(), commit };
144
+ let baseline = [];
145
+ if (resumeFrom) {
146
+ profile = readJson(path.join(dir, 'traffic.json'));
147
+ baseline = readJson(path.join(dir, 'baseline.json'))?.tests || [];
148
+ console.log(`\nResuming run ${runId}: baseline from ${profile.created}`);
149
+ } else console.log(`\nBaseline: ${tests.length} test(s) through the proxy at ${base}, healing off`);
150
+ for (const t of resumeFrom ? [] : tests) {
151
+ await tw.disarm();
152
+ await resetApp(cfg, base);
153
+ await tw.reset(t, true);
154
+ const r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(runDir, `baseline_${slug(t)}.ndjson`), onRetry: hiccup });
155
+ const v = await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml'));
156
+ const withoutChecks = metricFromVerdict(v.raw, 'metrics.tests_without_checks');
157
+ baseline.push({ test: t, code: r.code, durationMs: r.durationMs, pack: r.pack, verdict: v.verdict, testsWithoutChecks: withoutChecks, session: r.sessionId });
158
+ console.log(` ${r.code === 0 ? 'pass' : 'FAIL'} ${t} ${(r.durationMs / 1000).toFixed(1)}s${withoutChecks ? ' (no assertions!)' : ''}`);
159
+ if (r.code !== 0) {
160
+ console.error(`\nThe baseline must be green: ${t} exited ${r.code}. ${r.failureReason || ''}\nLog: ${path.join(runDir, `baseline_${slug(t)}.ndjson`)}`);
161
+ if (r.spawnError) console.error('Is kane-cli installed and on PATH? (npm install -g @testmuai/kane-cli)');
162
+ return 2;
163
+ }
164
+ profile.tests.push(profileEntry(t, await tw.traffic(), await tw.dom()));
165
+ }
166
+ await tw.reset(null, false);
167
+ if (!resumeFrom) {
168
+ fs.writeFileSync(path.join(dir, 'traffic.json'), JSON.stringify(profile, null, 1));
169
+ fs.writeFileSync(path.join(dir, 'baseline.json'), JSON.stringify({ run: runId, commit, tests: baseline, routes: summarize(profile) }, null, 2));
170
+ }
171
+ const noChecks = baseline.filter((b) => b.testsWithoutChecks > 0);
172
+ if (noChecks.length) console.log(`\n${noChecks.length} test(s) assert nothing at all: ${noChecks.map((b) => b.test).join(', ')}`);
173
+
174
+ // ---------- 2. generate ----------
175
+ const plan = plan_(profile, cfg, dir, values.only);
176
+ const { candidates, excluded, counts } = plan;
177
+ // when resuming, replay exactly the mutants the interrupted run had picked
178
+ const picked = resumeFrom ? resumeFrom.picked.map((id) => candidates.find((m) => m.id === id)).filter(Boolean) : plan.picked;
179
+ if (!values['baseline-only']) fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: runId, picked: picked.map((m) => m.id), since: values.since || null }, null, 2));
180
+ fs.writeFileSync(path.join(runDir, 'mutants.json'), JSON.stringify({ counts, picked, excluded: excluded.map((m) => m.id) }, null, 2));
181
+ console.log(`\nGenerated ${candidates.length} mutants; replaying ${picked.length}${excluded.length ? ` (${excluded.length} excluded as equivalent)` : ''}`);
182
+ if (values['baseline-only']) return 0;
183
+
184
+ // ---------- incremental ----------
185
+ let carried = [];
186
+ let todo = picked;
187
+ if (values.since) {
188
+ // --relative: paths relative to this folder, the same form touches_tests uses
189
+ const diff = sh(`git diff --name-only --relative ${values.since} --`);
190
+ if (diff === null) throw new Error(`--since ${values.since}: git diff failed (not a git repository, or unknown commit)`);
191
+ const changed = changedTests(diff);
192
+ const prev = new Map(latest(readLedger(dir)).map((r) => [r.id, r]));
193
+ todo = [];
194
+ for (const m of picked) {
195
+ const p = prev.get(m.id);
196
+ const touchedChanged = m.touches_tests.some((t) => changed.has(normalizePath(t)));
197
+ if (p && p.status === 'killed' && !touchedChanged) carried.push({ ...p, run: runId, carried: true });
198
+ else todo.push(m);
199
+ }
200
+ console.log(`Incremental since ${values.since}: ${changed.size} changed test(s); ${carried.length} earlier kills trusted, ${todo.length} to run`);
201
+ }
202
+
203
+ // ---------- 3. replay each mutant ----------
204
+ const rows = [...carried];
205
+ for (const r of carried) appendLedger(dir, r);
206
+ let healChecks = 0;
207
+ if (resumeFrom) {
208
+ const done = latest(readLedger(dir), runId);
209
+ const doneIds = new Set(done.map((r) => r.id));
210
+ rows.push(...done);
211
+ healChecks = done.filter((r) => r.healChecked).length;
212
+ todo = todo.filter((m) => !doneIds.has(m.id));
213
+ console.log(`${done.length} mutants already done, ${todo.length} to go`);
214
+ }
215
+ for (const [i, m] of todo.entries()) {
216
+ if (!values.quiet) console.log(` [${String(i + 1).padStart(2)}/${todo.length}] ${m.family} · ${m.operator}: ${m.describe}`);
217
+ const row = await runMutant(m, { tw, cfg, base, runDir, profile, healAllowed: cfg.healCheck && healChecks < cfg.healBudget, progress: values.quiet ? undefined : (msg) => console.log(` ${msg}`) });
218
+ if (row.healChecked) healChecks++;
219
+ Object.assign(row, { run: runId, commit, at: new Date().toISOString() });
220
+ rows.push(row);
221
+ appendLedger(dir, row);
222
+ console.log(` ${values.quiet ? `[${String(i + 1).padStart(2)}/${todo.length}] ` : ' → '}${values.quiet ? label(row.status) : label(row.status).trim()} ${values.quiet ? m.describe : ''}${row.killed_by ? `${values.quiet ? ' ' : ''}(by ${row.killed_by})` : ''}`);
223
+ }
224
+ await tw.disarm();
225
+ fs.rmSync(path.join(dir, 'current-run.json'), { force: true });
226
+
227
+ // ---------- 4. score + report ----------
228
+ const creditsAfter = await balance();
229
+ const s = score(rows);
230
+ const meta = { run: runId, commit, creditsUsed: creditsBefore !== undefined && creditsAfter !== undefined ? Math.round((creditsBefore - creditsAfter) * 100) / 100 : null, baseline };
231
+ fs.writeFileSync(path.join(runDir, 'summary.json'), JSON.stringify({ ...meta, score: s }, null, 2));
232
+ const report = writeReport(dir);
233
+ console.log(`\nScore ${s.score ?? '-'}% killed ${s.killed} · survived ${s.survived} · masked by heal ${s.masked} · not reached ${s['not-reached']}${s.error ? ` · errors ${s.error}` : ''}`);
234
+ if (meta.creditsUsed !== null) console.log(`Credits used: ${meta.creditsUsed}`);
235
+ console.log(`Report: ${report}`);
236
+ if (s.survived + s.masked > 0) console.log('Next: tripwire fix (hands each survivor to your coding agent and proves the new assertion)');
237
+ return 0;
238
+ });
239
+ }
240
+
241
+ function plan_(profile, cfg, dir, only) {
242
+ const eqFile = path.join(dir, 'equivalents.json');
243
+ const excludedIds = new Set(fs.existsSync(eqFile) ? JSON.parse(fs.readFileSync(eqFile, 'utf8')).map((e) => e.id) : []);
244
+ const { candidates, excluded, counts } = generateAll(profile, { dir, only: only ? only.split(',') : undefined, excluded: excludedIds });
245
+ return { candidates, excluded, counts, picked: select(candidates, cfg.maxMutants) };
246
+ }
247
+
248
+ // output-<stem>/ next to foo_test.md holds the recordings kane replays.
249
+ export function recordingDir(testFile) {
250
+ return path.join(path.dirname(testFile), 'output-' + path.basename(testFile).replace(/_test\.md$/, ''));
251
+ }
252
+
253
+ export function snapshotRecording(testFile) {
254
+ const src = recordingDir(testFile);
255
+ const md = fs.existsSync(testFile) ? fs.readFileSync(testFile) : null;
256
+ if (!fs.existsSync(src)) return () => { if (md) fs.writeFileSync(testFile, md); };
257
+ const tmp = fs.mkdtempSync(path.join(path.dirname(src), '.tw-snap-'));
258
+ fs.cpSync(src, path.join(tmp, 'rec'), { recursive: true });
259
+ return () => {
260
+ fs.rmSync(src, { recursive: true, force: true });
261
+ fs.cpSync(path.join(tmp, 'rec'), src, { recursive: true });
262
+ fs.rmSync(tmp, { recursive: true, force: true });
263
+ if (md) fs.writeFileSync(testFile, md);
264
+ };
265
+ }
266
+
267
+ const hiccup = (n, msg) => console.log(` kane service hiccup (${msg}); waiting and retrying (${n}/3)`);
268
+
269
+ const label = (s) => ({ killed: 'killed ', survived: 'SURVIVED ', masked: 'MASKED ', 'not-reached': 'not reached', error: 'error ' }[s] || s);
270
+
271
+ // Lines of text the pages of one test showed in the clean baseline.
272
+ // line -> the most times the clean run showed it in one page snapshot. `counted` is false for
273
+ // baselines recorded before per-snapshot counts existed (then only new lines can be detected).
274
+ export function baselineLines(profile, test) {
275
+ const t = profile?.tests?.find((x) => x.test === test);
276
+ const max = new Map();
277
+ let counted = true;
278
+ for (const p of t?.pages || []) {
279
+ if (!p.maxCounts) counted = false;
280
+ const counts = p.maxCounts ? Object.entries(p.maxCounts) : [...lineCounts(p.text)].map(([l]) => [l, Infinity]);
281
+ for (const [l, n] of counts) if (!(max.get(l) >= n)) max.set(l, n);
282
+ }
283
+ max.counted = counted;
284
+ return max;
285
+ }
286
+
287
+ // Did any page of this mutant run show a line the clean run never showed, or show a line more
288
+ // often than any clean snapshot did (a duplicated row)?
289
+ export function showedSomethingNew(dom, lines) {
290
+ for (const ev of dom) if (ev.kind === 'page') for (const [l, n] of lineCounts(ev.text)) {
291
+ if (!lines.has(l) || n > lines.get(l)) return true;
292
+ }
293
+ return false;
294
+ }
295
+
296
+ const DUPLICATING = new Set(['item-dup', 'rendered-item-dup', 'double-apply']);
297
+
298
+ // Arm one mutant, replay the tests that touch it (healing off) until one fails.
299
+ export async function runMutant(m, { tw, cfg, base, runDir, healAllowed, progress = () => {}, profile }) {
300
+ const row = { id: m.id, family: m.family, operator: m.operator, describe: m.describe, expect: m.expect, touches_tests: m.touches_tests, status: null, fired: false, runs: [] };
301
+ const mdir = path.join(runDir, slug(m.id));
302
+ fs.mkdirSync(mdir, { recursive: true });
303
+ let infra = 0;
304
+ for (const t of m.touches_tests) {
305
+ let attempt = 0;
306
+ let r;
307
+ let hits;
308
+ do {
309
+ await tw.disarm();
310
+ await resetApp(cfg, base);
311
+ // the recorder stays on so we can tell whether the bug ever changed what the user saw
312
+ await tw.reset(t, true);
313
+ await tw.arm(m);
314
+ // Kane can rewrite a recording during any run against the broken app (observed even on
315
+ // failed runs), so every mutant run gets the real recording back afterwards.
316
+ const restore = snapshotRecording(t);
317
+ progress(`replaying ${t}${attempt ? ' (retry)' : ''}`);
318
+ try {
319
+ r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(mdir, `${slug(t)}_${attempt}.ndjson`), onRetry: hiccup });
320
+ } finally { restore(); }
321
+ hits = (await tw.hit()).hits;
322
+ r.visibleChange = profile ? showedSomethingNew(await tw.dom(), baselineLines(profile, t)) : undefined;
323
+ attempt++;
324
+ } while ((r.code === 2 || r.code === 3) && attempt < 2); // infrastructure problem: retry once, never a kill
325
+ const fired = hits > 0;
326
+ row.fired = row.fired || fired;
327
+ const v = r.code === 0 || r.code === 1 ? await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.mutant.policy.yaml')) : { verdict: 'unknown' };
328
+ row.runs.push({ test: t, code: r.code, fired, hits, durationMs: r.durationMs, pack: r.pack, session: r.sessionId, policy: v.verdict, visibleChange: r.visibleChange, failure: r.failureReason, failedStep: r.failedStep, kaneTriage: r.bugVerdict, credits: r.credits || undefined });
329
+ if (r.code === 2 || r.code === 3) { infra++; continue; }
330
+ if (r.code === 1 && fired) {
331
+ row.status = 'killed';
332
+ row.killed_by = t;
333
+ break;
334
+ }
335
+ }
336
+ if (!row.status) {
337
+ if (!row.fired) row.status = infra === m.touches_tests.length ? 'error' : 'not-reached';
338
+ else row.status = infra && !row.runs.some((x) => x.code === 0) ? 'error' : 'survived';
339
+ }
340
+ // A survivor that fired but never changed a single line on any page is probably equivalent
341
+ // (for example a write response the page ignores because it re-reads the data). Flag it for a
342
+ // human; it still counts until someone adds it to equivalents.json with a reason.
343
+ // Without per-snapshot counts a duplicate cannot be seen, so duplicating mutants are not flagged.
344
+ const canTell = !DUPLICATING.has(m.operator) || profile?.tests?.every((t) => t.pages.every((p) => p.maxCounts));
345
+ if (row.status === 'survived' && canTell && row.runs.length && row.runs.every((x) => x.visibleChange === false)) {
346
+ row.suspectedEquivalent = 'The mutant fired, but no page showed anything the clean run did not show.';
347
+ }
348
+ // Heal check: would Kane's default healing have re-authored its way past this bug?
349
+ if (row.status === 'killed' && healAllowed) {
350
+ row.healChecked = true;
351
+ progress(`caught by ${row.killed_by}; heal check: same bug with Kane's default healing on`);
352
+ await tw.disarm();
353
+ await resetApp(cfg, base);
354
+ await tw.reset(row.killed_by, false);
355
+ await tw.arm(m);
356
+ // A run that passes after a heal commits the re-authored steps. Those steps were recorded
357
+ // against the broken app, so keep a copy of the real recording and put it back afterwards.
358
+ const restore = snapshotRecording(row.killed_by);
359
+ let h;
360
+ try {
361
+ h = await runTest(row.killed_by, { heal: true, timeout: cfg.timeout, logFile: path.join(mdir, `${slug(row.killed_by)}_heal.ndjson`) });
362
+ } finally { restore(); }
363
+ const hv = h.pack ? await verdict(h.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml')) : { raw: null };
364
+ const healed = metricFromVerdict(hv.raw, 'metrics.heal.healed_tests');
365
+ row.heal = { code: h.code, healedTests: healed, healEvents: h.healEvents, pack: h.pack, session: h.sessionId };
366
+ if (h.code === 0 && (healed > 0 || h.healEvents > 0 || h.mentionsHeal)) row.status = 'masked';
367
+ }
368
+ await tw.disarm();
369
+ return row;
370
+ }