tripwire-e2e 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +168 -2
- package/bin/tripwire.js +8 -0
- package/operators/_util.js +114 -0
- package/operators/agent-proposed.js +41 -0
- package/operators/auth-drop.js +30 -0
- package/operators/dead-control.js +23 -0
- package/operators/double-apply.js +20 -0
- package/operators/fake-success.js +21 -0
- package/operators/field-swap.js +42 -0
- package/operators/hide-control.js +30 -0
- package/operators/index.js +77 -0
- package/operators/item-drop.js +36 -0
- package/operators/item-dup.js +20 -0
- package/operators/label-swap.js +30 -0
- package/operators/missing-row-action.js +24 -0
- package/operators/number-drift.js +40 -0
- package/operators/silent-fail.js +19 -0
- package/operators/stale-read.js +34 -0
- package/operators/text-change.js +43 -0
- package/operators/validation-off.js +21 -0
- package/operators/wrong-link.js +28 -0
- package/operators/wrong-owner.js +41 -0
- package/package.json +41 -4
- package/policies/tripwire.clean.policy.yaml +23 -0
- package/policies/tripwire.mutant.policy.yaml +13 -0
- package/src/actions.js +59 -0
- package/src/agent.js +30 -0
- package/src/browser/mutate.js +113 -0
- package/src/browser/recorder.js +80 -0
- package/src/cli.js +37 -0
- package/src/commands/check.js +28 -0
- package/src/commands/fix.js +199 -0
- package/src/commands/list.js +24 -0
- package/src/commands/proxy.js +28 -0
- package/src/commands/report.js +11 -0
- package/src/commands/run.js +310 -0
- package/src/commands/sabotage.js +74 -0
- package/src/config.js +52 -0
- package/src/jsonpath.js +58 -0
- package/src/kane.js +150 -0
- package/src/ledger.js +53 -0
- package/src/mutant.js +26 -0
- package/src/profile.js +34 -0
- package/src/proxy.js +289 -0
- package/src/report.js +182 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
// tripwire run: baseline -> generate -> replay each mutant -> classify -> ledger -> report.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { execSync, spawnSync } from 'node:child_process';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { createProxy, proxyClient } from '../proxy.js';
|
|
7
|
+
import { loadConfig, findTests } from '../config.js';
|
|
8
|
+
import { runTest, verdict, metricFromVerdict, balance } from '../kane.js';
|
|
9
|
+
import { profileEntry, summarize } from '../profile.js';
|
|
10
|
+
import { generateAll, select } from '../../operators/index.js';
|
|
11
|
+
import { appendLedger, readLedger, latest, lastRunId, score } from '../ledger.js';
|
|
12
|
+
import { writeReport } from '../report.js';
|
|
13
|
+
|
|
14
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
export const POLICY_DIR = path.resolve(here, '../../policies');
|
|
16
|
+
|
|
17
|
+
const HELP = `tripwire run [options]
|
|
18
|
+
|
|
19
|
+
--tests <glob> test files (default from tripwire.config.json, else **/*_test.md)
|
|
20
|
+
--app <url> the real app behind the proxy (default http://localhost:3000)
|
|
21
|
+
--port <n> proxy port your tests point at (default 8080)
|
|
22
|
+
--max-mutants <n> cap on mutants replayed (default 40)
|
|
23
|
+
--only <ops> comma-separated operators or families (data, behaviour, interface, saboteur)
|
|
24
|
+
--since <commit> incremental: re-run only mutants whose tests changed; trust earlier kills
|
|
25
|
+
--no-heal-check skip the healing-on rerun that detects heal-masked mutants
|
|
26
|
+
--heal-budget <n> at most n heal checks per run (default 10)
|
|
27
|
+
--timeout <s> per-step kane timeout (default 180)
|
|
28
|
+
--baseline-only record the baseline and list mutants, replay nothing
|
|
29
|
+
--dry-run generate from the last baseline and print the plan
|
|
30
|
+
--resume continue the last run: reuse its baseline, skip mutants already in the ledger
|
|
31
|
+
--retry-errors re-run only the mutants that hit a Kane error in the last run
|
|
32
|
+
-q, --quiet one line per mutant, no per-replay progress
|
|
33
|
+
--dir <path> state directory (default .tripwire)`;
|
|
34
|
+
|
|
35
|
+
const readJson = (f) => { try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch { return null; } };
|
|
36
|
+
const sh = (cmd) => { try { return execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim(); } catch { return null; } };
|
|
37
|
+
const stamp = () => new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
38
|
+
const slug = (s) => s.replace(/[^\w.-]+/g, '_').slice(0, 80);
|
|
39
|
+
|
|
40
|
+
export async function resetApp(cfg, base) {
|
|
41
|
+
if (!cfg.reset) return;
|
|
42
|
+
if (cfg.reset.cmd) {
|
|
43
|
+
const r = spawnSync(cfg.reset.cmd, { shell: true, stdio: 'ignore' });
|
|
44
|
+
if (r.status !== 0) throw new Error(`reset command failed: ${cfg.reset.cmd}`);
|
|
45
|
+
} else if (cfg.reset.url) {
|
|
46
|
+
// reset through the app directly, never through the proxy, so it is not recorded or mutated
|
|
47
|
+
const url = cfg.reset.url.startsWith('http') ? cfg.reset.url : new URL(cfg.reset.url, base).toString();
|
|
48
|
+
const r = await fetch(url, { method: cfg.reset.method || 'POST' });
|
|
49
|
+
if (!r.ok) throw new Error(`reset ${url} returned ${r.status}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function withProxy(cfg, fn) {
|
|
54
|
+
// Reuse a proxy that is already running on the port (e.g. `tripwire proxy` in another terminal).
|
|
55
|
+
const base = `http://localhost:${cfg.port}`;
|
|
56
|
+
try {
|
|
57
|
+
const h = await (await fetch(base + '/_tw/health')).json();
|
|
58
|
+
if (h.ok) return fn(proxyClient(base), base, false);
|
|
59
|
+
} catch { /* none running */ }
|
|
60
|
+
const p = createProxy({ target: cfg.app, port: Number(cfg.port), log: () => {} });
|
|
61
|
+
await p.listen();
|
|
62
|
+
try { return await fn(proxyClient(base), base, true); } finally { await p.close(); }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export default async function run(argv, parseArgs) {
|
|
66
|
+
const { values } = parseArgs({
|
|
67
|
+
args: argv,
|
|
68
|
+
options: {
|
|
69
|
+
tests: { type: 'string' }, app: { type: 'string' }, port: { type: 'string' },
|
|
70
|
+
'max-mutants': { type: 'string' }, only: { type: 'string' }, since: { type: 'string' },
|
|
71
|
+
'no-heal-check': { type: 'boolean' }, 'heal-budget': { type: 'string' }, timeout: { type: 'string' },
|
|
72
|
+
'baseline-only': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, resume: { type: 'boolean' }, 'retry-errors': { type: 'boolean' }, quiet: { type: 'boolean', short: 'q' }, dir: { type: 'string' },
|
|
73
|
+
help: { type: 'boolean', short: 'h' }
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
if (values.help) { console.log(HELP); return 0; }
|
|
77
|
+
const cfg = loadConfig();
|
|
78
|
+
for (const [k, v] of Object.entries({ tests: values.tests, app: values.app, port: values.port, dir: values.dir })) if (v) cfg[k] = v;
|
|
79
|
+
if (values['max-mutants']) cfg.maxMutants = Number(values['max-mutants']);
|
|
80
|
+
if (values.timeout) cfg.timeout = Number(values.timeout);
|
|
81
|
+
if (values['heal-budget']) cfg.healBudget = Number(values['heal-budget']);
|
|
82
|
+
if (values['no-heal-check']) cfg.healCheck = false;
|
|
83
|
+
const dir = path.resolve(cfg.dir);
|
|
84
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
85
|
+
// --retry-errors: re-run only the mutants that ended in a Kane error (exit 2/3) in the last run,
|
|
86
|
+
// on that run's baseline. Their error rows are superseded by the new rows (latest row wins).
|
|
87
|
+
if (values['retry-errors']) {
|
|
88
|
+
const rows = readLedger(dir);
|
|
89
|
+
const last = lastRunId(rows);
|
|
90
|
+
const errored = latest(rows, last).filter((r) => r.status === 'error');
|
|
91
|
+
const pending = readJson(path.join(dir, 'current-run.json'));
|
|
92
|
+
if (!errored.length && !(pending?.run === last)) { console.log('No errored mutants in the last run.'); return 0; }
|
|
93
|
+
const kept = rows.filter((r) => !(r.run === last && r.status === 'error'));
|
|
94
|
+
fs.writeFileSync(path.join(dir, 'ledger.jsonl'), kept.map((r) => JSON.stringify(r)).join('\n') + (kept.length ? '\n' : ''));
|
|
95
|
+
// keep every mutant the run picked, including ones an interrupted run never reached
|
|
96
|
+
const prev = readJson(path.join(dir, 'current-run.json'));
|
|
97
|
+
const picked = [...new Set([...(prev?.run === last ? prev.picked : []), ...latest(rows, last).map((r) => r.id)])];
|
|
98
|
+
fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: last, picked }));
|
|
99
|
+
console.log(`Retrying ${errored.length} errored mutants of run ${last}`);
|
|
100
|
+
values.resume = true;
|
|
101
|
+
}
|
|
102
|
+
const resumeFrom = values.resume ? readJson(path.join(dir, 'current-run.json')) : null;
|
|
103
|
+
if (values.resume && !resumeFrom) throw new Error('nothing to resume: no interrupted run found in ' + dir);
|
|
104
|
+
const runId = resumeFrom?.run || stamp();
|
|
105
|
+
const runDir = path.join(dir, 'runs', runId);
|
|
106
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
107
|
+
const commit = sh('git rev-parse --short HEAD');
|
|
108
|
+
|
|
109
|
+
const tests = findTests(cfg.tests);
|
|
110
|
+
if (!tests.length) throw new Error(`no *_test.md files match ${cfg.tests}`);
|
|
111
|
+
|
|
112
|
+
if (values['dry-run']) {
|
|
113
|
+
const profile = JSON.parse(fs.readFileSync(path.join(dir, 'traffic.json'), 'utf8'));
|
|
114
|
+
const plan = plan_(profile, cfg, dir, values.only);
|
|
115
|
+
for (const m of plan.picked) console.log(`${m.family.padEnd(10)} ${m.operator.padEnd(19)} ${m.describe} [${m.touches_tests.join(', ')}]`);
|
|
116
|
+
console.log(`\n${plan.picked.length} of ${plan.candidates.length} candidates selected (${plan.excluded.length} excluded as equivalent)`);
|
|
117
|
+
return 0;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const creditsBefore = await balance();
|
|
121
|
+
return withProxy(cfg, async (tw, base) => {
|
|
122
|
+
// ---------- 1. baseline (reused when resuming) ----------
|
|
123
|
+
let profile = { tests: [], created: new Date().toISOString(), commit };
|
|
124
|
+
let baseline = [];
|
|
125
|
+
if (resumeFrom) {
|
|
126
|
+
profile = readJson(path.join(dir, 'traffic.json'));
|
|
127
|
+
baseline = readJson(path.join(dir, 'baseline.json'))?.tests || [];
|
|
128
|
+
console.log(`\nResuming run ${runId}: baseline from ${profile.created}`);
|
|
129
|
+
} else console.log(`\nBaseline: ${tests.length} test(s) through the proxy at ${base}, healing off`);
|
|
130
|
+
for (const t of resumeFrom ? [] : tests) {
|
|
131
|
+
await tw.disarm();
|
|
132
|
+
await resetApp(cfg, base);
|
|
133
|
+
await tw.reset(t, true);
|
|
134
|
+
const r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(runDir, `baseline_${slug(t)}.ndjson`), onRetry: hiccup });
|
|
135
|
+
const v = await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml'));
|
|
136
|
+
const withoutChecks = metricFromVerdict(v.raw, 'metrics.tests_without_checks');
|
|
137
|
+
baseline.push({ test: t, code: r.code, durationMs: r.durationMs, pack: r.pack, verdict: v.verdict, testsWithoutChecks: withoutChecks, session: r.sessionId });
|
|
138
|
+
console.log(` ${r.code === 0 ? 'pass' : 'FAIL'} ${t} ${(r.durationMs / 1000).toFixed(1)}s${withoutChecks ? ' (no assertions!)' : ''}`);
|
|
139
|
+
if (r.code !== 0) {
|
|
140
|
+
console.error(`\nThe baseline must be green: ${t} exited ${r.code}. ${r.failureReason || ''}\nLog: ${path.join(runDir, `baseline_${slug(t)}.ndjson`)}`);
|
|
141
|
+
if (r.spawnError) console.error('Is kane-cli installed and on PATH? (npm install -g @testmuai/kane-cli)');
|
|
142
|
+
return 2;
|
|
143
|
+
}
|
|
144
|
+
profile.tests.push(profileEntry(t, await tw.traffic(), await tw.dom()));
|
|
145
|
+
}
|
|
146
|
+
await tw.reset(null, false);
|
|
147
|
+
if (!resumeFrom) {
|
|
148
|
+
fs.writeFileSync(path.join(dir, 'traffic.json'), JSON.stringify(profile, null, 1));
|
|
149
|
+
fs.writeFileSync(path.join(dir, 'baseline.json'), JSON.stringify({ run: runId, commit, tests: baseline, routes: summarize(profile) }, null, 2));
|
|
150
|
+
}
|
|
151
|
+
const noChecks = baseline.filter((b) => b.testsWithoutChecks > 0);
|
|
152
|
+
if (noChecks.length) console.log(`\n${noChecks.length} test(s) assert nothing at all: ${noChecks.map((b) => b.test).join(', ')}`);
|
|
153
|
+
|
|
154
|
+
// ---------- 2. generate ----------
|
|
155
|
+
const plan = plan_(profile, cfg, dir, values.only);
|
|
156
|
+
const { candidates, excluded, counts } = plan;
|
|
157
|
+
// when resuming, replay exactly the mutants the interrupted run had picked
|
|
158
|
+
const picked = resumeFrom ? resumeFrom.picked.map((id) => candidates.find((m) => m.id === id)).filter(Boolean) : plan.picked;
|
|
159
|
+
if (!values['baseline-only']) fs.writeFileSync(path.join(dir, 'current-run.json'), JSON.stringify({ run: runId, picked: picked.map((m) => m.id), since: values.since || null }, null, 2));
|
|
160
|
+
fs.writeFileSync(path.join(runDir, 'mutants.json'), JSON.stringify({ counts, picked, excluded: excluded.map((m) => m.id) }, null, 2));
|
|
161
|
+
console.log(`\nGenerated ${candidates.length} mutants; replaying ${picked.length}${excluded.length ? ` (${excluded.length} excluded as equivalent)` : ''}`);
|
|
162
|
+
if (values['baseline-only']) return 0;
|
|
163
|
+
|
|
164
|
+
// ---------- incremental ----------
|
|
165
|
+
let carried = [];
|
|
166
|
+
let todo = picked;
|
|
167
|
+
if (values.since) {
|
|
168
|
+
const changed = new Set((sh(`git diff --name-only ${values.since} --`) || '').split('\n').filter(Boolean));
|
|
169
|
+
const prev = new Map(latest(readLedger(dir)).map((r) => [r.id, r]));
|
|
170
|
+
todo = [];
|
|
171
|
+
for (const m of picked) {
|
|
172
|
+
const p = prev.get(m.id);
|
|
173
|
+
const touchedChanged = m.touches_tests.some((t) => changed.has(t));
|
|
174
|
+
if (p && p.status === 'killed' && !touchedChanged) carried.push({ ...p, run: runId, carried: true });
|
|
175
|
+
else todo.push(m);
|
|
176
|
+
}
|
|
177
|
+
console.log(`Incremental since ${values.since}: ${carried.length} earlier kills trusted, ${todo.length} to run`);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ---------- 3. replay each mutant ----------
|
|
181
|
+
const rows = [...carried];
|
|
182
|
+
for (const r of carried) appendLedger(dir, r);
|
|
183
|
+
let healChecks = 0;
|
|
184
|
+
if (resumeFrom) {
|
|
185
|
+
const done = latest(readLedger(dir), runId);
|
|
186
|
+
const doneIds = new Set(done.map((r) => r.id));
|
|
187
|
+
rows.push(...done);
|
|
188
|
+
healChecks = done.filter((r) => r.healChecked).length;
|
|
189
|
+
todo = todo.filter((m) => !doneIds.has(m.id));
|
|
190
|
+
console.log(`${done.length} mutants already done, ${todo.length} to go`);
|
|
191
|
+
}
|
|
192
|
+
for (const [i, m] of todo.entries()) {
|
|
193
|
+
if (!values.quiet) console.log(` [${String(i + 1).padStart(2)}/${todo.length}] ${m.family} · ${m.operator}: ${m.describe}`);
|
|
194
|
+
const row = await runMutant(m, { tw, cfg, base, runDir, healAllowed: cfg.healCheck && healChecks < cfg.healBudget, progress: values.quiet ? undefined : (msg) => console.log(` ${msg}`) });
|
|
195
|
+
if (row.healChecked) healChecks++;
|
|
196
|
+
Object.assign(row, { run: runId, commit, at: new Date().toISOString() });
|
|
197
|
+
rows.push(row);
|
|
198
|
+
appendLedger(dir, row);
|
|
199
|
+
console.log(` ${values.quiet ? `[${String(i + 1).padStart(2)}/${todo.length}] ` : ' → '}${values.quiet ? label(row.status) : label(row.status).trim()} ${values.quiet ? m.describe : ''}${row.killed_by ? `${values.quiet ? ' ' : ''}(by ${row.killed_by})` : ''}`);
|
|
200
|
+
}
|
|
201
|
+
await tw.disarm();
|
|
202
|
+
fs.rmSync(path.join(dir, 'current-run.json'), { force: true });
|
|
203
|
+
|
|
204
|
+
// ---------- 4. score + report ----------
|
|
205
|
+
const creditsAfter = await balance();
|
|
206
|
+
const s = score(rows);
|
|
207
|
+
const meta = { run: runId, commit, creditsUsed: creditsBefore !== undefined && creditsAfter !== undefined ? Math.round((creditsBefore - creditsAfter) * 100) / 100 : null, baseline };
|
|
208
|
+
fs.writeFileSync(path.join(runDir, 'summary.json'), JSON.stringify({ ...meta, score: s }, null, 2));
|
|
209
|
+
const report = writeReport(dir);
|
|
210
|
+
console.log(`\nScore ${s.score ?? '-'}% killed ${s.killed} · survived ${s.survived} · masked by heal ${s.masked} · not reached ${s['not-reached']}${s.error ? ` · errors ${s.error}` : ''}`);
|
|
211
|
+
if (meta.creditsUsed !== null) console.log(`Credits used: ${meta.creditsUsed}`);
|
|
212
|
+
console.log(`Report: ${report}`);
|
|
213
|
+
if (s.survived + s.masked > 0) console.log('Next: tripwire fix (hands each survivor to your coding agent and proves the new assertion)');
|
|
214
|
+
return 0;
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function plan_(profile, cfg, dir, only) {
|
|
219
|
+
const eqFile = path.join(dir, 'equivalents.json');
|
|
220
|
+
const excludedIds = new Set(fs.existsSync(eqFile) ? JSON.parse(fs.readFileSync(eqFile, 'utf8')).map((e) => e.id) : []);
|
|
221
|
+
const { candidates, excluded, counts } = generateAll(profile, { dir, only: only ? only.split(',') : undefined, excluded: excludedIds });
|
|
222
|
+
return { candidates, excluded, counts, picked: select(candidates, cfg.maxMutants) };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// output-<stem>/ next to foo_test.md holds the recordings kane replays.
|
|
226
|
+
export function recordingDir(testFile) {
|
|
227
|
+
return path.join(path.dirname(testFile), 'output-' + path.basename(testFile).replace(/_test\.md$/, ''));
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export function snapshotRecording(testFile) {
|
|
231
|
+
const src = recordingDir(testFile);
|
|
232
|
+
const md = fs.existsSync(testFile) ? fs.readFileSync(testFile) : null;
|
|
233
|
+
if (!fs.existsSync(src)) return () => { if (md) fs.writeFileSync(testFile, md); };
|
|
234
|
+
const tmp = fs.mkdtempSync(path.join(path.dirname(src), '.tw-snap-'));
|
|
235
|
+
fs.cpSync(src, path.join(tmp, 'rec'), { recursive: true });
|
|
236
|
+
return () => {
|
|
237
|
+
fs.rmSync(src, { recursive: true, force: true });
|
|
238
|
+
fs.cpSync(path.join(tmp, 'rec'), src, { recursive: true });
|
|
239
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
240
|
+
if (md) fs.writeFileSync(testFile, md);
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const hiccup = (n, msg) => console.log(` kane service hiccup (${msg}); waiting and retrying (${n}/3)`);
|
|
245
|
+
|
|
246
|
+
const label = (s) => ({ killed: 'killed ', survived: 'SURVIVED ', masked: 'MASKED ', 'not-reached': 'not reached', error: 'error ' }[s] || s);
|
|
247
|
+
|
|
248
|
+
// Arm one mutant, replay the tests that touch it (healing off) until one fails.
|
|
249
|
+
export async function runMutant(m, { tw, cfg, base, runDir, healAllowed, progress = () => {} }) {
|
|
250
|
+
const row = { id: m.id, family: m.family, operator: m.operator, describe: m.describe, expect: m.expect, touches_tests: m.touches_tests, status: null, fired: false, runs: [] };
|
|
251
|
+
const mdir = path.join(runDir, slug(m.id));
|
|
252
|
+
fs.mkdirSync(mdir, { recursive: true });
|
|
253
|
+
let infra = 0;
|
|
254
|
+
for (const t of m.touches_tests) {
|
|
255
|
+
let attempt = 0;
|
|
256
|
+
let r;
|
|
257
|
+
let hits;
|
|
258
|
+
do {
|
|
259
|
+
await tw.disarm();
|
|
260
|
+
await resetApp(cfg, base);
|
|
261
|
+
await tw.reset(t, false);
|
|
262
|
+
await tw.arm(m);
|
|
263
|
+
// Kane can rewrite a recording during any run against the broken app (observed even on
|
|
264
|
+
// failed runs), so every mutant run gets the real recording back afterwards.
|
|
265
|
+
const restore = snapshotRecording(t);
|
|
266
|
+
progress(`replaying ${t}${attempt ? ' (retry)' : ''}`);
|
|
267
|
+
try {
|
|
268
|
+
r = await runTest(t, { timeout: cfg.timeout, logFile: path.join(mdir, `${slug(t)}_${attempt}.ndjson`), onRetry: hiccup });
|
|
269
|
+
} finally { restore(); }
|
|
270
|
+
hits = (await tw.hit()).hits;
|
|
271
|
+
attempt++;
|
|
272
|
+
} while ((r.code === 2 || r.code === 3) && attempt < 2); // infrastructure problem: retry once, never a kill
|
|
273
|
+
const fired = hits > 0;
|
|
274
|
+
row.fired = row.fired || fired;
|
|
275
|
+
const v = r.code === 0 || r.code === 1 ? await verdict(r.pack, path.join(POLICY_DIR, 'tripwire.mutant.policy.yaml')) : { verdict: 'unknown' };
|
|
276
|
+
row.runs.push({ test: t, code: r.code, fired, hits, durationMs: r.durationMs, pack: r.pack, session: r.sessionId, policy: v.verdict, failure: r.failureReason, failedStep: r.failedStep, kaneTriage: r.bugVerdict, credits: r.credits || undefined });
|
|
277
|
+
if (r.code === 2 || r.code === 3) { infra++; continue; }
|
|
278
|
+
if (r.code === 1 && fired) {
|
|
279
|
+
row.status = 'killed';
|
|
280
|
+
row.killed_by = t;
|
|
281
|
+
break;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (!row.status) {
|
|
285
|
+
if (!row.fired) row.status = infra === m.touches_tests.length ? 'error' : 'not-reached';
|
|
286
|
+
else row.status = infra && !row.runs.some((x) => x.code === 0) ? 'error' : 'survived';
|
|
287
|
+
}
|
|
288
|
+
// Heal check: would Kane's default healing have re-authored its way past this bug?
|
|
289
|
+
if (row.status === 'killed' && healAllowed) {
|
|
290
|
+
row.healChecked = true;
|
|
291
|
+
progress(`caught by ${row.killed_by}; heal check: same bug with Kane's default healing on`);
|
|
292
|
+
await tw.disarm();
|
|
293
|
+
await resetApp(cfg, base);
|
|
294
|
+
await tw.reset(row.killed_by, false);
|
|
295
|
+
await tw.arm(m);
|
|
296
|
+
// A run that passes after a heal commits the re-authored steps. Those steps were recorded
|
|
297
|
+
// against the broken app, so keep a copy of the real recording and put it back afterwards.
|
|
298
|
+
const restore = snapshotRecording(row.killed_by);
|
|
299
|
+
let h;
|
|
300
|
+
try {
|
|
301
|
+
h = await runTest(row.killed_by, { heal: true, timeout: cfg.timeout, logFile: path.join(mdir, `${slug(row.killed_by)}_heal.ndjson`) });
|
|
302
|
+
} finally { restore(); }
|
|
303
|
+
const hv = h.pack ? await verdict(h.pack, path.join(POLICY_DIR, 'tripwire.clean.policy.yaml')) : { raw: null };
|
|
304
|
+
const healed = metricFromVerdict(hv.raw, 'metrics.heal.healed_tests');
|
|
305
|
+
row.heal = { code: h.code, healedTests: healed, healEvents: h.healEvents, pack: h.pack, session: h.sessionId };
|
|
306
|
+
if (h.code === 0 && (healed > 0 || h.healEvents > 0 || h.mentionsHeal)) row.status = 'masked';
|
|
307
|
+
}
|
|
308
|
+
await tw.disarm();
|
|
309
|
+
return row;
|
|
310
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// tripwire sabotage: the coding agent in its second role. It reads the acceptance criteria, the
|
|
2
|
+
// tests and the traffic the suite produced, and proposes domain-specific bugs ("coupon applied
|
|
3
|
+
// twice", "free shipping one cent too early") as mutant definitions in .tripwire/saboteur.json.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { loadConfig, findTests } from '../config.js';
|
|
7
|
+
import { runAgent } from '../agent.js';
|
|
8
|
+
import { loadSaboteur } from '../../operators/agent-proposed.js';
|
|
9
|
+
|
|
10
|
+
const HELP = `tripwire sabotage [options]
|
|
11
|
+
|
|
12
|
+
--agent claude|codex which agent proposes mutants (default from config)
|
|
13
|
+
--criteria <file> acceptance criteria / spec to read (default: README.md, docs/*.md if present)
|
|
14
|
+
--count <n> how many mutants to ask for (default 8)
|
|
15
|
+
--dir <path> state directory (default .tripwire)`;
|
|
16
|
+
|
|
17
|
+
function trafficDigest(profile) {
|
|
18
|
+
const seen = new Map();
|
|
19
|
+
for (const t of profile.tests) for (const e of t.traffic) {
|
|
20
|
+
const k = `${e.method} ${e.path}`;
|
|
21
|
+
if (seen.has(k) || e.type !== 'json') continue;
|
|
22
|
+
seen.set(k, `${k} -> ${e.status}${e.req ? ` request ${JSON.stringify(e.req).slice(0, 200)}` : ''}${e.json !== undefined ? `\n response ${JSON.stringify(e.json).slice(0, 600)}` : ''}`);
|
|
23
|
+
}
|
|
24
|
+
return [...seen.values()].join('\n');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export default async function sabotage(argv, parseArgs) {
|
|
28
|
+
const { values } = parseArgs({ args: argv, options: { agent: { type: 'string' }, criteria: { type: 'string' }, count: { type: 'string' }, dir: { type: 'string' }, help: { type: 'boolean', short: 'h' } } });
|
|
29
|
+
if (values.help) { console.log(HELP); return 0; }
|
|
30
|
+
const cfg = loadConfig();
|
|
31
|
+
const dir = path.resolve(values.dir || cfg.dir);
|
|
32
|
+
const agent = values.agent || cfg.agent || 'claude';
|
|
33
|
+
const trafficFile = path.join(dir, 'traffic.json');
|
|
34
|
+
if (!fs.existsSync(trafficFile)) throw new Error('no baseline yet: run `tripwire run --baseline-only` first');
|
|
35
|
+
const profile = JSON.parse(fs.readFileSync(trafficFile, 'utf8'));
|
|
36
|
+
const tests = findTests(cfg.tests);
|
|
37
|
+
const criteriaFiles = values.criteria ? [values.criteria] : ['ACCEPTANCE.md', 'README.md'].filter((f) => fs.existsSync(f));
|
|
38
|
+
const out = path.join(dir, 'saboteur.json');
|
|
39
|
+
const count = Number(values.count || 8);
|
|
40
|
+
|
|
41
|
+
const prompt = `You are the saboteur. Propose ${count} realistic, domain-specific bugs for the web app below, the kind a developer could really ship, that break a business rule a user would notice. Express each one as a Tripwire mutant: a rewrite of one HTTP response the app already sends (the app code is never changed).
|
|
42
|
+
|
|
43
|
+
Write a JSON array to ${out} (create or overwrite the file). Each element:
|
|
44
|
+
{
|
|
45
|
+
"id": "short-kebab-name",
|
|
46
|
+
"describe": "One sentence a user would notice, e.g. 'The 10% coupon takes 20% off'",
|
|
47
|
+
"expect": "What a good test would check",
|
|
48
|
+
"match": { "method": "GET", "path": "/api/..." },
|
|
49
|
+
"action": <one of>
|
|
50
|
+
{ "type": "json", "ops": [ { "op": "set"|"add"|"scale"|"remove-index"|"dup-index"|"swap"|"push", "path": "$.field.or[0].nested", "value": ..., "index": 0, "a": "k1", "b": "k2" } ] }
|
|
51
|
+
{ "type": "respond", "status": 200, "body": { ...a whole fake response... } }
|
|
52
|
+
{ "type": "status-map", "from": [422], "to": 200 }
|
|
53
|
+
}
|
|
54
|
+
Use only routes and JSON paths that appear in the traffic below. Make every change consistent and visible (for a wrong discount, also set the total the page shows). Prefer rules from the acceptance criteria: discounts, thresholds, limits, ownership, state transitions.
|
|
55
|
+
|
|
56
|
+
${criteriaFiles.map((f) => `----- ${f} -----\n${fs.readFileSync(f, 'utf8').slice(0, 8000)}`).join('\n\n')}
|
|
57
|
+
|
|
58
|
+
----- Tests -----
|
|
59
|
+
${tests.map((t) => `# ${t}\n${fs.readFileSync(t, 'utf8')}`).join('\n\n').slice(0, 12000)}
|
|
60
|
+
|
|
61
|
+
----- Traffic recorded while the tests ran -----
|
|
62
|
+
${trafficDigest(profile).slice(0, 15000)}
|
|
63
|
+
|
|
64
|
+
When done, reply with the number of mutants written.`;
|
|
65
|
+
fs.writeFileSync(path.join(dir, 'sabotage_prompt.txt'), prompt);
|
|
66
|
+
console.log(`Asking ${agent} for ${count} domain-specific mutants...`);
|
|
67
|
+
const r = await runAgent(agent, prompt);
|
|
68
|
+
if (r.code === 127) throw new Error(`could not start the ${agent} CLI: ${r.stderr.trim()}`);
|
|
69
|
+
const { mutants, rejected } = loadSaboteur(dir);
|
|
70
|
+
for (const m of mutants) console.log(` + ${m.id}: ${m.describe}`);
|
|
71
|
+
for (const x of rejected) console.log(` - rejected ${x.id}: ${x.errors.join('; ')}`);
|
|
72
|
+
console.log(`${mutants.length} saboteur mutants ready in ${out}. They run first in the next \`tripwire run\`.`);
|
|
73
|
+
return mutants.length ? 0 : 1;
|
|
74
|
+
}
|
package/src/config.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Project configuration: tripwire.config.json in the working directory, overridden by flags.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
export const DEFAULTS = {
|
|
6
|
+
app: 'http://localhost:3000', // the real app, behind the proxy
|
|
7
|
+
port: 8080, // the proxy port your tests point at (url: in frontmatter)
|
|
8
|
+
tests: '**/*_test.md',
|
|
9
|
+
reset: null, // { "url": "http://localhost:3000/api/__reset", "method": "POST" } or { "cmd": "npm run db:reset" }
|
|
10
|
+
maxMutants: 40,
|
|
11
|
+
timeout: 180,
|
|
12
|
+
threshold: 80,
|
|
13
|
+
healCheck: true,
|
|
14
|
+
healBudget: 5, // each heal check re-authors steps: ~15 credits measured
|
|
15
|
+
dir: '.tripwire',
|
|
16
|
+
agent: 'claude'
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export function loadConfig(cwd = process.cwd()) {
|
|
20
|
+
const file = path.join(cwd, 'tripwire.config.json');
|
|
21
|
+
if (!fs.existsSync(file)) return { ...DEFAULTS };
|
|
22
|
+
return { ...DEFAULTS, ...JSON.parse(fs.readFileSync(file, 'utf8')) };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function globToRegExp(glob) {
|
|
26
|
+
let re = '';
|
|
27
|
+
for (let i = 0; i < glob.length; i++) {
|
|
28
|
+
const c = glob[i];
|
|
29
|
+
if (c === '*' && glob[i + 1] === '*') {
|
|
30
|
+
if (glob[i + 2] === '/') { re += '(?:.*/)?'; i += 2; } else { re += '.*'; i += 1; }
|
|
31
|
+
} else if (c === '*') re += '[^/]*';
|
|
32
|
+
else if (c === '?') re += '[^/]';
|
|
33
|
+
else re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
34
|
+
}
|
|
35
|
+
return new RegExp('^' + re + '$');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Minimal glob for *_test.md discovery: **, * and ?.
|
|
39
|
+
export function findTests(pattern, cwd = process.cwd()) {
|
|
40
|
+
const re = globToRegExp(pattern);
|
|
41
|
+
const out = [];
|
|
42
|
+
const walk = (dir, rel) => {
|
|
43
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
44
|
+
if (e.name === 'node_modules' || e.name.startsWith('.') || e.name.startsWith('output-') || e.name.startsWith('helper-output-')) continue;
|
|
45
|
+
const r = rel ? `${rel}/${e.name}` : e.name;
|
|
46
|
+
if (e.isDirectory()) walk(path.join(dir, e.name), r);
|
|
47
|
+
else if (e.name.endsWith('_test.md') && re.test(r)) out.push(r);
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
walk(cwd, '');
|
|
51
|
+
return out.sort();
|
|
52
|
+
}
|
package/src/jsonpath.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Minimal JSON path helpers for paths like $.items[0].price or $.orders[2].
|
|
2
|
+
// Enough for mutant definitions; not a full JSONPath implementation.
|
|
3
|
+
|
|
4
|
+
export function parsePath(p) {
|
|
5
|
+
if (!p.startsWith('$')) throw new Error(`json path must start with $: ${p}`);
|
|
6
|
+
const parts = [];
|
|
7
|
+
const re = /\.([A-Za-z_$][\w$-]*)|\[(\d+)\]|\["([^"]+)"\]/g;
|
|
8
|
+
let m;
|
|
9
|
+
let pos = 1;
|
|
10
|
+
while ((m = re.exec(p))) {
|
|
11
|
+
if (m.index !== pos) throw new Error(`bad json path: ${p}`);
|
|
12
|
+
parts.push(m[1] ?? (m[2] !== undefined ? Number(m[2]) : m[3]));
|
|
13
|
+
pos = re.lastIndex;
|
|
14
|
+
}
|
|
15
|
+
if (pos !== p.length) throw new Error(`bad json path: ${p}`);
|
|
16
|
+
return parts;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function formatPath(parts) {
|
|
20
|
+
return '$' + parts.map((k) => (typeof k === 'number' ? `[${k}]` : /^[A-Za-z_$][\w$-]*$/.test(k) ? `.${k}` : `["${k}"]`)).join('');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function get(obj, p) {
|
|
24
|
+
let cur = obj;
|
|
25
|
+
for (const k of parsePath(p)) {
|
|
26
|
+
if (cur == null) return undefined;
|
|
27
|
+
cur = cur[k];
|
|
28
|
+
}
|
|
29
|
+
return cur;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Returns { parent, key } for the last segment, or null if the path does not exist.
|
|
33
|
+
export function locate(obj, p) {
|
|
34
|
+
const parts = parsePath(p);
|
|
35
|
+
if (parts.length === 0) return null;
|
|
36
|
+
let cur = obj;
|
|
37
|
+
for (const k of parts.slice(0, -1)) {
|
|
38
|
+
if (cur == null || typeof cur !== 'object') return null;
|
|
39
|
+
cur = cur[k];
|
|
40
|
+
}
|
|
41
|
+
if (cur == null || typeof cur !== 'object') return null;
|
|
42
|
+
const key = parts[parts.length - 1];
|
|
43
|
+
if (!(key in cur)) return null;
|
|
44
|
+
return { parent: cur, key };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Walk every leaf: cb(pathString, value, parentArrayPath?)
|
|
48
|
+
export function walk(obj, cb, parts = []) {
|
|
49
|
+
if (Array.isArray(obj)) {
|
|
50
|
+
cb(formatPath(parts), obj, parts);
|
|
51
|
+
obj.forEach((v, i) => walk(v, cb, [...parts, i]));
|
|
52
|
+
} else if (obj && typeof obj === 'object') {
|
|
53
|
+
if (parts.length) cb(formatPath(parts), obj, parts);
|
|
54
|
+
for (const [k, v] of Object.entries(obj)) walk(v, cb, [...parts, k]);
|
|
55
|
+
} else {
|
|
56
|
+
cb(formatPath(parts), obj, parts);
|
|
57
|
+
}
|
|
58
|
+
}
|
package/src/kane.js
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// Thin wrapper around the kane-cli binary: run one test.md, collect NDJSON, find its evidence pack.
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
|
|
6
|
+
const isWin = process.platform === 'win32';
|
|
7
|
+
const quote = (a) => (/^[\w./:=@-]+$/.test(a) ? a : `"${String(a).replace(/"/g, '\\"')}"`);
|
|
8
|
+
|
|
9
|
+
export function kaneBin() {
|
|
10
|
+
return process.env.TRIPWIRE_KANE || 'kane-cli';
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Run kane-cli with stdin closed (non-TTY => NDJSON, no prompts). Our own kill timer guards against
|
|
14
|
+
// runs that ignore --timeout (kane-cli issue #207).
|
|
15
|
+
export function runKane(args, { cwd = process.cwd(), killAfterMs = 240000, logFile } = {}) {
|
|
16
|
+
return new Promise((resolve) => {
|
|
17
|
+
const started = Date.now();
|
|
18
|
+
const bin = kaneBin();
|
|
19
|
+
// TRIPWIRE_KANE may be a whole command ("node fake-kane.mjs"); then it is used as written.
|
|
20
|
+
const useShell = isWin || /\s/.test(bin);
|
|
21
|
+
const cmd = [/\s/.test(bin) ? bin : quote(bin), ...args.map(quote)].join(' ');
|
|
22
|
+
const child = useShell
|
|
23
|
+
? spawn(cmd, { cwd, shell: true, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
|
|
24
|
+
: spawn(bin, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
25
|
+
const events = [];
|
|
26
|
+
let out = '';
|
|
27
|
+
let err = '';
|
|
28
|
+
let buf = '';
|
|
29
|
+
const log = logFile ? fs.createWriteStream(logFile) : null;
|
|
30
|
+
child.stdout.on('data', (c) => {
|
|
31
|
+
const s = c.toString();
|
|
32
|
+
out += s;
|
|
33
|
+
log?.write(s);
|
|
34
|
+
buf += s;
|
|
35
|
+
let i;
|
|
36
|
+
while ((i = buf.indexOf('\n')) >= 0) {
|
|
37
|
+
const line = buf.slice(0, i).trim();
|
|
38
|
+
buf = buf.slice(i + 1);
|
|
39
|
+
if (line.startsWith('{')) { try { events.push(JSON.parse(line)); } catch { /* not json */ } }
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
child.stderr.on('data', (c) => { err += c.toString(); log?.write(c); });
|
|
43
|
+
let timedOut = false;
|
|
44
|
+
const timer = setTimeout(() => {
|
|
45
|
+
timedOut = true;
|
|
46
|
+
if (isWin) spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
|
|
47
|
+
else child.kill('SIGKILL');
|
|
48
|
+
}, killAfterMs);
|
|
49
|
+
child.on('close', (code) => {
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
log?.end();
|
|
52
|
+
resolve({ code: timedOut ? 3 : code ?? 2, timedOut, events, stdout: out, stderr: err, durationMs: Date.now() - started });
|
|
53
|
+
});
|
|
54
|
+
child.on('error', (e) => {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
resolve({ code: 2, events, stdout: out, stderr: String(e), durationMs: Date.now() - started, spawnError: true });
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Pull the useful bits out of a testmd run's NDJSON stream.
|
|
62
|
+
export function summarizeEvents(events) {
|
|
63
|
+
const done = [...events].reverse().find((e) => e.type === 'test_md_done' || e.event === 'test_md_done');
|
|
64
|
+
const runEnds = events.filter((e) => e.type === 'run_end' || e.event === 'run_end');
|
|
65
|
+
const text = JSON.stringify(events);
|
|
66
|
+
const heal = events.filter((e) => /heal/i.test(e.type || e.event || '') || e.healed === true || e.heal);
|
|
67
|
+
const credits = runEnds.reduce((s, e) => s + (Number(e.credits ?? e.credits_used ?? e.usage?.credits) || 0), 0);
|
|
68
|
+
const failedEnds = runEnds.filter((e) => e.status === 'failed' || e.status === 'error');
|
|
69
|
+
const failure = failedEnds.map((e) => e.failure_reason || e.reason || e.error).filter(Boolean)[0];
|
|
70
|
+
// Kane triages every failed replay; keep its verdict (product bug vs automation bug)
|
|
71
|
+
const bug = events.filter((e) => e.type === 'test_md_bug_verdict').map((e) => ({ confirmed: e.confirmed, family: e.family, category: e.category, title: e.bug_title, oneLiner: e.one_liner, confidence: e.confidence }))[0];
|
|
72
|
+
const failedStep = events.filter((e) => e.type === 'test_md_step_end' && e.status === 'failed').map((e) => e.step_index)[0];
|
|
73
|
+
return {
|
|
74
|
+
bugVerdict: bug,
|
|
75
|
+
failedStep,
|
|
76
|
+
status: done?.overall_status ?? done?.status,
|
|
77
|
+
sessionId: done?.session_id ?? runEnds.find((e) => e.session_id)?.session_id,
|
|
78
|
+
shareUrl: done?.share_url,
|
|
79
|
+
credits,
|
|
80
|
+
healEvents: heal.length,
|
|
81
|
+
mentionsHeal: /"(healed|heal_applied|reauthor|re-author)/i.test(text),
|
|
82
|
+
failureReason: failure
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function newestPack(cwd, sinceMs) {
|
|
87
|
+
const dir = path.join(cwd, '.testmuai', 'evidence');
|
|
88
|
+
if (!fs.existsSync(dir)) return null;
|
|
89
|
+
const packs = fs.readdirSync(dir)
|
|
90
|
+
.filter((f) => f.endsWith('.evidence'))
|
|
91
|
+
.map((f) => ({ f: path.join(dir, f), t: fs.statSync(path.join(dir, f)).mtimeMs }))
|
|
92
|
+
.filter((p) => p.t >= sinceMs - 2000)
|
|
93
|
+
.sort((a, b) => b.t - a.t);
|
|
94
|
+
return packs[0]?.f ?? null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Kane failures that happen before any step runs (no browser, no credits spent) and are worth
|
|
98
|
+
// waiting out: Test Manager not answering during auth resolution, network resets, rate limits.
|
|
99
|
+
export const TRANSIENT = /TMS did not return an org_id|auth resolution failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|fetch failed|network error|\b429\b|\b50[234]\b/i;
|
|
100
|
+
export const isTransient = (r) => r.code === 2 && !r.events.some((e) => e.type === 'test_md_step_start') && TRANSIENT.test(r.stderr + r.stdout);
|
|
101
|
+
|
|
102
|
+
const pause = (ms) => new Promise((res) => setTimeout(res, ms));
|
|
103
|
+
|
|
104
|
+
// One test against whatever the proxy currently serves. Transient pre-run failures are retried
|
|
105
|
+
// with backoff (15 s, 45 s, 90 s) since they cost nothing and usually clear on their own.
|
|
106
|
+
export async function runTest(file, { heal = false, timeout = 180, cwd, logFile, extra = [], backoff = [15000, 45000, 90000], onRetry } = {}) {
|
|
107
|
+
const args = ['testmd', 'run', file, '--agent', '--headless', '--timeout', String(timeout)];
|
|
108
|
+
if (!heal) args.push('--no-adaptive-heal');
|
|
109
|
+
args.push(...extra);
|
|
110
|
+
let r;
|
|
111
|
+
let started;
|
|
112
|
+
for (let i = 0; ; i++) {
|
|
113
|
+
started = Date.now();
|
|
114
|
+
r = await runKane(args, { cwd, killAfterMs: (timeout + 60) * 1000, logFile });
|
|
115
|
+
if (!isTransient(r) || i >= backoff.length) break;
|
|
116
|
+
onRetry?.(i + 1, (r.stderr || r.stdout).trim().split('\n').pop());
|
|
117
|
+
await pause(backoff[i]);
|
|
118
|
+
}
|
|
119
|
+
return { ...r, ...summarizeEvents(r.events), pack: newestPack(cwd || process.cwd(), started), transient: isTransient(r) };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Judge a sealed pack against a policy file; returns { verdict: pass|fail|errored|unknown, file, raw }.
|
|
123
|
+
export async function verdict(pack, policyFile, { cwd } = {}) {
|
|
124
|
+
if (!pack) return { verdict: 'unknown', reason: 'no evidence pack found' };
|
|
125
|
+
const r = await runKane(['evidence', 'verdict', pack, '--policy', policyFile], { cwd, killAfterMs: 60000 });
|
|
126
|
+
const v = r.code === 0 ? 'pass' : r.code === 1 ? 'fail' : r.code === 3 ? 'errored' : 'unknown';
|
|
127
|
+
const m = (r.stdout + r.stderr).match(/[^\s]+\.verdict\.json/);
|
|
128
|
+
let raw = null;
|
|
129
|
+
if (m && fs.existsSync(m[0])) { try { raw = JSON.parse(fs.readFileSync(m[0], 'utf8')); } catch { /* ignore */ } }
|
|
130
|
+
return { verdict: v, file: m?.[0], raw, output: (r.stdout + r.stderr).trim().split('\n').slice(-3).join('\n') };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Read a numeric metric out of a verdict file, whatever its exact shape.
|
|
134
|
+
export function metricFromVerdict(raw, metric) {
|
|
135
|
+
if (!raw) return undefined;
|
|
136
|
+
let found;
|
|
137
|
+
const visit = (o) => {
|
|
138
|
+
if (found !== undefined || !o || typeof o !== 'object') return;
|
|
139
|
+
if (o.metric === metric && (typeof o.value === 'number' || typeof o.actual === 'number')) { found = o.value ?? o.actual; return; }
|
|
140
|
+
for (const v of Object.values(o)) visit(v);
|
|
141
|
+
};
|
|
142
|
+
visit(raw);
|
|
143
|
+
return found;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function balance(cwd) {
|
|
147
|
+
const r = await runKane(['balance', '--json'], { cwd, killAfterMs: 30000 });
|
|
148
|
+
const m = (r.stdout || '').match(/"?(?:balance|credits|available)"?\s*[:=]\s*([\d.]+)/i) || (r.stdout || '').match(/([\d,]+(?:\.\d+)?)\s*credits/i);
|
|
149
|
+
return m ? Number(m[1].replace(/,/g, '')) : undefined;
|
|
150
|
+
}
|