tripwire-e2e 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,74 +1,85 @@
1
- // tripwire sabotage: the coding agent in its second role. It reads the acceptance criteria, the
2
- // tests and the traffic the suite produced, and proposes domain-specific bugs ("coupon applied
3
- // twice", "free shipping one cent too early") as mutant definitions in .tripwire/saboteur.json.
4
- import fs from 'node:fs';
5
- import path from 'node:path';
6
- import { loadConfig, findTests } from '../config.js';
7
- import { runAgent } from '../agent.js';
8
- import { loadSaboteur } from '../../operators/agent-proposed.js';
9
-
10
- const HELP = `tripwire sabotage [options]
11
-
12
- --agent claude|codex which agent proposes mutants (default from config)
13
- --criteria <file> acceptance criteria / spec to read (default: README.md, docs/*.md if present)
14
- --count <n> how many mutants to ask for (default 8)
15
- --dir <path> state directory (default .tripwire)`;
16
-
17
- function trafficDigest(profile) {
18
- const seen = new Map();
19
- for (const t of profile.tests) for (const e of t.traffic) {
20
- const k = `${e.method} ${e.path}`;
21
- if (seen.has(k) || e.type !== 'json') continue;
22
- seen.set(k, `${k} -> ${e.status}${e.req ? ` request ${JSON.stringify(e.req).slice(0, 200)}` : ''}${e.json !== undefined ? `\n response ${JSON.stringify(e.json).slice(0, 600)}` : ''}`);
23
- }
24
- return [...seen.values()].join('\n');
25
- }
26
-
27
- export default async function sabotage(argv, parseArgs) {
28
- const { values } = parseArgs({ args: argv, options: { agent: { type: 'string' }, criteria: { type: 'string' }, count: { type: 'string' }, dir: { type: 'string' }, help: { type: 'boolean', short: 'h' } } });
29
- if (values.help) { console.log(HELP); return 0; }
30
- const cfg = loadConfig();
31
- const dir = path.resolve(values.dir || cfg.dir);
32
- const agent = values.agent || cfg.agent || 'claude';
33
- const trafficFile = path.join(dir, 'traffic.json');
34
- if (!fs.existsSync(trafficFile)) throw new Error('no baseline yet: run `tripwire run --baseline-only` first');
35
- const profile = JSON.parse(fs.readFileSync(trafficFile, 'utf8'));
36
- const tests = findTests(cfg.tests);
37
- const criteriaFiles = values.criteria ? [values.criteria] : ['ACCEPTANCE.md', 'README.md'].filter((f) => fs.existsSync(f));
38
- const out = path.join(dir, 'saboteur.json');
39
- const count = Number(values.count || 8);
40
-
41
- const prompt = `You are the saboteur. Propose ${count} realistic, domain-specific bugs for the web app below, the kind a developer could really ship, that break a business rule a user would notice. Express each one as a Tripwire mutant: a rewrite of one HTTP response the app already sends (the app code is never changed).
42
-
43
- Write a JSON array to ${out} (create or overwrite the file). Each element:
44
- {
45
- "id": "short-kebab-name",
46
- "describe": "One sentence a user would notice, e.g. 'The 10% coupon takes 20% off'",
47
- "expect": "What a good test would check",
48
- "match": { "method": "GET", "path": "/api/..." },
49
- "action": <one of>
50
- { "type": "json", "ops": [ { "op": "set"|"add"|"scale"|"remove-index"|"dup-index"|"swap"|"push", "path": "$.field.or[0].nested", "value": ..., "index": 0, "a": "k1", "b": "k2" } ] }
51
- { "type": "respond", "status": 200, "body": { ...a whole fake response... } }
52
- { "type": "status-map", "from": [422], "to": 200 }
53
- }
54
- Use only routes and JSON paths that appear in the traffic below. Make every change consistent and visible (for a wrong discount, also set the total the page shows). Prefer rules from the acceptance criteria: discounts, thresholds, limits, ownership, state transitions.
55
-
56
- ${criteriaFiles.map((f) => `----- ${f} -----\n${fs.readFileSync(f, 'utf8').slice(0, 8000)}`).join('\n\n')}
57
-
58
- ----- Tests -----
59
- ${tests.map((t) => `# ${t}\n${fs.readFileSync(t, 'utf8')}`).join('\n\n').slice(0, 12000)}
60
-
61
- ----- Traffic recorded while the tests ran -----
62
- ${trafficDigest(profile).slice(0, 15000)}
63
-
64
- When done, reply with the number of mutants written.`;
65
- fs.writeFileSync(path.join(dir, 'sabotage_prompt.txt'), prompt);
66
- console.log(`Asking ${agent} for ${count} domain-specific mutants...`);
67
- const r = await runAgent(agent, prompt);
68
- if (r.code === 127) throw new Error(`could not start the ${agent} CLI: ${r.stderr.trim()}`);
69
- const { mutants, rejected } = loadSaboteur(dir);
70
- for (const m of mutants) console.log(` + ${m.id}: ${m.describe}`);
71
- for (const x of rejected) console.log(` - rejected ${x.id}: ${x.errors.join('; ')}`);
72
- console.log(`${mutants.length} saboteur mutants ready in ${out}. They run first in the next \`tripwire run\`.`);
73
- return mutants.length ? 0 : 1;
74
- }
1
+ // tripwire sabotage: the coding agent in its second role. It reads the acceptance criteria, the
2
+ // tests and the traffic the suite produced, and proposes domain-specific bugs ("coupon applied
3
+ // twice", "free shipping one cent too early") as mutant definitions in .tripwire/saboteur.json.
4
+ import fs from 'node:fs';
5
+ import path from 'node:path';
6
+ import { loadConfig, findTests } from '../config.js';
7
+ import { runAgent } from '../agent.js';
8
+ import { loadSaboteur } from '../../operators/agent-proposed.js';
9
+
10
+ const HELP = `tripwire sabotage [options]
11
+
12
+ --agent claude|codex which agent proposes mutants (default from config)
13
+ --criteria <file> acceptance criteria / spec to read (default: README.md, docs/*.md if present)
14
+ --count <n> how many mutants to ask for (default 8)
15
+ --dir <path> state directory (default .tripwire)`;
16
+
17
+ function trafficDigest(profile) {
18
+ const seen = new Map();
19
+ for (const t of profile.tests) for (const e of t.traffic) {
20
+ const k = `${e.method} ${e.path}`;
21
+ if (seen.has(k) || e.type !== 'json') continue;
22
+ seen.set(k, `${k} -> ${e.status}${e.req ? ` request ${JSON.stringify(e.req).slice(0, 200)}` : ''}${e.json !== undefined ? `\n response ${JSON.stringify(e.json).slice(0, 600)}` : ''}`);
23
+ }
24
+ return [...seen.values()].join('\n');
25
+ }
26
+
27
+ export default async function sabotage(argv, parseArgs) {
28
+ const { values } = parseArgs({ args: argv, options: { agent: { type: 'string' }, criteria: { type: 'string' }, count: { type: 'string' }, dir: { type: 'string' }, help: { type: 'boolean', short: 'h' } } });
29
+ if (values.help) { console.log(HELP); return 0; }
30
+ const cfg = loadConfig();
31
+ const dir = path.resolve(values.dir || cfg.dir);
32
+ const agent = values.agent || cfg.agent || 'claude';
33
+ const trafficFile = path.join(dir, 'traffic.json');
34
+ if (!fs.existsSync(trafficFile)) throw new Error('no baseline yet: run `tripwire run --baseline-only` first');
35
+ const profile = JSON.parse(fs.readFileSync(trafficFile, 'utf8'));
36
+ const tests = findTests(cfg.tests);
37
+ const criteriaFiles = values.criteria ? [values.criteria] : ['ACCEPTANCE.md', 'README.md'].filter((f) => fs.existsSync(f));
38
+ const out = path.join(dir, 'saboteur.json');
39
+ const count = Number(values.count || 8);
40
+
41
+ const prompt = `You are the saboteur. Propose ${count} realistic, domain-specific bugs for the web app below, the kind a developer could really ship, that break a business rule a user would notice. Express each one as a Tripwire mutant: a rewrite of one HTTP response the app already sends (the app code is never changed).
42
+
43
+ Write a JSON array to ${out} (create or overwrite the file). Each element:
44
+ {
45
+ "id": "short-kebab-name",
46
+ "describe": "One sentence a user would notice, e.g. 'The 10% coupon takes 20% off'",
47
+ "expect": "What a good test would check",
48
+ "match": { "method": "GET", "path": "/api/..." },
49
+ "action": <one of>
50
+ { "type": "json", "ops": [ { "op": "set"|"add"|"scale"|"remove-index"|"dup-index"|"swap"|"push", "path": "$.field.or[0].nested", "value": ..., "index": 0, "a": "k1", "b": "k2" } ] }
51
+ { "type": "respond", "status": 200, "body": { ...a whole fake response... } }
52
+ { "type": "status-map", "from": [422], "to": 200 }
53
+ }
54
+ Use only routes and JSON paths that appear in the traffic below. Make every change consistent and visible (for a wrong discount, also set the total the page shows). Prefer rules from the acceptance criteria: discounts, thresholds, limits, ownership, state transitions. Prefer responses the page actually displays (usually GET responses): many pages ignore the body of a write response and re-read the data, so a bug placed only in a write response may never reach the screen.
55
+
56
+ ${criteriaFiles.map((f) => `----- ${f} -----\n${fs.readFileSync(f, 'utf8').slice(0, 8000)}`).join('\n\n')}
57
+
58
+ ----- Tests -----
59
+ ${tests.map((t) => `# ${t}\n${fs.readFileSync(t, 'utf8')}`).join('\n\n').slice(0, 12000)}
60
+
61
+ ----- Traffic recorded while the tests ran -----
62
+ ${trafficDigest(profile).slice(0, 15000)}
63
+
64
+ When done, reply with the number of mutants written.`;
65
+ fs.writeFileSync(path.join(dir, 'sabotage_prompt.txt'), prompt);
66
+ console.log(`Asking ${agent} for ${count} domain-specific mutants...`);
67
+ const r = await runAgent(agent, prompt);
68
+ if (r.code === 127) throw new Error(`could not start the ${agent} CLI: ${r.stderr.trim()}`);
69
+ fs.writeFileSync(path.join(dir, 'sabotage_reply.txt'), `${r.stdout}\n----- stderr -----\n${r.stderr}`);
70
+ // Some agents answer with the JSON instead of writing the file: take the largest JSON array.
71
+ if (!fs.existsSync(out)) {
72
+ const arrays = [...r.stdout.matchAll(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/g)].map((x) => x[1]);
73
+ const whole = r.stdout.indexOf('['), last = r.stdout.lastIndexOf(']');
74
+ if (whole >= 0 && last > whole) arrays.push(r.stdout.slice(whole, last + 1));
75
+ for (const a of arrays.sort((x, y) => y.length - x.length)) {
76
+ try { const parsed = JSON.parse(a); if (Array.isArray(parsed) && parsed.length) { fs.writeFileSync(out, JSON.stringify(parsed, null, 2)); break; } } catch { /* try the next */ }
77
+ }
78
+ }
79
+ if (!fs.existsSync(out)) console.log(`The agent did not produce mutants. Its reply is in ${path.join(dir, 'sabotage_reply.txt')}`);
80
+ const { mutants, rejected } = loadSaboteur(dir);
81
+ for (const m of mutants) console.log(` + ${m.id}: ${m.describe}`);
82
+ for (const x of rejected) console.log(` - rejected ${x.id}: ${x.errors.join('; ')}`);
83
+ console.log(`${mutants.length} saboteur mutants ready in ${out}. They run first in the next \`tripwire run\`.`);
84
+ return mutants.length ? 0 : 1;
85
+ }
package/src/kane.js CHANGED
@@ -1,150 +1,158 @@
1
- // Thin wrapper around the kane-cli binary: run one test.md, collect NDJSON, find its evidence pack.
2
- import { spawn } from 'node:child_process';
3
- import fs from 'node:fs';
4
- import path from 'node:path';
5
-
6
- const isWin = process.platform === 'win32';
7
- const quote = (a) => (/^[\w./:=@-]+$/.test(a) ? a : `"${String(a).replace(/"/g, '\\"')}"`);
8
-
9
- export function kaneBin() {
10
- return process.env.TRIPWIRE_KANE || 'kane-cli';
11
- }
12
-
13
- // Run kane-cli with stdin closed (non-TTY => NDJSON, no prompts). Our own kill timer guards against
14
- // runs that ignore --timeout (kane-cli issue #207).
15
- export function runKane(args, { cwd = process.cwd(), killAfterMs = 240000, logFile } = {}) {
16
- return new Promise((resolve) => {
17
- const started = Date.now();
18
- const bin = kaneBin();
19
- // TRIPWIRE_KANE may be a whole command ("node fake-kane.mjs"); then it is used as written.
20
- const useShell = isWin || /\s/.test(bin);
21
- const cmd = [/\s/.test(bin) ? bin : quote(bin), ...args.map(quote)].join(' ');
22
- const child = useShell
23
- ? spawn(cmd, { cwd, shell: true, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
24
- : spawn(bin, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
25
- const events = [];
26
- let out = '';
27
- let err = '';
28
- let buf = '';
29
- const log = logFile ? fs.createWriteStream(logFile) : null;
30
- child.stdout.on('data', (c) => {
31
- const s = c.toString();
32
- out += s;
33
- log?.write(s);
34
- buf += s;
35
- let i;
36
- while ((i = buf.indexOf('\n')) >= 0) {
37
- const line = buf.slice(0, i).trim();
38
- buf = buf.slice(i + 1);
39
- if (line.startsWith('{')) { try { events.push(JSON.parse(line)); } catch { /* not json */ } }
40
- }
41
- });
42
- child.stderr.on('data', (c) => { err += c.toString(); log?.write(c); });
43
- let timedOut = false;
44
- const timer = setTimeout(() => {
45
- timedOut = true;
46
- if (isWin) spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
47
- else child.kill('SIGKILL');
48
- }, killAfterMs);
49
- child.on('close', (code) => {
50
- clearTimeout(timer);
51
- log?.end();
52
- resolve({ code: timedOut ? 3 : code ?? 2, timedOut, events, stdout: out, stderr: err, durationMs: Date.now() - started });
53
- });
54
- child.on('error', (e) => {
55
- clearTimeout(timer);
56
- resolve({ code: 2, events, stdout: out, stderr: String(e), durationMs: Date.now() - started, spawnError: true });
57
- });
58
- });
59
- }
60
-
61
- // Pull the useful bits out of a testmd run's NDJSON stream.
62
- export function summarizeEvents(events) {
63
- const done = [...events].reverse().find((e) => e.type === 'test_md_done' || e.event === 'test_md_done');
64
- const runEnds = events.filter((e) => e.type === 'run_end' || e.event === 'run_end');
65
- const text = JSON.stringify(events);
66
- const heal = events.filter((e) => /heal/i.test(e.type || e.event || '') || e.healed === true || e.heal);
67
- const credits = runEnds.reduce((s, e) => s + (Number(e.credits ?? e.credits_used ?? e.usage?.credits) || 0), 0);
68
- const failedEnds = runEnds.filter((e) => e.status === 'failed' || e.status === 'error');
69
- const failure = failedEnds.map((e) => e.failure_reason || e.reason || e.error).filter(Boolean)[0];
70
- // Kane triages every failed replay; keep its verdict (product bug vs automation bug)
71
- const bug = events.filter((e) => e.type === 'test_md_bug_verdict').map((e) => ({ confirmed: e.confirmed, family: e.family, category: e.category, title: e.bug_title, oneLiner: e.one_liner, confidence: e.confidence }))[0];
72
- const failedStep = events.filter((e) => e.type === 'test_md_step_end' && e.status === 'failed').map((e) => e.step_index)[0];
73
- return {
74
- bugVerdict: bug,
75
- failedStep,
76
- status: done?.overall_status ?? done?.status,
77
- sessionId: done?.session_id ?? runEnds.find((e) => e.session_id)?.session_id,
78
- shareUrl: done?.share_url,
79
- credits,
80
- healEvents: heal.length,
81
- mentionsHeal: /"(healed|heal_applied|reauthor|re-author)/i.test(text),
82
- failureReason: failure
83
- };
84
- }
85
-
86
- export function newestPack(cwd, sinceMs) {
87
- const dir = path.join(cwd, '.testmuai', 'evidence');
88
- if (!fs.existsSync(dir)) return null;
89
- const packs = fs.readdirSync(dir)
90
- .filter((f) => f.endsWith('.evidence'))
91
- .map((f) => ({ f: path.join(dir, f), t: fs.statSync(path.join(dir, f)).mtimeMs }))
92
- .filter((p) => p.t >= sinceMs - 2000)
93
- .sort((a, b) => b.t - a.t);
94
- return packs[0]?.f ?? null;
95
- }
96
-
97
- // Kane failures that happen before any step runs (no browser, no credits spent) and are worth
98
- // waiting out: Test Manager not answering during auth resolution, network resets, rate limits.
99
- export const TRANSIENT = /TMS did not return an org_id|auth resolution failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|fetch failed|network error|\b429\b|\b50[234]\b/i;
100
- export const isTransient = (r) => r.code === 2 && !r.events.some((e) => e.type === 'test_md_step_start') && TRANSIENT.test(r.stderr + r.stdout);
101
-
102
- const pause = (ms) => new Promise((res) => setTimeout(res, ms));
103
-
104
- // One test against whatever the proxy currently serves. Transient pre-run failures are retried
105
- // with backoff (15 s, 45 s, 90 s) since they cost nothing and usually clear on their own.
106
- export async function runTest(file, { heal = false, timeout = 180, cwd, logFile, extra = [], backoff = [15000, 45000, 90000], onRetry } = {}) {
107
- const args = ['testmd', 'run', file, '--agent', '--headless', '--timeout', String(timeout)];
108
- if (!heal) args.push('--no-adaptive-heal');
109
- args.push(...extra);
110
- let r;
111
- let started;
112
- for (let i = 0; ; i++) {
113
- started = Date.now();
114
- r = await runKane(args, { cwd, killAfterMs: (timeout + 60) * 1000, logFile });
115
- if (!isTransient(r) || i >= backoff.length) break;
116
- onRetry?.(i + 1, (r.stderr || r.stdout).trim().split('\n').pop());
117
- await pause(backoff[i]);
118
- }
119
- return { ...r, ...summarizeEvents(r.events), pack: newestPack(cwd || process.cwd(), started), transient: isTransient(r) };
120
- }
121
-
122
- // Judge a sealed pack against a policy file; returns { verdict: pass|fail|errored|unknown, file, raw }.
123
- export async function verdict(pack, policyFile, { cwd } = {}) {
124
- if (!pack) return { verdict: 'unknown', reason: 'no evidence pack found' };
125
- const r = await runKane(['evidence', 'verdict', pack, '--policy', policyFile], { cwd, killAfterMs: 60000 });
126
- const v = r.code === 0 ? 'pass' : r.code === 1 ? 'fail' : r.code === 3 ? 'errored' : 'unknown';
127
- const m = (r.stdout + r.stderr).match(/[^\s]+\.verdict\.json/);
128
- let raw = null;
129
- if (m && fs.existsSync(m[0])) { try { raw = JSON.parse(fs.readFileSync(m[0], 'utf8')); } catch { /* ignore */ } }
130
- return { verdict: v, file: m?.[0], raw, output: (r.stdout + r.stderr).trim().split('\n').slice(-3).join('\n') };
131
- }
132
-
133
- // Read a numeric metric out of a verdict file, whatever its exact shape.
134
- export function metricFromVerdict(raw, metric) {
135
- if (!raw) return undefined;
136
- let found;
137
- const visit = (o) => {
138
- if (found !== undefined || !o || typeof o !== 'object') return;
139
- if (o.metric === metric && (typeof o.value === 'number' || typeof o.actual === 'number')) { found = o.value ?? o.actual; return; }
140
- for (const v of Object.values(o)) visit(v);
141
- };
142
- visit(raw);
143
- return found;
144
- }
145
-
146
- export async function balance(cwd) {
147
- const r = await runKane(['balance', '--json'], { cwd, killAfterMs: 30000 });
148
- const m = (r.stdout || '').match(/"?(?:balance|credits|available)"?\s*[:=]\s*([\d.]+)/i) || (r.stdout || '').match(/([\d,]+(?:\.\d+)?)\s*credits/i);
149
- return m ? Number(m[1].replace(/,/g, '')) : undefined;
150
- }
1
+ // Thin wrapper around the kane-cli binary: run one test.md, collect NDJSON, find its evidence pack.
2
+ import { spawn } from 'node:child_process';
3
+ import fs from 'node:fs';
4
+ import path from 'node:path';
5
+
6
+ const isWin = process.platform === 'win32';
7
+ const quote = (a) => (/^[\w./:=@-]+$/.test(a) ? a : `"${String(a).replace(/"/g, '\\"')}"`);
8
+
9
+ export function kaneBin() {
10
+ return process.env.TRIPWIRE_KANE || 'kane-cli';
11
+ }
12
+
13
+ // Run kane-cli with stdin closed (non-TTY => NDJSON, no prompts). Our own kill timer guards against
14
+ // runs that ignore --timeout (kane-cli issue #207).
15
+ export function runKane(args, { cwd = process.cwd(), killAfterMs = 240000, logFile } = {}) {
16
+ return new Promise((resolve) => {
17
+ const started = Date.now();
18
+ const bin = kaneBin();
19
+ // TRIPWIRE_KANE may be a whole command ("node fake-kane.mjs"); then it is used as written.
20
+ const useShell = isWin || /\s/.test(bin);
21
+ const cmd = [/\s/.test(bin) ? bin : quote(bin), ...args.map(quote)].join(' ');
22
+ const child = useShell
23
+ ? spawn(cmd, { cwd, shell: true, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
24
+ : spawn(bin, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
25
+ const events = [];
26
+ let out = '';
27
+ let err = '';
28
+ let buf = '';
29
+ const log = logFile ? fs.createWriteStream(logFile) : null;
30
+ child.stdout.on('data', (c) => {
31
+ const s = c.toString();
32
+ out += s;
33
+ log?.write(s);
34
+ buf += s;
35
+ let i;
36
+ while ((i = buf.indexOf('\n')) >= 0) {
37
+ const line = buf.slice(0, i).trim();
38
+ buf = buf.slice(i + 1);
39
+ if (line.startsWith('{')) { try { events.push(JSON.parse(line)); } catch { /* not json */ } }
40
+ }
41
+ });
42
+ child.stderr.on('data', (c) => { err += c.toString(); log?.write(c); });
43
+ let timedOut = false;
44
+ const timer = setTimeout(() => {
45
+ timedOut = true;
46
+ if (isWin) spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
47
+ else child.kill('SIGKILL');
48
+ }, killAfterMs);
49
+ child.on('close', (code) => {
50
+ clearTimeout(timer);
51
+ log?.end();
52
+ resolve({ code: timedOut ? 3 : code ?? 2, timedOut, events, stdout: out, stderr: err, durationMs: Date.now() - started });
53
+ });
54
+ child.on('error', (e) => {
55
+ clearTimeout(timer);
56
+ resolve({ code: 2, events, stdout: out, stderr: String(e), durationMs: Date.now() - started, spawnError: true });
57
+ });
58
+ });
59
+ }
60
+
61
+ // Pull the useful bits out of a testmd run's NDJSON stream.
62
+ export function summarizeEvents(events) {
63
+ const done = [...events].reverse().find((e) => e.type === 'test_md_done' || e.event === 'test_md_done');
64
+ const runEnds = events.filter((e) => e.type === 'run_end' || e.event === 'run_end');
65
+ const text = JSON.stringify(events);
66
+ const heal = events.filter((e) => /heal/i.test(e.type || e.event || '') || e.healed === true || e.heal);
67
+ const credits = runEnds.reduce((s, e) => s + (Number(e.credits ?? e.credits_used ?? e.usage?.credits) || 0), 0);
68
+ const failedEnds = runEnds.filter((e) => e.status === 'failed' || e.status === 'error');
69
+ const failure = failedEnds.map((e) => e.failure_reason || e.reason || e.error).filter(Boolean)[0];
70
+ // Kane triages every failed replay; keep its verdict (product bug vs automation bug)
71
+ const bug = events.filter((e) => e.type === 'test_md_bug_verdict').map((e) => ({ confirmed: e.confirmed, family: e.family, category: e.category, title: e.bug_title, oneLiner: e.one_liner, confidence: e.confidence }))[0];
72
+ const failedStep = events.filter((e) => e.type === 'test_md_step_end' && e.status === 'failed').map((e) => e.step_index)[0];
73
+ return {
74
+ bugVerdict: bug,
75
+ failedStep,
76
+ status: done?.overall_status ?? done?.status,
77
+ sessionId: done?.session_id ?? runEnds.find((e) => e.session_id)?.session_id,
78
+ shareUrl: done?.share_url,
79
+ credits,
80
+ healEvents: heal.length,
81
+ mentionsHeal: /"(healed|heal_applied|reauthor|re-author)/i.test(text),
82
+ failureReason: failure
83
+ };
84
+ }
85
+
86
+ export function newestPack(cwd, sinceMs) {
87
+ const dir = path.join(cwd, '.testmuai', 'evidence');
88
+ if (!fs.existsSync(dir)) return null;
89
+ const packs = fs.readdirSync(dir)
90
+ .filter((f) => f.endsWith('.evidence'))
91
+ .map((f) => ({ f: path.join(dir, f), t: fs.statSync(path.join(dir, f)).mtimeMs }))
92
+ .filter((p) => p.t >= sinceMs - 2000)
93
+ .sort((a, b) => b.t - a.t);
94
+ return packs[0]?.f ?? null;
95
+ }
96
+
97
+ // Kane failures that happen before any step runs (no browser, no credits spent) and are worth
98
+ // waiting out: Test Manager not answering during auth resolution, network resets, rate limits.
99
+ export const TRANSIENT = /TMS did not return an org_id|Token refresh failed|auth resolution failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|fetch failed|network error|\b429\b|\b50[234]\b/i;
100
+ export const isTransient = (r) => r.code === 2 && !r.events.some((e) => e.type === 'test_md_step_start') && TRANSIENT.test(r.stderr + r.stdout);
101
+
102
+ const pause = (ms) => new Promise((res) => setTimeout(res, ms));
103
+
104
+ // kane-cli's --timeout is per step, so the safety timer around a whole test allows every step its
105
+ // timeout, plus time for Kane to triage a failure and seal the evidence pack (issue #207 guard).
106
+ export function killAfterFor(file, timeout, cwd = process.cwd()) {
107
+ let steps = 1;
108
+ try { steps = Math.max(1, (fs.readFileSync(path.resolve(cwd, file), 'utf8').match(/^## /gm) || []).length); } catch { /* unreadable: one step */ }
109
+ return (timeout * steps + 180) * 1000;
110
+ }
111
+
112
+ // One test against whatever the proxy currently serves. Transient pre-run failures are retried
113
+ // with backoff (15 s, 45 s, 90 s) since they cost nothing and usually clear on their own.
114
+ export async function runTest(file, { heal = false, timeout = 180, cwd, logFile, extra = [], backoff = [15000, 45000, 90000], onRetry } = {}) {
115
+ const args = ['testmd', 'run', file, '--agent', '--headless', '--timeout', String(timeout)];
116
+ if (!heal) args.push('--no-adaptive-heal');
117
+ args.push(...extra);
118
+ let r;
119
+ let started;
120
+ for (let i = 0; ; i++) {
121
+ started = Date.now();
122
+ r = await runKane(args, { cwd, killAfterMs: killAfterFor(file, timeout, cwd), logFile });
123
+ if (!isTransient(r) || i >= backoff.length) break;
124
+ onRetry?.(i + 1, (r.stderr || r.stdout).trim().split('\n').pop());
125
+ await pause(backoff[i]);
126
+ }
127
+ return { ...r, ...summarizeEvents(r.events), pack: newestPack(cwd || process.cwd(), started), transient: isTransient(r) };
128
+ }
129
+
130
+ // Judge a sealed pack against a policy file; returns { verdict: pass|fail|errored|unknown, file, raw }.
131
+ export async function verdict(pack, policyFile, { cwd } = {}) {
132
+ if (!pack) return { verdict: 'unknown', reason: 'no evidence pack found' };
133
+ const r = await runKane(['evidence', 'verdict', pack, '--policy', policyFile], { cwd, killAfterMs: 60000 });
134
+ const v = r.code === 0 ? 'pass' : r.code === 1 ? 'fail' : r.code === 3 ? 'errored' : 'unknown';
135
+ const m = (r.stdout + r.stderr).match(/[^\s]+\.verdict\.json/);
136
+ let raw = null;
137
+ if (m && fs.existsSync(m[0])) { try { raw = JSON.parse(fs.readFileSync(m[0], 'utf8')); } catch { /* ignore */ } }
138
+ return { verdict: v, file: m?.[0], raw, output: (r.stdout + r.stderr).trim().split('\n').slice(-3).join('\n') };
139
+ }
140
+
141
+ // Read a numeric metric out of a verdict file, whatever its exact shape.
142
+ export function metricFromVerdict(raw, metric) {
143
+ if (!raw) return undefined;
144
+ let found;
145
+ const visit = (o) => {
146
+ if (found !== undefined || !o || typeof o !== 'object') return;
147
+ if (o.metric === metric && (typeof o.value === 'number' || typeof o.actual === 'number')) { found = o.value ?? o.actual; return; }
148
+ for (const v of Object.values(o)) visit(v);
149
+ };
150
+ visit(raw);
151
+ return found;
152
+ }
153
+
154
+ export async function balance(cwd) {
155
+ const r = await runKane(['balance', '--json'], { cwd, killAfterMs: 30000 });
156
+ const m = (r.stdout || '').match(/"?(?:balance|credits|available)"?\s*[:=]\s*([\d.]+)/i) || (r.stdout || '').match(/([\d,]+(?:\.\d+)?)\s*credits/i);
157
+ return m ? Number(m[1].replace(/,/g, '')) : undefined;
158
+ }
package/src/mutant.js CHANGED
@@ -1,26 +1,26 @@
1
- // Mutant definition validation. Used for generated, saboteur and hand-written mutants alike.
2
- const ACTIONS = {
3
- json: (a) => Array.isArray(a.ops) && a.ops.length > 0 && a.ops.every((o) => typeof o.op === 'string' && typeof o.path === 'string' && o.path.startsWith('$')),
4
- respond: (a) => Number.isInteger(a.status),
5
- 'forward-twice': () => true,
6
- 'status-map': (a) => Array.isArray(a.from) && Number.isInteger(a.to),
7
- 'stale-read': () => true,
8
- dom: (a) => a.dom && typeof a.dom.op === 'string' && a.dom.target && typeof a.dom.target.text === 'string',
9
- 'inject-css': (a) => typeof a.css === 'string',
10
- 'html-replace': (a) => typeof a.find === 'string' && typeof a.replace === 'string'
11
- };
12
- const DOM_OPS = ['hide', 'dead', 'swap-labels', 'replace-text', 'relink', 'remove-row-action'];
13
- const JSON_OPS = ['add', 'scale', 'set', 'remove-index', 'dup-index', 'swap', 'push'];
14
-
15
- export function validateMutant(m) {
16
- const errors = [];
17
- if (!m || typeof m !== 'object') return ['mutant must be an object'];
18
- if (typeof m.id !== 'string' || !m.id) errors.push('id is required');
19
- if (typeof m.describe !== 'string' || !m.describe) errors.push('describe is required (one sentence a user would notice)');
20
- if (!m.match || (!m.match.document && !m.match.path)) errors.push('match needs a path (and method) or document: true');
21
- if (!m.action || !ACTIONS[m.action.type]) errors.push(`action.type must be one of ${Object.keys(ACTIONS).join(', ')}`);
22
- else if (!ACTIONS[m.action.type](m.action)) errors.push(`action is malformed for type ${m.action.type}`);
23
- if (m.action?.type === 'json') for (const o of m.action.ops || []) if (!JSON_OPS.includes(o.op)) errors.push(`unknown json op ${o.op}`);
24
- if (m.action?.type === 'dom' && !DOM_OPS.includes(m.action.dom?.op)) errors.push(`unknown dom op ${m.action.dom?.op}`);
25
- return errors;
26
- }
1
+ // Mutant definition validation. Used for generated, saboteur and hand-written mutants alike.
2
+ const ACTIONS = {
3
+ json: (a) => Array.isArray(a.ops) && a.ops.length > 0 && a.ops.every((o) => typeof o.op === 'string' && typeof o.path === 'string' && o.path.startsWith('$')),
4
+ respond: (a) => Number.isInteger(a.status),
5
+ 'forward-twice': () => true,
6
+ 'status-map': (a) => Array.isArray(a.from) && Number.isInteger(a.to),
7
+ 'stale-read': () => true,
8
+ dom: (a) => a.dom && typeof a.dom.op === 'string' && a.dom.target && typeof a.dom.target.text === 'string',
9
+ 'inject-css': (a) => typeof a.css === 'string',
10
+ 'html-replace': (a) => typeof a.find === 'string' && typeof a.replace === 'string'
11
+ };
12
+ const DOM_OPS = ['hide', 'dead', 'swap-labels', 'replace-text', 'relink', 'remove-row-action', 'drift-number', 'hide-row', 'dup-row'];
13
+ const JSON_OPS = ['add', 'scale', 'set', 'remove-index', 'dup-index', 'swap', 'push'];
14
+
15
+ export function validateMutant(m) {
16
+ const errors = [];
17
+ if (!m || typeof m !== 'object') return ['mutant must be an object'];
18
+ if (typeof m.id !== 'string' || !m.id) errors.push('id is required');
19
+ if (typeof m.describe !== 'string' || !m.describe) errors.push('describe is required (one sentence a user would notice)');
20
+ if (!m.match || (!m.match.document && !m.match.path)) errors.push('match needs a path (and method) or document: true');
21
+ if (!m.action || !ACTIONS[m.action.type]) errors.push(`action.type must be one of ${Object.keys(ACTIONS).join(', ')}`);
22
+ else if (!ACTIONS[m.action.type](m.action)) errors.push(`action is malformed for type ${m.action.type}`);
23
+ if (m.action?.type === 'json') for (const o of m.action.ops || []) if (!JSON_OPS.includes(o.op)) errors.push(`unknown json op ${o.op}`);
24
+ if (m.action?.type === 'dom' && !DOM_OPS.includes(m.action.dom?.op)) errors.push(`unknown dom op ${m.action.dom?.op}`);
25
+ return errors;
26
+ }
package/src/orphans.js ADDED
@@ -0,0 +1,40 @@
1
+ // Kane launches a private headless Chrome (profile folder "kane-clean-*") per run. When a run is
2
+ // killed from outside (a closed terminal, a sleeping laptop) that Chrome can outlive it, keep pages
3
+ // open, and change a shared app's state under the next run. Found in practice: three such browsers,
4
+ // up to two days old, broke a green baseline.
5
+ import { execFileSync } from 'node:child_process';
6
+
7
+ const isWin = process.platform === 'win32';
8
+
9
+ // [{ pid, ageMinutes, profile }] for Kane Chrome root processes older than minAgeMinutes.
10
+ export function findOrphanKaneBrowsers(minAgeMinutes = 120) {
11
+ try {
12
+ if (isWin) {
13
+ const ps = "Get-CimInstance Win32_Process -Filter \"Name='chrome.exe'\" | Where-Object { $_.CommandLine -match 'kane-clean-' -and $_.CommandLine -match 'remote-debugging-port' } | ForEach-Object { '{0}|{1}|{2}' -f $_.ProcessId, [int]((Get-Date) - $_.CreationDate).TotalMinutes, $_.CommandLine }";
14
+ const out = execFileSync('powershell', ['-NoProfile', '-Command', ps], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 20000 });
15
+ return parse(out.split(/\r?\n/).filter(Boolean).map((l) => { const [pid, age, ...cmd] = l.split('|'); return { pid: Number(pid), ageMinutes: Number(age), cmd: cmd.join('|') }; }), minAgeMinutes);
16
+ }
17
+ const out = execFileSync('ps', ['-eo', 'pid=,etimes=,args='], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 20000 });
18
+ const rows = out.split('\n').map((l) => l.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/)).filter(Boolean)
19
+ .filter((m) => /kane-clean-/.test(m[3]) && /remote-debugging-port/.test(m[3]))
20
+ .map((m) => ({ pid: Number(m[1]), ageMinutes: Math.round(Number(m[2]) / 60), cmd: m[3] }));
21
+ return parse(rows, minAgeMinutes);
22
+ } catch {
23
+ return [];
24
+ }
25
+ }
26
+
27
+ function parse(rows, minAgeMinutes) {
28
+ return rows.filter((r) => r.ageMinutes >= minAgeMinutes).map((r) => ({ pid: r.pid, ageMinutes: r.ageMinutes, profile: (r.cmd.match(/kane-clean-[\w-]+/) || [''])[0] }));
29
+ }
30
+
31
+ // Stop a Kane Chrome and every process sharing its private profile.
32
+ export function killKaneBrowser(profile) {
33
+ if (!/^kane-clean-[\w-]+$/.test(profile)) throw new Error(`refusing to stop a non-Kane profile: ${profile}`);
34
+ if (isWin) {
35
+ const ps = `Get-CimInstance Win32_Process -Filter "Name='chrome.exe'" | Where-Object { $_.CommandLine -like '*${profile}*' } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }`;
36
+ execFileSync('powershell', ['-NoProfile', '-Command', ps], { stdio: 'ignore', timeout: 20000 });
37
+ } else {
38
+ execFileSync('pkill', ['-f', profile], { stdio: 'ignore' });
39
+ }
40
+ }
package/src/profile.js CHANGED
@@ -1,13 +1,25 @@
1
+ export const normLine = (l) => l.replace(/\s+/g, ' ').trim();
2
+
3
+ // line -> how many times it appears in one page snapshot
4
+ export function lineCounts(text) {
5
+ const m = new Map();
6
+ for (const l of String(text || '').split(/\n/)) { const x = normLine(l); if (x) m.set(x, (m.get(x) || 0) + 1); }
7
+ return m;
8
+ }
9
+
1
10
  // Turns what the proxy recorded during one baseline test run into that test's profile entry.
2
11
  export function profileEntry(test, traffic, dom) {
3
12
  const pagesByUrl = new Map();
4
13
  const clicks = [];
5
14
  for (const ev of dom) {
6
15
  if (ev.kind === 'page') {
16
+ // the most times each line appeared in any single snapshot (merged text would inflate counts)
17
+ const counts = lineCounts(ev.text);
7
18
  // keep the richest snapshot per URL (the page after it finished rendering), but merge controls
8
19
  const prev = pagesByUrl.get(ev.url);
9
- if (!prev) pagesByUrl.set(ev.url, { url: ev.url, controls: ev.controls || [], text: ev.text || '' });
20
+ if (!prev) pagesByUrl.set(ev.url, { url: ev.url, controls: ev.controls || [], text: ev.text || '', maxCounts: Object.fromEntries(counts) });
10
21
  else {
22
+ for (const [l, n] of counts) if (!(prev.maxCounts[l] >= n)) prev.maxCounts[l] = n;
11
23
  const seen = new Set(prev.controls.map((c) => `${c.role}|${c.text}|${c.count}`));
12
24
  for (const c of ev.controls || []) if (!seen.has(`${c.role}|${c.text}|${c.count}`)) prev.controls.push(c);
13
25
  if ((ev.text || '').length) prev.text = prev.text.includes(ev.text) ? prev.text : `${prev.text}\n${ev.text}`.slice(-60000);