workers-port 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/workers.js CHANGED
@@ -2,7 +2,7 @@
2
2
  import { parseArgs } from 'node:util';
3
3
  import { dirname, join } from 'node:path';
4
4
  import { fileURLToPath } from 'node:url';
5
- import { cpSync, existsSync, symlinkSync } from 'node:fs';
5
+ import { cpSync, existsSync, lstatSync, renameSync, rmSync, symlinkSync, unlinkSync } from 'node:fs';
6
6
  import { createRequire } from 'node:module';
7
7
  import { port } from '../src/port.js';
8
8
 
@@ -16,11 +16,19 @@ function prepareDemo() {
16
16
  if (!/[\\/]node_modules[\\/]/.test(src)) return src;
17
17
  const dest = join(process.cwd(), 'workers-demo');
18
18
  if (!existsSync(dest)) {
19
- cpSync(src, dest, { recursive: true });
19
+ // copy beside it, then rename: a copy cut short never leaves a half-filled workers-demo behind
20
+ const tmp = dest + '.tmp';
21
+ rmSync(tmp, { recursive: true, force: true });
22
+ cpSync(src, tmp, { recursive: true });
23
+ renameSync(tmp, dest);
20
24
  console.log(`Copied the demo app and its Playwright tests to ${dest}`);
21
25
  }
22
26
  const link = join(dest, 'node_modules');
23
27
  if (!existsSync(link)) {
28
+ try {
29
+ lstatSync(link); // a dangling junction from an earlier install: replace it
30
+ unlinkSync(link);
31
+ } catch {}
24
32
  const pwEntry = createRequire(import.meta.url).resolve('@playwright/test');
25
33
  symlinkSync(join(dirname(pwEntry), '..', '..'), link, 'junction');
26
34
  }
@@ -45,6 +53,8 @@ Options:
45
53
  --agent claude|codex coding agent that writes the tests (default: claude; falls back to
46
54
  codex if claude is not logged in and codex is)
47
55
  --agent-model <name> model name passed to the agent
56
+ --resume reuse the _test.md files already in --out (after an interrupted run);
57
+ Kane replays their recordings instead of authoring them again
48
58
  --dry-run everything except Kane: list, translate, run Playwright on every
49
59
  mutant, then print the plan and a credit estimate
50
60
  -h, --help show this help
@@ -66,6 +76,7 @@ const options = {
66
76
  agent: { type: 'string', default: 'claude' },
67
77
  'agent-model': { type: 'string' },
68
78
  'dry-run': { type: 'boolean', default: false },
79
+ resume: { type: 'boolean', default: false },
69
80
  help: { type: 'boolean', short: 'h', default: false },
70
81
  };
71
82
 
@@ -101,6 +112,7 @@ async function main() {
101
112
  agent: v.agent,
102
113
  agentModel: v['agent-model'],
103
114
  dryRun: v['dry-run'],
115
+ resume: v.resume,
104
116
  };
105
117
  for (const k of ['config', 'app', 'mutants']) {
106
118
  if (!opts[k]) {
@@ -116,4 +128,8 @@ async function main() {
116
128
  }
117
129
  }
118
130
 
119
- main();
131
+ main().catch((e) => {
132
+ console.error(`
133
+ workers: ${e.message}`);
134
+ process.exit(1);
135
+ });
package/demo/serve.js CHANGED
@@ -1,7 +1,7 @@
1
1
  // Tiny zero-dependency static file server for demo/app on port 4173.
2
2
  import { createServer } from 'node:http';
3
3
  import { readFile } from 'node:fs/promises';
4
- import { extname, join, normalize, dirname } from 'node:path';
4
+ import { extname, join, normalize, dirname, sep } from 'node:path';
5
5
  import { fileURLToPath } from 'node:url';
6
6
 
7
7
  const root = join(dirname(fileURLToPath(import.meta.url)), 'app');
@@ -15,10 +15,16 @@ const types = {
15
15
  };
16
16
 
17
17
  createServer(async (req, res) => {
18
- const urlPath = decodeURIComponent(new URL(req.url, 'http://x').pathname);
18
+ let urlPath;
19
+ try {
20
+ urlPath = decodeURIComponent(new URL(req.url, 'http://x').pathname);
21
+ } catch {
22
+ res.writeHead(400).end('Bad request');
23
+ return;
24
+ }
19
25
  const rel = normalize(urlPath === '/' ? '/index.html' : urlPath);
20
26
  const file = join(root, rel);
21
- if (!file.startsWith(root)) {
27
+ if (file !== root && !file.startsWith(root + sep)) {
22
28
  res.writeHead(403).end('Forbidden');
23
29
  return;
24
30
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "workers-port",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Port a Playwright suite to Kane CLI tests and prove the new tests catch the same bugs, by breaking your app on purpose.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/src/loop.js CHANGED
@@ -9,7 +9,7 @@ const clip = (s, n = 1500) => (s.length > n ? s.slice(0, n) + ' ...' : s);
9
9
 
10
10
  // weaknesses: [{playwrightError, kaneSteps, kaneNotes}] one per place Kane was weaker
11
11
  // brokenOnClean: null | {failedStep, kaneSteps, kaneNotes}
12
- export function buildRepairPrompt({ title, pwSource, currentMd, weaknesses = [], brokenOnClean = null, rules = loadRules() }) {
12
+ export function buildRepairPrompt({ title, pwSource, pwImports = '', currentMd, weaknesses = [], brokenOnClean = null, rules = loadRules() }) {
13
13
  let evidence = '';
14
14
  if (brokenOnClean) {
15
15
  evidence += `## The Kane test FAILED on the unchanged app, where the original Playwright test passes.
@@ -76,7 +76,7 @@ export async function repairLoop(state, mutants, { rounds = 2, agent, model, wri
76
76
  state.loop = state.loop ?? [];
77
77
  for (let round = 1; round <= rounds && needsRepair(state.cells); round++) {
78
78
  const weak = Object.keys(state.cells).filter((k) => state.cells[k] === 'kane-weaker' || state.cells[k] === 'broken-on-clean');
79
- const prompt = buildRepairPrompt({ title: state.test.title, pwSource: state.pwSource, currentMd: state.md, ...collectEvidence(state, mutants) });
79
+ const prompt = buildRepairPrompt({ title: state.test.title, pwSource: state.pwSource, pwImports: state.pwImports, currentMd: state.md, ...collectEvidence(state, mutants) });
80
80
  log(` repair round ${round}/${rounds} for "${state.test.title}" (weaker on: ${weak.join(', ')})`);
81
81
  let md;
82
82
  try {
package/src/mutants.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // <file>.workers-backup); revert() restores them. Restored on success, on errors, on SIGINT, and at
3
3
  // next startup if a previous run died.
4
4
  import { copyFileSync, existsSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
5
- import { join, resolve } from 'node:path';
5
+ import { isAbsolute, join, relative, resolve } from 'node:path';
6
6
 
7
7
  export const BACKUP_SUFFIX = '.workers-backup';
8
8
 
@@ -29,6 +29,8 @@ function revertFile(file) {
29
29
 
30
30
  export function apply(appDir, mutant) {
31
31
  const file = resolve(appDir, mutant.file);
32
+ const rel = relative(resolve(appDir), file);
33
+ if (rel === '' || rel.startsWith('..') || isAbsolute(rel)) throw new Error(`Mutant ${mutant.id}: file "${mutant.file}" is outside the app directory`);
32
34
  const backup = file + BACKUP_SUFFIX;
33
35
  if (existsSync(backup)) throw new Error(`${backup} already exists; a previous run was interrupted. Remove or restore it first.`);
34
36
  const original = readFileSync(file);
@@ -91,8 +93,10 @@ export function installSigintHandler(exit = (code) => process.exit(code)) {
91
93
  };
92
94
  process.on('SIGINT', handler);
93
95
  process.on('SIGTERM', handler);
96
+ process.on('SIGHUP', handler);
94
97
  return () => {
95
98
  process.off('SIGINT', handler);
96
99
  process.off('SIGTERM', handler);
100
+ process.off('SIGHUP', handler);
97
101
  };
98
102
  }
package/src/port.js CHANGED
@@ -1,180 +1,191 @@
1
- import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
- import { dirname, join, resolve } from 'node:path';
3
- import { adapters, resolveAgent } from './agents.js';
4
- import { restoreCache, snapshotCache, withRestoredCache } from './kane-cache.js';
5
- import { listTests } from './inventory.js';
6
- import { repairLoop } from './loop.js';
7
- import { computeCells } from './parity.js';
8
- import { doneLine, writeReport } from './report.js';
9
- import { runKane } from './run-kane.js';
10
- import { runPlaywright } from './run-playwright.js';
11
- import { ensureServer } from './server.js';
12
- import { installSigintHandler, loadMutants, restoreLeftovers, withMutant } from './mutants.js';
13
- import { countExpects, extractSource, translateTest } from './translate.js';
14
- import { slugify } from './util.js';
15
-
16
- const CREDITS_PER_AUTHOR = 40;
17
- const CREDITS_PER_HEALED_RUN = '10-30';
18
- const lines = (s) => s.trim().split('\n').length;
19
-
20
- // Plain-text matrix of the Playwright verdicts (used for the dry run).
21
- export function pwMatrixText(states, mutants) {
22
- const cols = ['clean', ...mutants.map((m) => m.id)];
23
- const w = Math.max(...states.map((s) => s.test.title.length), 4);
24
- const row = (name, cells) => name.padEnd(w) + ' ' + cells.map((c) => c.padEnd(5)).join(' ');
25
- return [row('test', cols), ...states.map((s) => row(s.test.title, cols.map((c) => (s.pw[c].passed ? 'pass' : 'FAIL'))))].join('\n');
26
- }
27
-
28
- export async function port(opts) {
29
- const log = opts.log ?? console.log;
30
- const config = resolve(opts.config);
31
- const appDir = resolve(opts.app);
32
- const outDir = resolve(opts.out ?? 'workers-out');
33
-
34
- const restored = restoreLeftovers(appDir);
35
- if (restored.length) log(`Restored ${restored.length} file(s) left mutated by an interrupted run: ${restored.join(', ')}`);
36
- const mutants = loadMutants(resolve(opts.mutants));
37
- const stopSigint = installSigintHandler();
38
-
39
- let stopServer = async () => {};
40
- try {
41
- // 1. inventory
42
- log('Listing Playwright tests ...');
43
- const inv = await listTests(config);
44
- let tests = inv.tests;
45
- if (opts.grep) {
46
- const wanted = opts.grep.toLowerCase().split('|').map((s) => s.trim()).filter(Boolean);
47
- tests = tests.filter((t) => wanted.some((w) => t.titlePath.join(' ').toLowerCase().includes(w)));
48
- }
49
- if (opts.maxTests) tests = tests.slice(0, opts.maxTests);
50
- if (!tests.length) throw new Error('No Playwright tests selected (check --grep / --config).');
51
- log(`Found ${inv.tests.length} tests, selected ${tests.length}:`);
52
- tests.forEach((t, i) => log(` ${i + 1}. ${t.titlePath.join(' > ')}`));
53
- log(`Mutants: ${mutants.map((m) => `${m.id} (${m.kind})`).join(', ')}`);
54
- const url = opts.url ?? (inv.webServer?.url ? inv.webServer.url.replace(/\/?$/, '/') : 'http://localhost:4173/');
55
-
56
- // 2. translate
57
- const agent = opts.agentImpl ?? (await resolveAgent(opts.agent ?? 'claude', log, adapters));
58
- log(`Translating with ${agent.name}${opts.agentModel ? ` (${opts.agentModel})` : ''} ...`);
59
- const states = [];
60
- for (const [i, test] of tests.entries()) {
61
- const src = readFileSync(test.file, 'utf8');
62
- const { source, testBlock } = extractSource(src, test.line);
63
- const slug = `${String(i + 1).padStart(2, '0')}-${slugify(test.title)}`;
64
- const mdPath = join(outDir, 'tests', slug, `${slug}_test.md`);
65
- log(` [${i + 1}/${tests.length}] ${test.title}`);
66
- const { md, attempts } = await translateTest({ test, url, agent, model: opts.agentModel });
67
- mkdirSync(dirname(mdPath), { recursive: true });
68
- writeFileSync(mdPath, md);
69
- log(` wrote ${mdPath}${attempts > 1 ? ' (needed a retry)' : ''}`);
70
- states.push({ test, slug, mdPath, md, pwSource: source, pwLines: lines(testBlock), mdLines: lines(md), pw: {}, kane: {}, cells: {}, loop: [], credits: 0, status: 'translated' });
71
- }
72
-
73
- // 3. Playwright matrix: one run per column
74
- log('Playwright: clean run ...');
75
- const clean = await runPlaywright(config, tests);
76
- states.forEach((s) => (s.pw.clean = clean[s.test.id]));
77
- for (const m of mutants) {
78
- log(`Playwright: mutant ${m.id} (${m.kind}) ...`);
79
- const res = await withMutant(appDir, m, () => runPlaywright(config, tests));
80
- states.forEach((s) => (s.pw[m.id] = res[s.test.id]));
81
- log(` fails ${states.filter((s) => !s.pw[m.id].passed).length} of ${states.length} selected tests`);
82
- }
83
- states.forEach((s) => (s.cells = computeCells(mutants, s.pw, null)));
84
- log('\nPlaywright matrix (old suite):\n' + pwMatrixText(states, mutants) + '\n');
85
-
86
- const eligible = states.filter((s) => s.pw.clean.passed);
87
- states.filter((s) => !s.pw.clean.passed).forEach((s) => ((s.status = 'old-test-failing'), log(` ! "${s.test.title}" fails on the clean app in Playwright; excluded.`)));
88
-
89
- const build = () => ({
90
- generatedAt: new Date().toISOString(),
91
- dryRun: !!opts.dryRun,
92
- mutants: mutants.map(({ id, kind, description }) => ({ id, kind, description })),
93
- tests: states.map((s) => ({ id: s.test.id, title: s.test.title, status: s.status, reason: s.reason, cells: s.cells, pw: s.pw, kane: s.kane, loop: s.loop, pwLines: s.pwLines, mdLines: s.mdLines, credits: s.credits, mdPath: s.mdPath })),
94
- });
95
-
96
- // dry run stops here
97
- if (opts.dryRun) {
98
- const reportPath = writeReport(outDir, build());
99
- log('Plan (dry run, Kane not executed):');
100
- log(` 1. Author ${eligible.length} test(s) on the clean app with Kane (about ${CREDITS_PER_AUTHOR} credits each)`);
101
- log(` 2. Run each against ${mutants.length} mutants with Kane's self-healing on, restoring its local cache around every run (clean replay: 0 credits; a run where Kane heals: about ${CREDITS_PER_HEALED_RUN} credits)`);
102
- log(` 3. Repair any test where Kane is weaker (up to ${opts.repairRounds ?? 2} rounds; each round re-authors, about ${CREDITS_PER_AUTHOR} credits)`);
103
- log(` Estimated Kane credits: about ${eligible.length * CREDITS_PER_AUTHOR} for authoring (plus repairs)`);
104
- log(`Report (Playwright only): ${reportPath}`);
105
- return build();
106
- }
107
-
108
- // 4. Kane
109
- stopServer = await ensureServer({ url, command: inv.webServer?.command, cwd: inv.configDir });
110
- const kane = async (state, col) => {
111
- const r = await runKane(state.mdPath);
112
- state.kane[col] = r;
113
- state.credits += r.credits;
114
- if (r.status === 'error') throw new Error(`Kane could not run (${r.errorText ?? 'exit ' + r.exitCode}). If this is an auth problem, run \`kane-cli login\`.`);
115
- return r;
116
- };
117
- const verdict = (r) => (r.status === 'passed' ? 'pass' : 'FAIL') + (r.healTriggered ? ' (healed)' : '');
118
- // Mutant run: the test's Kane cache is restored before and after, also if the run throws.
119
- const kaneMutant = (state, m) => withRestoredCache(state.mdPath, outDir, () => kane(state, m.id));
120
- // Clean authoring run: on a pass, snapshot the cache for later restores.
121
- const kaneClean = async (state) => {
122
- restoreCache(state.mdPath, outDir); // reset to the last snapshot, if any
123
- const r = await kane(state, 'clean');
124
- if (r.status === 'passed') snapshotCache(state.mdPath, outDir);
125
- return r;
126
- };
127
-
128
- for (const [i, s] of eligible.entries()) {
129
- log(`Kane: authoring on the clean app [${i + 1}/${eligible.length}] ${s.test.title}`);
130
- const r = await kaneClean(s);
131
- log(` ${verdict(r)} (${r.credits.toFixed(1)} credits)${r.failedStep ? ': ' + r.failedStep.summary : ''}`);
132
- }
133
- for (const m of mutants) {
134
- log(`Kane: running against mutant ${m.id} (self-healing on) ...`);
135
- await withMutant(appDir, m, async () => {
136
- for (const s of eligible.filter((x) => x.kane.clean.status === 'passed')) {
137
- const r = await kaneMutant(s, m);
138
- log(` ${s.test.title}: ${verdict(r)}`);
139
- }
140
- });
141
- }
142
- const recompute = (s) => (s.cells = computeCells(mutants, s.pw, s.kane));
143
- eligible.forEach(recompute);
144
-
145
- // 5. repair loop
146
- const evaluate = async (s) => {
147
- s.kane = {};
148
- await kaneClean(s); // a pass takes a fresh snapshot
149
- if (s.kane.clean.status === 'passed') {
150
- for (const m of mutants) await withMutant(appDir, m, () => kaneMutant(s, m));
151
- }
152
- recompute(s);
153
- };
154
- for (const s of eligible) {
155
- await repairLoop(s, mutants, {
156
- rounds: opts.repairRounds ?? 2,
157
- agent,
158
- model: opts.agentModel,
159
- writeMd: (st, md) => {
160
- writeFileSync(st.mdPath, md);
161
- st.mdLines = lines(md);
162
- },
163
- evaluate,
164
- log,
165
- });
166
- if (s.status === 'translated') s.status = 'ported';
167
- }
168
-
169
- const reportPath = writeReport(outDir, build());
170
- const h = JSON.parse(readFileSync(join(outDir, 'report.json'), 'utf8')).headline;
171
- log('\n' + doneLine(h));
172
- log(`Report: ${reportPath}`);
173
- return build();
174
- } finally {
175
- await stopServer();
176
- stopSigint();
177
- // belt and braces: nothing may stay mutated
178
- restoreLeftovers(appDir);
179
- }
180
- }
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { dirname, join, resolve } from 'node:path';
3
+ import { adapters, resolveAgent } from './agents.js';
4
+ import { restoreCache, snapshotCache, withRestoredCache } from './kane-cache.js';
5
+ import { listTests } from './inventory.js';
6
+ import { repairLoop } from './loop.js';
7
+ import { computeCells } from './parity.js';
8
+ import { doneLine, writeReport } from './report.js';
9
+ import { runKane } from './run-kane.js';
10
+ import { runPlaywright } from './run-playwright.js';
11
+ import { ensureServer } from './server.js';
12
+ import { installSigintHandler, loadMutants, restoreLeftovers, withMutant } from './mutants.js';
13
+ import { collectImports, countExpects, extractSource, formatImports, translateTest } from './translate.js';
14
+ import { slugify } from './util.js';
15
+
16
+ const CREDITS_PER_AUTHOR = 40;
17
+ const CREDITS_PER_HEALED_RUN = '10-30';
18
+ const lines = (s) => s.trim().split('\n').length;
19
+
20
+ // Plain-text matrix of the Playwright verdicts (used for the dry run).
21
+ export function pwMatrixText(states, mutants) {
22
+ const cols = ['clean', ...mutants.map((m) => m.id)];
23
+ const w = Math.max(...states.map((s) => s.test.title.length), 4);
24
+ const row = (name, cells) => name.padEnd(w) + ' ' + cells.map((c) => c.padEnd(5)).join(' ');
25
+ return [row('test', cols), ...states.map((s) => row(s.test.title, cols.map((c) => (s.pw[c].passed ? 'pass' : 'FAIL'))))].join('\n');
26
+ }
27
+
28
+ export async function port(opts) {
29
+ const log = opts.log ?? console.log;
30
+ const config = resolve(opts.config);
31
+ const appDir = resolve(opts.app);
32
+ const outDir = resolve(opts.out ?? 'workers-out');
33
+
34
+ const restored = restoreLeftovers(appDir);
35
+ if (restored.length) log(`Restored ${restored.length} file(s) left mutated by an interrupted run: ${restored.join(', ')}`);
36
+ const mutants = loadMutants(resolve(opts.mutants));
37
+ const stopSigint = installSigintHandler();
38
+
39
+ let stopServer = async () => {};
40
+ try {
41
+ // 1. inventory
42
+ log('Listing Playwright tests ...');
43
+ const inv = await listTests(config);
44
+ let tests = inv.tests;
45
+ if (opts.grep) {
46
+ const wanted = opts.grep.toLowerCase().split('|').map((s) => s.trim()).filter(Boolean);
47
+ tests = tests.filter((t) => wanted.some((w) => t.titlePath.join(' ').toLowerCase().includes(w)));
48
+ }
49
+ if (opts.maxTests) tests = tests.slice(0, opts.maxTests);
50
+ if (!tests.length) throw new Error('No Playwright tests selected (check --grep / --config).');
51
+ log(`Found ${inv.tests.length} tests, selected ${tests.length}:`);
52
+ tests.forEach((t, i) => log(` ${i + 1}. ${t.titlePath.join(' > ')}`));
53
+ log(`Mutants: ${mutants.map((m) => `${m.id} (${m.kind})`).join(', ')}`);
54
+ const url = opts.url ?? (inv.webServer?.url ? inv.webServer.url.replace(/\/?$/, '/') : 'http://localhost:4173/');
55
+
56
+ // 2. translate
57
+ const agent = opts.agentImpl ?? (await resolveAgent(opts.agent ?? 'claude', log, adapters));
58
+ log(`Translating with ${agent.name}${opts.agentModel ? ` (${opts.agentModel})` : ''} ...`);
59
+ const states = [];
60
+ for (const [i, test] of tests.entries()) {
61
+ const src = readFileSync(test.file, 'utf8');
62
+ const { source, testBlock } = extractSource(src, test.line);
63
+ const slug = `${String(i + 1).padStart(2, '0')}-${slugify(test.title)}`;
64
+ const mdPath = join(outDir, 'tests', slug, `${slug}_test.md`);
65
+ log(` [${i + 1}/${tests.length}] ${test.title}`);
66
+ let md;
67
+ if (opts.resume && existsSync(mdPath)) {
68
+ // --resume: keep the translation from an interrupted run, so Kane replays its recordings instead of re-authoring
69
+ md = readFileSync(mdPath, 'utf8');
70
+ log(` reused ${mdPath}`);
71
+ } else {
72
+ const r = await translateTest({ test, url, agent, model: opts.agentModel });
73
+ md = r.md;
74
+ mkdirSync(dirname(mdPath), { recursive: true });
75
+ writeFileSync(mdPath, md);
76
+ log(` wrote ${mdPath}${r.attempts > 1 ? ' (needed a retry)' : ''}`);
77
+ }
78
+ states.push({ test, slug, mdPath, md, pwSource: source, pwImports: formatImports(collectImports(test.file)), pwLines: lines(testBlock), mdLines: lines(md), pw: {}, kane: {}, cells: {}, loop: [], credits: 0, status: 'translated' });
79
+ }
80
+
81
+ // 3. Playwright matrix: one run per column
82
+ log('Playwright: clean run ...');
83
+ const clean = await runPlaywright(config, tests);
84
+ states.forEach((s) => (s.pw.clean = clean[s.test.id]));
85
+ for (const m of mutants) {
86
+ log(`Playwright: mutant ${m.id} (${m.kind}) ...`);
87
+ const res = await withMutant(appDir, m, () => runPlaywright(config, tests));
88
+ // A test that did not run is not a failure: counting it as one would fake a caught bug.
89
+ const missing = tests.filter((t) => res[t.id].didNotRun);
90
+ if (missing.length) throw new Error(`Playwright did not run ${missing.length} selected test(s) on mutant ${m.id}: ${missing.map((t) => t.title).join(', ')}`);
91
+ states.forEach((s) => (s.pw[m.id] = res[s.test.id]));
92
+ log(` fails ${states.filter((s) => !s.pw[m.id].passed).length} of ${states.length} selected tests`);
93
+ }
94
+ states.forEach((s) => (s.cells = computeCells(mutants, s.pw, null)));
95
+ log('\nPlaywright matrix (old suite):\n' + pwMatrixText(states, mutants) + '\n');
96
+
97
+ const eligible = states.filter((s) => s.pw.clean.passed);
98
+ states.filter((s) => !s.pw.clean.passed).forEach((s) => ((s.status = 'old-test-failing'), log(` ! "${s.test.title}" fails on the clean app in Playwright; excluded.`)));
99
+
100
+ const build = () => ({
101
+ generatedAt: new Date().toISOString(),
102
+ dryRun: !!opts.dryRun,
103
+ mutants: mutants.map(({ id, kind, description }) => ({ id, kind, description })),
104
+ tests: states.map((s) => ({ id: s.test.id, title: s.test.title, status: s.status, reason: s.reason, cells: s.cells, pw: s.pw, kane: s.kane, loop: s.loop, pwLines: s.pwLines, mdLines: s.mdLines, credits: s.credits, mdPath: s.mdPath })),
105
+ });
106
+
107
+ // dry run stops here
108
+ if (opts.dryRun) {
109
+ const reportPath = writeReport(outDir, build());
110
+ log('Plan (dry run, Kane not executed):');
111
+ log(` 1. Author ${eligible.length} test(s) on the clean app with Kane (about ${CREDITS_PER_AUTHOR} credits each)`);
112
+ log(` 2. Run each against ${mutants.length} mutants with Kane's self-healing on, restoring its local cache around every run (clean replay: 0 credits; a run where Kane heals: about ${CREDITS_PER_HEALED_RUN} credits)`);
113
+ log(` 3. Repair any test where Kane is weaker (up to ${opts.repairRounds ?? 2} rounds; each round re-authors, about ${CREDITS_PER_AUTHOR} credits)`);
114
+ log(` Estimated Kane credits: about ${eligible.length * CREDITS_PER_AUTHOR} for authoring (plus repairs)`);
115
+ log(`Report (Playwright only): ${reportPath}`);
116
+ return build();
117
+ }
118
+
119
+ // 4. Kane
120
+ stopServer = await ensureServer({ url, command: inv.webServer?.command, cwd: inv.configDir, env: inv.webServer?.env });
121
+ const kane = async (state, col) => {
122
+ const r = await runKane(state.mdPath);
123
+ state.kane[col] = r;
124
+ state.credits += r.credits;
125
+ if (r.status === 'error') throw new Error(`Kane could not run (${r.errorText ?? 'exit ' + r.exitCode}). If this is an auth problem, run \`kane-cli login\`.`);
126
+ return r;
127
+ };
128
+ const verdict = (r) => (r.status === 'passed' ? 'pass' : 'FAIL') + (r.healTriggered ? ' (healed)' : '');
129
+ // Mutant run: the test's Kane cache is restored before and after, also if the run throws.
130
+ const kaneMutant = (state, m) => withRestoredCache(state.mdPath, outDir, () => kane(state, m.id));
131
+ // Clean authoring run: on a pass, snapshot the cache for later restores.
132
+ const kaneClean = async (state) => {
133
+ restoreCache(state.mdPath, outDir); // reset to the last snapshot, if any
134
+ const r = await kane(state, 'clean');
135
+ if (r.status === 'passed') snapshotCache(state.mdPath, outDir);
136
+ return r;
137
+ };
138
+
139
+ for (const [i, s] of eligible.entries()) {
140
+ log(`Kane: authoring on the clean app [${i + 1}/${eligible.length}] ${s.test.title}`);
141
+ const r = await kaneClean(s);
142
+ log(` ${verdict(r)} (${r.credits.toFixed(1)} credits)${r.failedStep ? ': ' + r.failedStep.summary : ''}`);
143
+ }
144
+ for (const m of mutants) {
145
+ log(`Kane: running against mutant ${m.id} (self-healing on) ...`);
146
+ await withMutant(appDir, m, async () => {
147
+ for (const s of eligible.filter((x) => x.kane.clean.status === 'passed')) {
148
+ const r = await kaneMutant(s, m);
149
+ log(` ${s.test.title}: ${verdict(r)}`);
150
+ }
151
+ });
152
+ }
153
+ const recompute = (s) => (s.cells = computeCells(mutants, s.pw, s.kane));
154
+ eligible.forEach(recompute);
155
+
156
+ // 5. repair loop
157
+ const evaluate = async (s) => {
158
+ s.kane = {};
159
+ await kaneClean(s); // a pass takes a fresh snapshot
160
+ if (s.kane.clean.status === 'passed') {
161
+ for (const m of mutants) await withMutant(appDir, m, () => kaneMutant(s, m));
162
+ }
163
+ recompute(s);
164
+ };
165
+ for (const s of eligible) {
166
+ await repairLoop(s, mutants, {
167
+ rounds: opts.repairRounds ?? 2,
168
+ agent,
169
+ model: opts.agentModel,
170
+ writeMd: (st, md) => {
171
+ writeFileSync(st.mdPath, md);
172
+ st.mdLines = lines(md);
173
+ },
174
+ evaluate,
175
+ log,
176
+ });
177
+ if (s.status === 'translated') s.status = 'ported';
178
+ }
179
+
180
+ const reportPath = writeReport(outDir, build());
181
+ const h = JSON.parse(readFileSync(join(outDir, 'report.json'), 'utf8')).headline;
182
+ log('\n' + doneLine(h));
183
+ log(`Report: ${reportPath}`);
184
+ return build();
185
+ } finally {
186
+ await stopServer();
187
+ stopSigint();
188
+ // belt and braces: nothing may stay mutated
189
+ restoreLeftovers(appDir);
190
+ }
191
+ }
package/src/report.js CHANGED
@@ -173,7 +173,7 @@ export function renderHtml(report) {
173
173
  const pw = verdictGlyph(t.pw?.[c] ? t.pw[c].passed : undefined);
174
174
  const kane = verdictGlyph(t.kane?.[c] ? t.kane[c].status : undefined);
175
175
  const url = t.kane?.[c]?.shareUrl;
176
- const link = url && url !== 'REDACTED' ? ` <a href="${esc(url)}">run</a>` : '';
176
+ const link = typeof url === 'string' && /^https?:\/\//i.test(url) ? ` <a href="${esc(url)}">run</a>` : '';
177
177
  const healed = t.kane?.[c]?.healTriggered ? '<span class="heal">healed</span>' : '';
178
178
  return `<td class="${esc(cls)} ${tone(cls)}" data-col="${esc(c)}${mutantOf(c) ? ` (${esc(mutantOf(c).kind)})` : ' (clean app)'}"><span class="cls">${esc(label(cls))}</span><span class="pair">PW ${pw} &middot; Kane ${kane}${link}</span>${healed}</td>`;
179
179
  }).join('');
package/src/rules.md CHANGED
@@ -17,6 +17,7 @@ Kane is an AI browser agent. It reads plain-English steps, drives a real browser
17
17
  3. **Lists.** When Playwright checks a list of items (`toHaveText([...])`), write `Verify the list shows exactly these items, in this order: "a", "b", "c"`.
18
18
  4. **localStorage cannot be read by Kane.** Replace any localStorage check (`waitForFunction` on localStorage, helper functions like `checkNumberOfTodosInLocalStorage`) with visible persistence: reload the page, then verify what is still shown. If the test has no reload, add one only where a localStorage check was made, and verify the same visible items afterwards.
19
19
  5. **Inline helpers.** Helper functions (`createDefaultTodos`, `beforeEach` hooks) are written out as plain steps with their concrete values, for example the three todo titles typed one by one. Resolve constants (like `TODO_ITEMS`) to their literal strings.
20
+ Page-object methods and fixtures (shown under "Imported local files") are expanded the same way, into the concrete steps and values they perform. A `loggedInPage` fixture becomes the login steps with the literal credentials; `cartPage.applyCoupon('INK10')` becomes typing "INK10" into the coupon box and clicking its button. Assertions inside page-object methods count as `Verify` lines too.
20
21
  6. **Describe what a user does and sees**, not how Playwright finds it. No CSS selectors, no `data-testid` names, no `getByRole` syntax. Refer to things by their visible label or role in words ("the 'What needs to be done?' box", "the 'Clear completed' button", "the todo counter in the footer").
21
22
  7. **States that Playwright checks through classes or attributes** (`toHaveClass('completed')`, `toBeChecked`) should be stated as what a user can see: "is shown as completed (checked and struck through)", "the checkbox is checked".
22
23
  8. **Keep the original intent.** Do not add checks the test did not make, and do not remove any. Do not weaken exact values.
package/src/run-kane.js CHANGED
@@ -2,7 +2,7 @@ import { basename, dirname } from 'node:path';
2
2
  import { run } from './util.js';
3
3
 
4
4
  // Parse Kane --agent NDJSON (ignores non-JSON lines) into a result object.
5
- export function parseKane(text, exitCode) {
5
+ export function parseKane(text, exitCode, { timedOut = false } = {}) {
6
6
  const events = [];
7
7
  for (const line of String(text).split('\n')) {
8
8
  const s = line.trim();
@@ -19,8 +19,12 @@ export function parseKane(text, exitCode) {
19
19
  const failedRun = runEnds.find((e) => e.status === 'failed');
20
20
 
21
21
  let status;
22
- if (exitCode === 0) status = summary?.overall_status === 'failed' ? 'failed' : summary || done ? 'passed' : 'error';
23
- else if (exitCode === 1) status = 'failed';
22
+ // A verdict needs evidence in the stream. Exit 1 alone is not enough: cmd.exe also exits 1 when kane-cli is
23
+ // not found, and a killed (timed out) run can exit 1. Those are errors, never a pass or a fail.
24
+ const failEvidence = !!(failedStepEnd || failedRun || summary?.overall_status === 'failed' || done?.overall_status === 'failed');
25
+ if (timedOut || summary?.cancelled === true) status = 'error';
26
+ else if (exitCode === 0) status = summary?.overall_status === 'failed' || done?.overall_status === 'failed' ? 'failed' : summary || done ? 'passed' : 'error';
27
+ else if (exitCode === 1) status = failEvidence ? 'failed' : 'error';
24
28
  else status = 'error';
25
29
 
26
30
  let failedStep = null;
@@ -48,9 +52,9 @@ export function parseKane(text, exitCode) {
48
52
  export async function runKane(file, { timeoutMs = 900000 } = {}) {
49
53
  const args = ['testmd', 'run', basename(file), '--agent', '--headless'];
50
54
  const r = await run('kane-cli', args, { cwd: dirname(file), env: { KANE_CLI_USER_AGENT: 'workers' }, timeoutMs });
51
- const result = parseKane(r.stdout, r.code);
55
+ const result = parseKane(r.stdout, r.code, { timedOut: r.timedOut });
52
56
  if (result.status === 'error' && !result.failedStep) {
53
- result.errorText = (r.stderr || '').trim().split('\n').slice(-2).join(' ') || `kane-cli exited with code ${r.code}`;
57
+ result.errorText = (r.stderr || '').trim().split('\n').slice(-2).join(' ') || `kane-cli exited with code ${r.code}${r.timedOut ? ' (timed out)' : ''}`;
54
58
  }
55
59
  return result;
56
60
  }
@@ -21,15 +21,19 @@ export function parseResults(json) {
21
21
  return map;
22
22
  }
23
23
 
24
+ // Regex for a test title that is safe inside a shell command: letters, digits, space, _ - and non-ASCII are kept,
25
+ // every other character (all shell and regex metacharacters) becomes '.', which still matches it. Matching a few
26
+ // extra tests is harmless: results are mapped by test id.
27
+ export const grepPattern = (title) => String(title).replace(/[\x00-\x1f\x21-\x2c\x2e-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]/g, '.');
28
+
24
29
  // Run the selected tests once (one Playwright process). Results are mapped by id.
25
30
  export async function runPlaywright(config, tests, { all = false } = {}) {
26
31
  const args = [];
27
32
  if (!all) {
28
- const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
29
- args.push('--grep', `(${tests.map((t) => esc(t.title)).join('|')})`);
33
+ args.push('--grep', `(${tests.map((t) => grepPattern(t.title)).join('|')})`);
30
34
  }
31
35
  const map = parseResults(await playwrightJson(args, config));
32
36
  const out = {};
33
- for (const t of tests) out[t.id] = map.get(t.id) ?? { passed: false, errorMessage: 'test did not run' };
37
+ for (const t of tests) out[t.id] = map.get(t.id) ?? { passed: false, errorMessage: 'test did not run', didNotRun: true };
34
38
  return out;
35
39
  }