workers-port 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +308 -236
- package/bin/workers.js +19 -3
- package/demo/serve.js +9 -3
- package/package.json +1 -1
- package/src/loop.js +2 -2
- package/src/mutants.js +5 -1
- package/src/port.js +191 -180
- package/src/report.js +1 -1
- package/src/rules.md +1 -0
- package/src/run-kane.js +9 -5
- package/src/run-playwright.js +7 -3
- package/src/server.js +24 -23
- package/src/translate.js +83 -5
- package/src/util.js +9 -5
package/bin/workers.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { cpSync, existsSync, symlinkSync } from 'node:fs';
|
|
5
|
+
import { cpSync, existsSync, lstatSync, renameSync, rmSync, symlinkSync, unlinkSync } from 'node:fs';
|
|
6
6
|
import { createRequire } from 'node:module';
|
|
7
7
|
import { port } from '../src/port.js';
|
|
8
8
|
|
|
@@ -16,11 +16,19 @@ function prepareDemo() {
|
|
|
16
16
|
if (!/[\\/]node_modules[\\/]/.test(src)) return src;
|
|
17
17
|
const dest = join(process.cwd(), 'workers-demo');
|
|
18
18
|
if (!existsSync(dest)) {
|
|
19
|
-
|
|
19
|
+
// copy beside it, then rename: a copy cut short never leaves a half-filled workers-demo behind
|
|
20
|
+
const tmp = dest + '.tmp';
|
|
21
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
22
|
+
cpSync(src, tmp, { recursive: true });
|
|
23
|
+
renameSync(tmp, dest);
|
|
20
24
|
console.log(`Copied the demo app and its Playwright tests to ${dest}`);
|
|
21
25
|
}
|
|
22
26
|
const link = join(dest, 'node_modules');
|
|
23
27
|
if (!existsSync(link)) {
|
|
28
|
+
try {
|
|
29
|
+
lstatSync(link); // a dangling junction from an earlier install: replace it
|
|
30
|
+
unlinkSync(link);
|
|
31
|
+
} catch {}
|
|
24
32
|
const pwEntry = createRequire(import.meta.url).resolve('@playwright/test');
|
|
25
33
|
symlinkSync(join(dirname(pwEntry), '..', '..'), link, 'junction');
|
|
26
34
|
}
|
|
@@ -45,6 +53,8 @@ Options:
|
|
|
45
53
|
--agent claude|codex coding agent that writes the tests (default: claude; falls back to
|
|
46
54
|
codex if claude is not logged in and codex is)
|
|
47
55
|
--agent-model <name> model name passed to the agent
|
|
56
|
+
--resume reuse the _test.md files already in --out (after an interrupted run);
|
|
57
|
+
Kane replays their recordings instead of authoring them again
|
|
48
58
|
--dry-run everything except Kane: list, translate, run Playwright on every
|
|
49
59
|
mutant, then print the plan and a credit estimate
|
|
50
60
|
-h, --help show this help
|
|
@@ -66,6 +76,7 @@ const options = {
|
|
|
66
76
|
agent: { type: 'string', default: 'claude' },
|
|
67
77
|
'agent-model': { type: 'string' },
|
|
68
78
|
'dry-run': { type: 'boolean', default: false },
|
|
79
|
+
resume: { type: 'boolean', default: false },
|
|
69
80
|
help: { type: 'boolean', short: 'h', default: false },
|
|
70
81
|
};
|
|
71
82
|
|
|
@@ -101,6 +112,7 @@ async function main() {
|
|
|
101
112
|
agent: v.agent,
|
|
102
113
|
agentModel: v['agent-model'],
|
|
103
114
|
dryRun: v['dry-run'],
|
|
115
|
+
resume: v.resume,
|
|
104
116
|
};
|
|
105
117
|
for (const k of ['config', 'app', 'mutants']) {
|
|
106
118
|
if (!opts[k]) {
|
|
@@ -116,4 +128,8 @@ async function main() {
|
|
|
116
128
|
}
|
|
117
129
|
}
|
|
118
130
|
|
|
119
|
-
main()
|
|
131
|
+
main().catch((e) => {
|
|
132
|
+
console.error(`
|
|
133
|
+
workers: ${e.message}`);
|
|
134
|
+
process.exit(1);
|
|
135
|
+
});
|
package/demo/serve.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Tiny zero-dependency static file server for demo/app on port 4173.
|
|
2
2
|
import { createServer } from 'node:http';
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
|
-
import { extname, join, normalize, dirname } from 'node:path';
|
|
4
|
+
import { extname, join, normalize, dirname, sep } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
|
|
7
7
|
const root = join(dirname(fileURLToPath(import.meta.url)), 'app');
|
|
@@ -15,10 +15,16 @@ const types = {
|
|
|
15
15
|
};
|
|
16
16
|
|
|
17
17
|
createServer(async (req, res) => {
|
|
18
|
-
|
|
18
|
+
let urlPath;
|
|
19
|
+
try {
|
|
20
|
+
urlPath = decodeURIComponent(new URL(req.url, 'http://x').pathname);
|
|
21
|
+
} catch {
|
|
22
|
+
res.writeHead(400).end('Bad request');
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
19
25
|
const rel = normalize(urlPath === '/' ? '/index.html' : urlPath);
|
|
20
26
|
const file = join(root, rel);
|
|
21
|
-
if (!file.startsWith(root)) {
|
|
27
|
+
if (file !== root && !file.startsWith(root + sep)) {
|
|
22
28
|
res.writeHead(403).end('Forbidden');
|
|
23
29
|
return;
|
|
24
30
|
}
|
package/package.json
CHANGED
package/src/loop.js
CHANGED
|
@@ -9,7 +9,7 @@ const clip = (s, n = 1500) => (s.length > n ? s.slice(0, n) + ' ...' : s);
|
|
|
9
9
|
|
|
10
10
|
// weaknesses: [{playwrightError, kaneSteps, kaneNotes}] one per place Kane was weaker
|
|
11
11
|
// brokenOnClean: null | {failedStep, kaneSteps, kaneNotes}
|
|
12
|
-
export function buildRepairPrompt({ title, pwSource, currentMd, weaknesses = [], brokenOnClean = null, rules = loadRules() }) {
|
|
12
|
+
export function buildRepairPrompt({ title, pwSource, pwImports = '', currentMd, weaknesses = [], brokenOnClean = null, rules = loadRules() }) {
|
|
13
13
|
let evidence = '';
|
|
14
14
|
if (brokenOnClean) {
|
|
15
15
|
evidence += `## The Kane test FAILED on the unchanged app, where the original Playwright test passes.
|
|
@@ -76,7 +76,7 @@ export async function repairLoop(state, mutants, { rounds = 2, agent, model, wri
|
|
|
76
76
|
state.loop = state.loop ?? [];
|
|
77
77
|
for (let round = 1; round <= rounds && needsRepair(state.cells); round++) {
|
|
78
78
|
const weak = Object.keys(state.cells).filter((k) => state.cells[k] === 'kane-weaker' || state.cells[k] === 'broken-on-clean');
|
|
79
|
-
const prompt = buildRepairPrompt({ title: state.test.title, pwSource: state.pwSource, currentMd: state.md, ...collectEvidence(state, mutants) });
|
|
79
|
+
const prompt = buildRepairPrompt({ title: state.test.title, pwSource: state.pwSource, pwImports: state.pwImports, currentMd: state.md, ...collectEvidence(state, mutants) });
|
|
80
80
|
log(` repair round ${round}/${rounds} for "${state.test.title}" (weaker on: ${weak.join(', ')})`);
|
|
81
81
|
let md;
|
|
82
82
|
try {
|
package/src/mutants.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// <file>.workers-backup); revert() restores them. Restored on success, on errors, on SIGINT, and at
|
|
3
3
|
// next startup if a previous run died.
|
|
4
4
|
import { copyFileSync, existsSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
5
|
-
import { join, resolve } from 'node:path';
|
|
5
|
+
import { isAbsolute, join, relative, resolve } from 'node:path';
|
|
6
6
|
|
|
7
7
|
export const BACKUP_SUFFIX = '.workers-backup';
|
|
8
8
|
|
|
@@ -29,6 +29,8 @@ function revertFile(file) {
|
|
|
29
29
|
|
|
30
30
|
export function apply(appDir, mutant) {
|
|
31
31
|
const file = resolve(appDir, mutant.file);
|
|
32
|
+
const rel = relative(resolve(appDir), file);
|
|
33
|
+
if (rel === '' || rel.startsWith('..') || isAbsolute(rel)) throw new Error(`Mutant ${mutant.id}: file "${mutant.file}" is outside the app directory`);
|
|
32
34
|
const backup = file + BACKUP_SUFFIX;
|
|
33
35
|
if (existsSync(backup)) throw new Error(`${backup} already exists; a previous run was interrupted. Remove or restore it first.`);
|
|
34
36
|
const original = readFileSync(file);
|
|
@@ -91,8 +93,10 @@ export function installSigintHandler(exit = (code) => process.exit(code)) {
|
|
|
91
93
|
};
|
|
92
94
|
process.on('SIGINT', handler);
|
|
93
95
|
process.on('SIGTERM', handler);
|
|
96
|
+
process.on('SIGHUP', handler);
|
|
94
97
|
return () => {
|
|
95
98
|
process.off('SIGINT', handler);
|
|
96
99
|
process.off('SIGTERM', handler);
|
|
100
|
+
process.off('SIGHUP', handler);
|
|
97
101
|
};
|
|
98
102
|
}
|
package/src/port.js
CHANGED
|
@@ -1,180 +1,191 @@
|
|
|
1
|
-
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { dirname, join, resolve } from 'node:path';
|
|
3
|
-
import { adapters, resolveAgent } from './agents.js';
|
|
4
|
-
import { restoreCache, snapshotCache, withRestoredCache } from './kane-cache.js';
|
|
5
|
-
import { listTests } from './inventory.js';
|
|
6
|
-
import { repairLoop } from './loop.js';
|
|
7
|
-
import { computeCells } from './parity.js';
|
|
8
|
-
import { doneLine, writeReport } from './report.js';
|
|
9
|
-
import { runKane } from './run-kane.js';
|
|
10
|
-
import { runPlaywright } from './run-playwright.js';
|
|
11
|
-
import { ensureServer } from './server.js';
|
|
12
|
-
import { installSigintHandler, loadMutants, restoreLeftovers, withMutant } from './mutants.js';
|
|
13
|
-
import { countExpects, extractSource, translateTest } from './translate.js';
|
|
14
|
-
import { slugify } from './util.js';
|
|
15
|
-
|
|
16
|
-
const CREDITS_PER_AUTHOR = 40;
|
|
17
|
-
const CREDITS_PER_HEALED_RUN = '10-30';
|
|
18
|
-
const lines = (s) => s.trim().split('\n').length;
|
|
19
|
-
|
|
20
|
-
// Plain-text matrix of the Playwright verdicts (used for the dry run).
|
|
21
|
-
export function pwMatrixText(states, mutants) {
|
|
22
|
-
const cols = ['clean', ...mutants.map((m) => m.id)];
|
|
23
|
-
const w = Math.max(...states.map((s) => s.test.title.length), 4);
|
|
24
|
-
const row = (name, cells) => name.padEnd(w) + ' ' + cells.map((c) => c.padEnd(5)).join(' ');
|
|
25
|
-
return [row('test', cols), ...states.map((s) => row(s.test.title, cols.map((c) => (s.pw[c].passed ? 'pass' : 'FAIL'))))].join('\n');
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export async function port(opts) {
|
|
29
|
-
const log = opts.log ?? console.log;
|
|
30
|
-
const config = resolve(opts.config);
|
|
31
|
-
const appDir = resolve(opts.app);
|
|
32
|
-
const outDir = resolve(opts.out ?? 'workers-out');
|
|
33
|
-
|
|
34
|
-
const restored = restoreLeftovers(appDir);
|
|
35
|
-
if (restored.length) log(`Restored ${restored.length} file(s) left mutated by an interrupted run: ${restored.join(', ')}`);
|
|
36
|
-
const mutants = loadMutants(resolve(opts.mutants));
|
|
37
|
-
const stopSigint = installSigintHandler();
|
|
38
|
-
|
|
39
|
-
let stopServer = async () => {};
|
|
40
|
-
try {
|
|
41
|
-
// 1. inventory
|
|
42
|
-
log('Listing Playwright tests ...');
|
|
43
|
-
const inv = await listTests(config);
|
|
44
|
-
let tests = inv.tests;
|
|
45
|
-
if (opts.grep) {
|
|
46
|
-
const wanted = opts.grep.toLowerCase().split('|').map((s) => s.trim()).filter(Boolean);
|
|
47
|
-
tests = tests.filter((t) => wanted.some((w) => t.titlePath.join(' ').toLowerCase().includes(w)));
|
|
48
|
-
}
|
|
49
|
-
if (opts.maxTests) tests = tests.slice(0, opts.maxTests);
|
|
50
|
-
if (!tests.length) throw new Error('No Playwright tests selected (check --grep / --config).');
|
|
51
|
-
log(`Found ${inv.tests.length} tests, selected ${tests.length}:`);
|
|
52
|
-
tests.forEach((t, i) => log(` ${i + 1}. ${t.titlePath.join(' > ')}`));
|
|
53
|
-
log(`Mutants: ${mutants.map((m) => `${m.id} (${m.kind})`).join(', ')}`);
|
|
54
|
-
const url = opts.url ?? (inv.webServer?.url ? inv.webServer.url.replace(/\/?$/, '/') : 'http://localhost:4173/');
|
|
55
|
-
|
|
56
|
-
// 2. translate
|
|
57
|
-
const agent = opts.agentImpl ?? (await resolveAgent(opts.agent ?? 'claude', log, adapters));
|
|
58
|
-
log(`Translating with ${agent.name}${opts.agentModel ? ` (${opts.agentModel})` : ''} ...`);
|
|
59
|
-
const states = [];
|
|
60
|
-
for (const [i, test] of tests.entries()) {
|
|
61
|
-
const src = readFileSync(test.file, 'utf8');
|
|
62
|
-
const { source, testBlock } = extractSource(src, test.line);
|
|
63
|
-
const slug = `${String(i + 1).padStart(2, '0')}-${slugify(test.title)}`;
|
|
64
|
-
const mdPath = join(outDir, 'tests', slug, `${slug}_test.md`);
|
|
65
|
-
log(` [${i + 1}/${tests.length}] ${test.title}`);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
});
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
|
+
import { adapters, resolveAgent } from './agents.js';
|
|
4
|
+
import { restoreCache, snapshotCache, withRestoredCache } from './kane-cache.js';
|
|
5
|
+
import { listTests } from './inventory.js';
|
|
6
|
+
import { repairLoop } from './loop.js';
|
|
7
|
+
import { computeCells } from './parity.js';
|
|
8
|
+
import { doneLine, writeReport } from './report.js';
|
|
9
|
+
import { runKane } from './run-kane.js';
|
|
10
|
+
import { runPlaywright } from './run-playwright.js';
|
|
11
|
+
import { ensureServer } from './server.js';
|
|
12
|
+
import { installSigintHandler, loadMutants, restoreLeftovers, withMutant } from './mutants.js';
|
|
13
|
+
import { collectImports, countExpects, extractSource, formatImports, translateTest } from './translate.js';
|
|
14
|
+
import { slugify } from './util.js';
|
|
15
|
+
|
|
16
|
+
const CREDITS_PER_AUTHOR = 40;
|
|
17
|
+
const CREDITS_PER_HEALED_RUN = '10-30';
|
|
18
|
+
const lines = (s) => s.trim().split('\n').length;
|
|
19
|
+
|
|
20
|
+
// Plain-text matrix of the Playwright verdicts (used for the dry run).
|
|
21
|
+
export function pwMatrixText(states, mutants) {
|
|
22
|
+
const cols = ['clean', ...mutants.map((m) => m.id)];
|
|
23
|
+
const w = Math.max(...states.map((s) => s.test.title.length), 4);
|
|
24
|
+
const row = (name, cells) => name.padEnd(w) + ' ' + cells.map((c) => c.padEnd(5)).join(' ');
|
|
25
|
+
return [row('test', cols), ...states.map((s) => row(s.test.title, cols.map((c) => (s.pw[c].passed ? 'pass' : 'FAIL'))))].join('\n');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function port(opts) {
|
|
29
|
+
const log = opts.log ?? console.log;
|
|
30
|
+
const config = resolve(opts.config);
|
|
31
|
+
const appDir = resolve(opts.app);
|
|
32
|
+
const outDir = resolve(opts.out ?? 'workers-out');
|
|
33
|
+
|
|
34
|
+
const restored = restoreLeftovers(appDir);
|
|
35
|
+
if (restored.length) log(`Restored ${restored.length} file(s) left mutated by an interrupted run: ${restored.join(', ')}`);
|
|
36
|
+
const mutants = loadMutants(resolve(opts.mutants));
|
|
37
|
+
const stopSigint = installSigintHandler();
|
|
38
|
+
|
|
39
|
+
let stopServer = async () => {};
|
|
40
|
+
try {
|
|
41
|
+
// 1. inventory
|
|
42
|
+
log('Listing Playwright tests ...');
|
|
43
|
+
const inv = await listTests(config);
|
|
44
|
+
let tests = inv.tests;
|
|
45
|
+
if (opts.grep) {
|
|
46
|
+
const wanted = opts.grep.toLowerCase().split('|').map((s) => s.trim()).filter(Boolean);
|
|
47
|
+
tests = tests.filter((t) => wanted.some((w) => t.titlePath.join(' ').toLowerCase().includes(w)));
|
|
48
|
+
}
|
|
49
|
+
if (opts.maxTests) tests = tests.slice(0, opts.maxTests);
|
|
50
|
+
if (!tests.length) throw new Error('No Playwright tests selected (check --grep / --config).');
|
|
51
|
+
log(`Found ${inv.tests.length} tests, selected ${tests.length}:`);
|
|
52
|
+
tests.forEach((t, i) => log(` ${i + 1}. ${t.titlePath.join(' > ')}`));
|
|
53
|
+
log(`Mutants: ${mutants.map((m) => `${m.id} (${m.kind})`).join(', ')}`);
|
|
54
|
+
const url = opts.url ?? (inv.webServer?.url ? inv.webServer.url.replace(/\/?$/, '/') : 'http://localhost:4173/');
|
|
55
|
+
|
|
56
|
+
// 2. translate
|
|
57
|
+
const agent = opts.agentImpl ?? (await resolveAgent(opts.agent ?? 'claude', log, adapters));
|
|
58
|
+
log(`Translating with ${agent.name}${opts.agentModel ? ` (${opts.agentModel})` : ''} ...`);
|
|
59
|
+
const states = [];
|
|
60
|
+
for (const [i, test] of tests.entries()) {
|
|
61
|
+
const src = readFileSync(test.file, 'utf8');
|
|
62
|
+
const { source, testBlock } = extractSource(src, test.line);
|
|
63
|
+
const slug = `${String(i + 1).padStart(2, '0')}-${slugify(test.title)}`;
|
|
64
|
+
const mdPath = join(outDir, 'tests', slug, `${slug}_test.md`);
|
|
65
|
+
log(` [${i + 1}/${tests.length}] ${test.title}`);
|
|
66
|
+
let md;
|
|
67
|
+
if (opts.resume && existsSync(mdPath)) {
|
|
68
|
+
// --resume: keep the translation from an interrupted run, so Kane replays its recordings instead of re-authoring
|
|
69
|
+
md = readFileSync(mdPath, 'utf8');
|
|
70
|
+
log(` reused ${mdPath}`);
|
|
71
|
+
} else {
|
|
72
|
+
const r = await translateTest({ test, url, agent, model: opts.agentModel });
|
|
73
|
+
md = r.md;
|
|
74
|
+
mkdirSync(dirname(mdPath), { recursive: true });
|
|
75
|
+
writeFileSync(mdPath, md);
|
|
76
|
+
log(` wrote ${mdPath}${r.attempts > 1 ? ' (needed a retry)' : ''}`);
|
|
77
|
+
}
|
|
78
|
+
states.push({ test, slug, mdPath, md, pwSource: source, pwImports: formatImports(collectImports(test.file)), pwLines: lines(testBlock), mdLines: lines(md), pw: {}, kane: {}, cells: {}, loop: [], credits: 0, status: 'translated' });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// 3. Playwright matrix: one run per column
|
|
82
|
+
log('Playwright: clean run ...');
|
|
83
|
+
const clean = await runPlaywright(config, tests);
|
|
84
|
+
states.forEach((s) => (s.pw.clean = clean[s.test.id]));
|
|
85
|
+
for (const m of mutants) {
|
|
86
|
+
log(`Playwright: mutant ${m.id} (${m.kind}) ...`);
|
|
87
|
+
const res = await withMutant(appDir, m, () => runPlaywright(config, tests));
|
|
88
|
+
// A test that did not run is not a failure: counting it as one would fake a caught bug.
|
|
89
|
+
const missing = tests.filter((t) => res[t.id].didNotRun);
|
|
90
|
+
if (missing.length) throw new Error(`Playwright did not run ${missing.length} selected test(s) on mutant ${m.id}: ${missing.map((t) => t.title).join(', ')}`);
|
|
91
|
+
states.forEach((s) => (s.pw[m.id] = res[s.test.id]));
|
|
92
|
+
log(` fails ${states.filter((s) => !s.pw[m.id].passed).length} of ${states.length} selected tests`);
|
|
93
|
+
}
|
|
94
|
+
states.forEach((s) => (s.cells = computeCells(mutants, s.pw, null)));
|
|
95
|
+
log('\nPlaywright matrix (old suite):\n' + pwMatrixText(states, mutants) + '\n');
|
|
96
|
+
|
|
97
|
+
const eligible = states.filter((s) => s.pw.clean.passed);
|
|
98
|
+
states.filter((s) => !s.pw.clean.passed).forEach((s) => ((s.status = 'old-test-failing'), log(` ! "${s.test.title}" fails on the clean app in Playwright; excluded.`)));
|
|
99
|
+
|
|
100
|
+
const build = () => ({
|
|
101
|
+
generatedAt: new Date().toISOString(),
|
|
102
|
+
dryRun: !!opts.dryRun,
|
|
103
|
+
mutants: mutants.map(({ id, kind, description }) => ({ id, kind, description })),
|
|
104
|
+
tests: states.map((s) => ({ id: s.test.id, title: s.test.title, status: s.status, reason: s.reason, cells: s.cells, pw: s.pw, kane: s.kane, loop: s.loop, pwLines: s.pwLines, mdLines: s.mdLines, credits: s.credits, mdPath: s.mdPath })),
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// dry run stops here
|
|
108
|
+
if (opts.dryRun) {
|
|
109
|
+
const reportPath = writeReport(outDir, build());
|
|
110
|
+
log('Plan (dry run, Kane not executed):');
|
|
111
|
+
log(` 1. Author ${eligible.length} test(s) on the clean app with Kane (about ${CREDITS_PER_AUTHOR} credits each)`);
|
|
112
|
+
log(` 2. Run each against ${mutants.length} mutants with Kane's self-healing on, restoring its local cache around every run (clean replay: 0 credits; a run where Kane heals: about ${CREDITS_PER_HEALED_RUN} credits)`);
|
|
113
|
+
log(` 3. Repair any test where Kane is weaker (up to ${opts.repairRounds ?? 2} rounds; each round re-authors, about ${CREDITS_PER_AUTHOR} credits)`);
|
|
114
|
+
log(` Estimated Kane credits: about ${eligible.length * CREDITS_PER_AUTHOR} for authoring (plus repairs)`);
|
|
115
|
+
log(`Report (Playwright only): ${reportPath}`);
|
|
116
|
+
return build();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// 4. Kane
|
|
120
|
+
stopServer = await ensureServer({ url, command: inv.webServer?.command, cwd: inv.configDir, env: inv.webServer?.env });
|
|
121
|
+
const kane = async (state, col) => {
|
|
122
|
+
const r = await runKane(state.mdPath);
|
|
123
|
+
state.kane[col] = r;
|
|
124
|
+
state.credits += r.credits;
|
|
125
|
+
if (r.status === 'error') throw new Error(`Kane could not run (${r.errorText ?? 'exit ' + r.exitCode}). If this is an auth problem, run \`kane-cli login\`.`);
|
|
126
|
+
return r;
|
|
127
|
+
};
|
|
128
|
+
const verdict = (r) => (r.status === 'passed' ? 'pass' : 'FAIL') + (r.healTriggered ? ' (healed)' : '');
|
|
129
|
+
// Mutant run: the test's Kane cache is restored before and after, also if the run throws.
|
|
130
|
+
const kaneMutant = (state, m) => withRestoredCache(state.mdPath, outDir, () => kane(state, m.id));
|
|
131
|
+
// Clean authoring run: on a pass, snapshot the cache for later restores.
|
|
132
|
+
const kaneClean = async (state) => {
|
|
133
|
+
restoreCache(state.mdPath, outDir); // reset to the last snapshot, if any
|
|
134
|
+
const r = await kane(state, 'clean');
|
|
135
|
+
if (r.status === 'passed') snapshotCache(state.mdPath, outDir);
|
|
136
|
+
return r;
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
for (const [i, s] of eligible.entries()) {
|
|
140
|
+
log(`Kane: authoring on the clean app [${i + 1}/${eligible.length}] ${s.test.title}`);
|
|
141
|
+
const r = await kaneClean(s);
|
|
142
|
+
log(` ${verdict(r)} (${r.credits.toFixed(1)} credits)${r.failedStep ? ': ' + r.failedStep.summary : ''}`);
|
|
143
|
+
}
|
|
144
|
+
for (const m of mutants) {
|
|
145
|
+
log(`Kane: running against mutant ${m.id} (self-healing on) ...`);
|
|
146
|
+
await withMutant(appDir, m, async () => {
|
|
147
|
+
for (const s of eligible.filter((x) => x.kane.clean.status === 'passed')) {
|
|
148
|
+
const r = await kaneMutant(s, m);
|
|
149
|
+
log(` ${s.test.title}: ${verdict(r)}`);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
const recompute = (s) => (s.cells = computeCells(mutants, s.pw, s.kane));
|
|
154
|
+
eligible.forEach(recompute);
|
|
155
|
+
|
|
156
|
+
// 5. repair loop
|
|
157
|
+
const evaluate = async (s) => {
|
|
158
|
+
s.kane = {};
|
|
159
|
+
await kaneClean(s); // a pass takes a fresh snapshot
|
|
160
|
+
if (s.kane.clean.status === 'passed') {
|
|
161
|
+
for (const m of mutants) await withMutant(appDir, m, () => kaneMutant(s, m));
|
|
162
|
+
}
|
|
163
|
+
recompute(s);
|
|
164
|
+
};
|
|
165
|
+
for (const s of eligible) {
|
|
166
|
+
await repairLoop(s, mutants, {
|
|
167
|
+
rounds: opts.repairRounds ?? 2,
|
|
168
|
+
agent,
|
|
169
|
+
model: opts.agentModel,
|
|
170
|
+
writeMd: (st, md) => {
|
|
171
|
+
writeFileSync(st.mdPath, md);
|
|
172
|
+
st.mdLines = lines(md);
|
|
173
|
+
},
|
|
174
|
+
evaluate,
|
|
175
|
+
log,
|
|
176
|
+
});
|
|
177
|
+
if (s.status === 'translated') s.status = 'ported';
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const reportPath = writeReport(outDir, build());
|
|
181
|
+
const h = JSON.parse(readFileSync(join(outDir, 'report.json'), 'utf8')).headline;
|
|
182
|
+
log('\n' + doneLine(h));
|
|
183
|
+
log(`Report: ${reportPath}`);
|
|
184
|
+
return build();
|
|
185
|
+
} finally {
|
|
186
|
+
await stopServer();
|
|
187
|
+
stopSigint();
|
|
188
|
+
// belt and braces: nothing may stay mutated
|
|
189
|
+
restoreLeftovers(appDir);
|
|
190
|
+
}
|
|
191
|
+
}
|
package/src/report.js
CHANGED
|
@@ -173,7 +173,7 @@ export function renderHtml(report) {
|
|
|
173
173
|
const pw = verdictGlyph(t.pw?.[c] ? t.pw[c].passed : undefined);
|
|
174
174
|
const kane = verdictGlyph(t.kane?.[c] ? t.kane[c].status : undefined);
|
|
175
175
|
const url = t.kane?.[c]?.shareUrl;
|
|
176
|
-
const link =
|
|
176
|
+
const link = typeof url === 'string' && /^https?:\/\//i.test(url) ? ` <a href="${esc(url)}">run</a>` : '';
|
|
177
177
|
const healed = t.kane?.[c]?.healTriggered ? '<span class="heal">healed</span>' : '';
|
|
178
178
|
return `<td class="${esc(cls)} ${tone(cls)}" data-col="${esc(c)}${mutantOf(c) ? ` (${esc(mutantOf(c).kind)})` : ' (clean app)'}"><span class="cls">${esc(label(cls))}</span><span class="pair">PW ${pw} · Kane ${kane}${link}</span>${healed}</td>`;
|
|
179
179
|
}).join('');
|
package/src/rules.md
CHANGED
|
@@ -17,6 +17,7 @@ Kane is an AI browser agent. It reads plain-English steps, drives a real browser
|
|
|
17
17
|
3. **Lists.** When Playwright checks a list of items (`toHaveText([...])`), write `Verify the list shows exactly these items, in this order: "a", "b", "c"`.
|
|
18
18
|
4. **localStorage cannot be read by Kane.** Replace any localStorage check (`waitForFunction` on localStorage, helper functions like `checkNumberOfTodosInLocalStorage`) with visible persistence: reload the page, then verify what is still shown. If the test has no reload, add one only where a localStorage check was made, and verify the same visible items afterwards.
|
|
19
19
|
5. **Inline helpers.** Helper functions (`createDefaultTodos`, `beforeEach` hooks) are written out as plain steps with their concrete values, for example the three todo titles typed one by one. Resolve constants (like `TODO_ITEMS`) to their literal strings.
|
|
20
|
+
Page-object methods and fixtures (shown under "Imported local files") are expanded the same way, into the concrete steps and values they perform. A `loggedInPage` fixture becomes the login steps with the literal credentials; `cartPage.applyCoupon('INK10')` becomes typing "INK10" into the coupon box and clicking its button. Assertions inside page-object methods count as `Verify` lines too.
|
|
20
21
|
6. **Describe what a user does and sees**, not how Playwright finds it. No CSS selectors, no `data-testid` names, no `getByRole` syntax. Refer to things by their visible label or role in words ("the 'What needs to be done?' box", "the 'Clear completed' button", "the todo counter in the footer").
|
|
21
22
|
7. **States that Playwright checks through classes or attributes** (`toHaveClass('completed')`, `toBeChecked`) should be stated as what a user can see: "is shown as completed (checked and struck through)", "the checkbox is checked".
|
|
22
23
|
8. **Keep the original intent.** Do not add checks the test did not make, and do not remove any. Do not weaken exact values.
|
package/src/run-kane.js
CHANGED
|
@@ -2,7 +2,7 @@ import { basename, dirname } from 'node:path';
|
|
|
2
2
|
import { run } from './util.js';
|
|
3
3
|
|
|
4
4
|
// Parse Kane --agent NDJSON (ignores non-JSON lines) into a result object.
|
|
5
|
-
export function parseKane(text, exitCode) {
|
|
5
|
+
export function parseKane(text, exitCode, { timedOut = false } = {}) {
|
|
6
6
|
const events = [];
|
|
7
7
|
for (const line of String(text).split('\n')) {
|
|
8
8
|
const s = line.trim();
|
|
@@ -19,8 +19,12 @@ export function parseKane(text, exitCode) {
|
|
|
19
19
|
const failedRun = runEnds.find((e) => e.status === 'failed');
|
|
20
20
|
|
|
21
21
|
let status;
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
// A verdict needs evidence in the stream. Exit 1 alone is not enough: cmd.exe also exits 1 when kane-cli is
|
|
23
|
+
// not found, and a killed (timed out) run can exit 1. Those are errors, never a pass or a fail.
|
|
24
|
+
const failEvidence = !!(failedStepEnd || failedRun || summary?.overall_status === 'failed' || done?.overall_status === 'failed');
|
|
25
|
+
if (timedOut || summary?.cancelled === true) status = 'error';
|
|
26
|
+
else if (exitCode === 0) status = summary?.overall_status === 'failed' || done?.overall_status === 'failed' ? 'failed' : summary || done ? 'passed' : 'error';
|
|
27
|
+
else if (exitCode === 1) status = failEvidence ? 'failed' : 'error';
|
|
24
28
|
else status = 'error';
|
|
25
29
|
|
|
26
30
|
let failedStep = null;
|
|
@@ -48,9 +52,9 @@ export function parseKane(text, exitCode) {
|
|
|
48
52
|
export async function runKane(file, { timeoutMs = 900000 } = {}) {
|
|
49
53
|
const args = ['testmd', 'run', basename(file), '--agent', '--headless'];
|
|
50
54
|
const r = await run('kane-cli', args, { cwd: dirname(file), env: { KANE_CLI_USER_AGENT: 'workers' }, timeoutMs });
|
|
51
|
-
const result = parseKane(r.stdout, r.code);
|
|
55
|
+
const result = parseKane(r.stdout, r.code, { timedOut: r.timedOut });
|
|
52
56
|
if (result.status === 'error' && !result.failedStep) {
|
|
53
|
-
result.errorText = (r.stderr || '').trim().split('\n').slice(-2).join(' ') || `kane-cli exited with code ${r.code}`;
|
|
57
|
+
result.errorText = (r.stderr || '').trim().split('\n').slice(-2).join(' ') || `kane-cli exited with code ${r.code}${r.timedOut ? ' (timed out)' : ''}`;
|
|
54
58
|
}
|
|
55
59
|
return result;
|
|
56
60
|
}
|
package/src/run-playwright.js
CHANGED
|
@@ -21,15 +21,19 @@ export function parseResults(json) {
|
|
|
21
21
|
return map;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
// Regex for a test title that is safe inside a shell command: letters, digits, space, _ - and non-ASCII are kept,
|
|
25
|
+
// every other character (all shell and regex metacharacters) becomes '.', which still matches it. Matching a few
|
|
26
|
+
// extra tests is harmless: results are mapped by test id.
|
|
27
|
+
export const grepPattern = (title) => String(title).replace(/[\x00-\x1f\x21-\x2c\x2e-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]/g, '.');
|
|
28
|
+
|
|
24
29
|
// Run the selected tests once (one Playwright process). Results are mapped by id.
|
|
25
30
|
export async function runPlaywright(config, tests, { all = false } = {}) {
|
|
26
31
|
const args = [];
|
|
27
32
|
if (!all) {
|
|
28
|
-
|
|
29
|
-
args.push('--grep', `(${tests.map((t) => esc(t.title)).join('|')})`);
|
|
33
|
+
args.push('--grep', `(${tests.map((t) => grepPattern(t.title)).join('|')})`);
|
|
30
34
|
}
|
|
31
35
|
const map = parseResults(await playwrightJson(args, config));
|
|
32
36
|
const out = {};
|
|
33
|
-
for (const t of tests) out[t.id] = map.get(t.id) ?? { passed: false, errorMessage: 'test did not run' };
|
|
37
|
+
for (const t of tests) out[t.id] = map.get(t.id) ?? { passed: false, errorMessage: 'test did not run', didNotRun: true };
|
|
34
38
|
return out;
|
|
35
39
|
}
|