tewip-ai 0.4.0

package/dist/engine/quality/write-command.js

@@ -0,0 +1,146 @@
+// `tewip write`: turn a plan's "to automate next" scenarios into tests in the project's own
+// framework. Kept tests stay beside the project's tests for the team to review and commit; every
+// other draft is moved to .tewip/drafts with the reason, and nothing is committed or pushed.
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { dirname, extname, join, relative } from 'node:path';
+import { noModelHelp } from '../llm/providers.js';
+import { usageLine } from '../llm/usage.js';
+import { chooseFramework, detectFrameworks } from './frameworks.js';
+import { readPages, relatedCodeSources, relatedTestSources } from './grounding.js';
+import { writerFromEnv } from './model.js';
+import { writeAndVerify } from './write-verify.js';
+function latestPlan(root) {
+    const dir = join(root, '.tewip', 'plans');
+    const plans = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith('.json')).map((f) => join(dir, f)) : [];
+    if (plans.length === 0)
+        throw new Error('no plan to write from: run `tewip plan "<feature>"` first, or pass --plan <file.json>');
+    return plans.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0];
+}
+export async function runWrite(cmd) {
+    const model = writerFromEnv(cmd.root);
+    if (!model)
+        throw new Error(`writing tests needs a model.
+${noModelHelp()}`);
+    const planPath = cmd.plan ?? latestPlan(cmd.root);
+    const saved = JSON.parse(readFileSync(planPath, 'utf8'));
+    const found = detectFrameworks(cmd.root);
+    const urls = cmd.urls.length ? cmd.urls : saved.urls;
+    const detected = chooseFramework(found, cmd.framework, urls.length > 0);
+    if (!detected) {
+        const seen = found.map((f) => f.id).join(', ') || 'none';
+        throw new Error(`no ${cmd.framework ?? 'test'} framework found to write in (found: ${seen}). Tewip writes Playwright, Cypress, WebdriverIO, Angular (ng test), Jest, Vitest, pytest, JUnit (Maven/Gradle), .NET and Go tests.`);
+    }
+    // Copy the style of the existing test closest to the feature, when it is one of this framework's
+    // tests: it shows the helpers and conventions this part of the code base is tested with.
+    // Only a test this framework runs: a browser framework runs what is in its test folder; a unit
+    // framework runs tests anywhere in its project.
+    const within = detected.browser ? detected.testDir : detected.cwd;
+    // .ts and .tsx (or .js and .jsx) are the same language; the x only allows JSX, which component tests need.
+    const family = (f) => extname(f).replace(/x$/, '');
+    const sameKind = (f) => detected.example !== null && family(f) === family(detected.example) && f.startsWith(within);
+    const closest = relatedTestSources(cmd.root, saved.feature, 10).map((t) => join(cmd.root, t.file)).find(sameKind);
+    // New tests take the example's own extension, so a component test gets .tsx when its neighbours do.
+    const fileFor = (id) => (closest ? detected.fileFor(id).replace(/\.[^.]+$/, extname(closest)) : detected.fileFor(id));
+    const framework = closest ? { ...detected, example: closest, testDir: dirname(closest), fileFor } : detected;
+    const wanted = new Set(cmd.only.map((s) => s.toUpperCase()));
+    const scenarios = saved.plan.scenarios.filter((s) => (wanted.size ? wanted.has(s.id.toUpperCase()) : s.automate && !s.covered_by)).slice(0, cmd.max);
+    if (scenarios.length === 0)
+        throw new Error(wanted.size ? `none of ${cmd.only.join(', ')} is in ${planPath}` : 'the plan has nothing left to automate');
+    const code = relatedCodeSources(cmd.root, saved.feature);
+    const { pages, note } = await readPages(urls, cmd.root);
+    console.error(`Tewip is writing ${scenarios.length} test(s) in ${framework.label}, in ${relative(cmd.root, framework.testDir) || '.'}${framework.example ? `, in the style of ${relative(cmd.root, framework.example)}` : ''}.`);
+    if (note)
+        console.error(`  ${note}`);
+    if (!framework.browser && pages.length)
+        console.error('  note: this framework does not drive a browser; pass --framework to pick a browser one if the scenarios need pages.');
+    const drafts = join(cmd.root, '.tewip', 'drafts');
+    const results = [];
+    const started = Date.now();
+    let done = 0;
+    for (const scenario of scenarios) {
+        const fileName = framework.fileFor(scenario.id);
+        const file = join(framework.testDir, fileName);
+        if (existsSync(file) || (cmd.moveTo && existsSync(join(cmd.moveTo, fileName)))) {
+            results.push({ id: scenario.id, title: scenario.title, file, outcome: { kind: 'rejected', reason: `${fileName} already exists; left untouched`, repairs: 0, passedReliably: false } });
+            continue;
+        }
+        console.error(`  ${scenario.id} ${scenario.title}`);
+        // Stopped part-way (Ctrl+C), a draft that was never verified must not stay in the project.
+        const leaveNothing = () => {
+            if (existsSync(file)) {
+                mkdirSync(drafts, { recursive: true });
+                renameSync(file, join(drafts, fileName));
+            }
+            process.exit(130);
+        };
+        process.once('SIGINT', leaveNothing).once('SIGTERM', leaveNothing);
+        let outcome;
+        try {
+            ({ outcome } = await writeAndVerify(model, { framework, fileName, pages, feature: saved.feature, code }, scenario, file, { runs: cmd.runs, repairs: cmd.repairs, log: (l) => console.error(l) }));
+        }
+        catch (error) {
+            outcome = { kind: 'rejected', reason: `stopped: ${error.message}`, repairs: 0, passedReliably: false };
+        }
+        finally {
+            process.off('SIGINT', leaveNothing).off('SIGTERM', leaveNothing);
+        }
+        // Some runners take minutes per run (Angular rebuilds the app for every `ng test`): say how long
+        // the rest will take, so a slow run is not mistaken for a stuck one.
+        done++;
+        const left = scenarios.length - results.length - 1;
+        if (done === 1 && left > 0)
+            console.error(`    (about ${Math.max(1, Math.round((Date.now() - started) / 60_000))} min per scenario here; ${left} more to go)`);
+        if (outcome.kind !== 'kept' && existsSync(file)) {
+            mkdirSync(drafts, { recursive: true });
+            renameSync(file, join(drafts, fileName));
+            if ('lastError' in outcome && outcome.lastError)
+                writeFileSync(join(drafts, `${fileName}.log`), outcome.lastError, 'utf8');
+        }
+        if (outcome.kind === 'kept' && cmd.moveTo) {
+            mkdirSync(cmd.moveTo, { recursive: true });
+            renameSync(file, join(cmd.moveTo, fileName));
+            results.push({ id: scenario.id, title: scenario.title, file: join(cmd.moveTo, fileName), outcome });
+            console.error(`    → kept, moved to ${relative(cmd.root, cmd.moveTo) || '.'}`);
+            continue;
+        }
+        console.error(`    → ${outcome.kind}${outcome.kind === 'kept' ? '' : `: ${outcome.reason}`}`);
+        results.push({ id: scenario.id, title: scenario.title, file, outcome });
+    }
+    const report = renderWriteReport(results, { root: cmd.root, planPath, framework: framework.label, runs: cmd.runs, model: model.model });
+    const out = join(cmd.root, '.tewip', 'write', `${new Date().toISOString().replace(/[:.]/g, '-')}.md`);
+    mkdirSync(join(out, '..'), { recursive: true });
+    writeFileSync(out, report, 'utf8');
+    console.log(report);
+    console.error(`\nSaved to ${out}\n${usageLine()}`);
+}
+export function renderWriteReport(results, meta) {
+    const kept = results.filter((r) => r.outcome.kind === 'kept');
+    const bugs = results.filter((r) => r.outcome.kind === 'possible-bug');
+    const reliable = results.filter((r) => r.outcome.kind === 'kept' || (r.outcome.kind === 'rejected' && r.outcome.passedReliably));
+    const reason = (r) => (r.outcome.kind === 'rejected' ? r.outcome.reason : '');
+    const cannotFail = reliable.filter((r) => reason(r).startsWith('still passes')).length;
+    const unproven = reliable.length - kept.length - cannotFail;
+    const lines = [
+        '# Tewip: tests written',
+        '',
+        `From ${relative(meta.root, meta.planPath)}, in ${meta.framework}, by ${meta.model}. A test is kept only if it passed every run against the working app (${meta.runs} runs, or 2 where a single run takes over 90 seconds) **and** failed once its main check was broken on purpose.`,
+        '',
+        `- **Kept: ${kept.length} of ${results.length} drafts.**`,
+        `- Mutation check: ${kept.length} of ${reliable.length} test(s) that passed reliably also failed when their check was broken.${cannotFail ? ` ${cannotFail} still passed with it broken, so they could not catch the bug and were dropped.` : ''}${unproven ? ` ${unproven} could not be checked (the break could not be applied or did not run) and were dropped unproven.` : ''}`,
+        ...(bugs.length ? [`- **Possible bugs: ${bugs.length}.** The test looked right and the app did something else; a person should look.`] : []),
+        '',
+        '| Scenario | Result | Why |',
+        '|---|---|---|',
+        ...results.map((r) => {
+            const o = r.outcome;
+            const watch = o.kind === 'kept' && o.watch.length ? ` **Review:** may break when the data changes, not the app: \`${o.watch.join('`, `')}\`` : '';
+            const why = o.kind === 'kept' ? `passed ${o.runs} run(s); checks: ${o.checks}${o.repairs ? ` (after ${o.repairs} repair(s))` : ''}${watch}` : o.reason;
+            const where = o.kind === 'kept' ? `kept: \`${relative(meta.root, r.file).replaceAll('\\', '/')}\`` : o.kind === 'possible-bug' ? '**possible bug**' : 'not kept';
+            return `| ${r.id} ${r.title.replaceAll('|', '/')} | ${where} | ${why.replace(/\s+/g, ' ').replaceAll('|', '/')} |`;
+        }),
+        '',
+        'Kept tests are new files (beside your tests, or where --move-to put them): review, then commit them yourself. Drafts not kept are in `.tewip/drafts/` with their last error. The broken-on-purpose change is chosen by the model, so it shows the test can fail on its own main check, not that it would catch every bug.',
+        '',
+    ];
+    return lines.join('\n');
+}

package/dist/engine/quality/write-verify.js

@@ -0,0 +1,140 @@
+// The verification half of `tewip write`: a draft is kept only when it passes against the working
+// app every time it is run, and fails once its main check is broken on purpose. Nothing here asks
+// the model whether a test is good; it runs the test and reads the result.
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { isAbsolute, join } from 'node:path';
+import { brittleChecks, draftTest, failureHead, mutate, remutate, repairTest, weakChecks as weakOf } from './write.js';
+/**
+ * Every test in the run was skipped (a runner exits 0 for that): nothing was checked, so it is not
+ * a pass. Playwright, pytest and Jest print "1 skipped", Mocha "1 pending", Maven "Skipped: 1".
+ */
+export function onlySkipped(output) {
+    const karma = output.match(/Executed (\d+) of (\d+) \(skipped (\d+)\)/);
+    if (karma)
+        return karma[1] === '0';
+    const maven = output.match(/Tests run: (\d+), Failures: 0, Errors: 0, Skipped: (\d+)/);
+    if (maven)
+        return maven[1] === maven[2] && maven[1] !== '0';
+    return /\b[1-9]\d* (?:skipped|pending)\b/i.test(output) && !/\b[1-9]\d* pass(?:ed|ing)\b/i.test(output);
+}
+/** The page as it was when the test failed, when the runner saved it (Playwright's error context). */
+function pageAtFailure(output, cwd) {
+    const path = output.match(/Error Context: (.+\.md)/)?.[1]?.trim();
+    if (!path)
+        return '';
+    const full = isAbsolute(path) ? path : join(cwd, path);
+    return existsSync(full) ? `\n\nThe page when the test failed:\n${readFileSync(full, 'utf8').slice(0, 5000)}` : '';
+}
+/**
+ * The runner itself broke before or around the test: the browser never started, a port was taken,
+ * the app's server did not come up, the run was cut off. That says nothing about the test, so the
+ * run is tried again, and if it keeps breaking, the environment is named, never the test.
+ */
+const RUNNER_BROKE = /has not captured in \d+ ?ms|failed \d+ times \(timeout\)\. Giving up|Disconnected[, ].{0,40}because no message|No captured browser|EADDRINUSE|Timed out waiting \d+ms from config\.webServer|webServer.{0,60}(?:exited|timed out)|ENOSPC|spawnSync .* ETIMEDOUT/i;
+export function runTest(framework, file, timeoutMs = 180_000) {
+    let result = runOnce(framework, file, timeoutMs);
+    for (let retry = 0; retry < 2 && !result.passed && RUNNER_BROKE.test(result.output); retry++)
+        result = runOnce(framework, file, timeoutMs);
+    if (!result.passed && RUNNER_BROKE.test(result.output))
+        return { ...result, environment: true };
+    return result;
+}
+function runOnce(framework, file, timeoutMs) {
+    const r = spawnSync(framework.command(file), { cwd: framework.cwd, env: { ...process.env, ...framework.env }, shell: true, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 20 * 1024 * 1024 });
+    // Colour codes are noise to a reader and would stop a quoted line from matching.
+    const output = `${r.stdout ?? ''}\n${r.stderr ?? ''}${r.error ? `\n${r.error.message}` : ''}`.replace(/\u001b\[[0-9;]*[A-Za-z]/g, '').trim();
+    if (r.status === 0 && onlySkipped(output))
+        return { passed: false, skipped: true, output: `The test was skipped, so nothing was checked. Make it run here.\n${output.slice(-3000)}` };
+    return { passed: r.status === 0, output: output.slice(-6000) + (r.status === 0 ? '' : pageAtFailure(output, framework.cwd)) };
+}
+/** Above this, one run of the test is slow enough to trade a reliability run for time. */
+const SLOW_RUN_MS = 90_000;
+/** The broken version did not run at all, so its failure proves nothing about the check. */
+const DID_NOT_RUN = /SyntaxError|error TS\d{4}|IndentationError|NameError|ReferenceError|Unexpected token|compilation (?:error|failure)|cannot find symbol|error CS\d{4}/;
+export async function writeAndVerify(model, ctx, scenario, file, opts) {
+    const run = opts.run ?? runTest;
+    const log = opts.log ?? (() => { });
+    let draft = await draftTest(model, ctx, scenario);
+    let repairs = 0;
+    let result;
+    const earlier = [];
+    let slowest = 0;
+    for (;;) {
+        writeFileSync(file, draft.code, 'utf8');
+        const startedAt = Date.now();
+        result = run(ctx.framework, file);
+        slowest = Math.max(slowest, Date.now() - startedAt);
+        if (result.passed)
+            break;
+        if (result.environment)
+            return { draft, outcome: { kind: 'rejected', reason: 'the test runner could not start (environment: browser, server or port), so the test was not judged', repairs, lastError: result.output, passedReliably: false } };
+        if (repairs >= opts.repairs)
+            return { draft, outcome: { kind: 'rejected', reason: `still failing after ${repairs} repair(s)`, repairs, lastError: result.output, passedReliably: false } };
+        repairs++;
+        log(`    failed; repair ${repairs} of ${opts.repairs} from the real error`);
+        const repaired = await repairTest(model, ctx, scenario, draft, result.output, earlier);
+        earlier.push(failureHead(result.output));
+        if (repaired.fault === 'app') {
+            // The model's word is not enough: its quote must really be in what the run printed.
+            // A bare label ("Received:") is in every failure and shows nothing, so the quote must say more.
+            const quote = (repaired.observed ?? '').trim();
+            const telling = quote.length >= 16 && quote.replace(/\b(?:Received|Expected|Error|Actual|Call log)\b|[\s:'"`.,-]/g, '').length >= 8;
+            if (telling && result.output.includes(quote))
+                return { draft, outcome: { kind: 'possible-bug', reason: `${repaired.reason} (seen: "${quote.slice(0, 200)}")`, repairs, lastError: result.output } };
+            return { draft, outcome: { kind: 'rejected', reason: `the model blamed the app but could not quote the run's output to show it, so it is not reported as a bug: ${repaired.reason.slice(0, 200)}`, repairs, lastError: result.output, passedReliably: false } };
+        }
+        draft = repaired;
+    }
+    // A runner that takes minutes per run (Angular rebuilds the app each time) gets one reliability run
+    // less: the report always says how many runs a kept test passed.
+    const runs = slowest > SLOW_RUN_MS ? Math.min(opts.runs, 2) : opts.runs;
+    if (runs < opts.runs)
+        log(`    each run takes about ${Math.round(slowest / 1000)}s here, so ${runs} runs instead of ${opts.runs}`);
+    for (let i = 2; i <= runs; i++) {
+        const again = run(ctx.framework, file);
+        if (again.environment)
+            return { draft, outcome: { kind: 'rejected', reason: `the test runner broke on run ${i} (environment: browser, server or port), so reliability was not judged`, repairs, lastError: again.output, passedReliably: false } };
+        if (!again.passed)
+            return { draft, outcome: { kind: 'rejected', reason: `flaky: passed ${i - 1} time(s), then failed on run ${i}`, repairs, lastError: again.output, passedReliably: false } };
+    }
+    log(`    passed ${runs} of ${runs} runs; breaking its check on purpose`);
+    let mutant = mutate(draft);
+    if ('problem' in mutant) {
+        log(`    the deliberate break could not be applied (${mutant.problem}); asking once more`);
+        draft = await remutate(model, draft, mutant.problem);
+        mutant = mutate(draft);
+    }
+    if ('problem' in mutant)
+        return { draft, outcome: { kind: 'rejected', reason: `could not prove it can fail: ${mutant.problem}`, repairs, passedReliably: true } };
+    const runBroken = (code) => {
+        try {
+            writeFileSync(file, code, 'utf8');
+            return run(ctx.framework, file);
+        }
+        finally {
+            writeFileSync(file, draft.code, 'utf8');
+        }
+    };
+    let broken = runBroken(mutant.code);
+    // A break that does not compile (a type error, a syntax slip) proves nothing: ask once for another,
+    // showing the compiler's complaint.
+    if (!broken.passed && DID_NOT_RUN.test(broken.output)) {
+        log('    the deliberate break did not compile; asking once more');
+        draft = await remutate(model, draft, `the broken version did not compile or run: ${failureHead(broken.output)}`);
+        const again = mutate(draft);
+        if ('problem' in again)
+            return { draft, outcome: { kind: 'rejected', reason: `could not prove it can fail: ${again.problem}`, repairs, passedReliably: true } };
+        broken = runBroken(again.code);
+    }
+    if (broken.environment)
+        return { draft, outcome: { kind: 'rejected', reason: 'the test runner broke while checking the deliberate break (environment), so the check was not proven', repairs, lastError: broken.output, passedReliably: true } };
+    if (broken.skipped)
+        return { draft, outcome: { kind: 'rejected', reason: 'the broken version was skipped, so the check was not proven', repairs, passedReliably: true } };
+    if (broken.passed)
+        return { draft, outcome: { kind: 'rejected', reason: `still passes with its check broken (${draft.mutation.find} → ${draft.mutation.replace}): it could not catch the bug it is for`, repairs, passedReliably: true } };
+    if (DID_NOT_RUN.test(broken.output))
+        return { draft, outcome: { kind: 'rejected', reason: 'the broken version did not even run, so the check was not proven', repairs, lastError: broken.output, passedReliably: true } };
+    const weak = weakOf(draft.code);
+    return { draft, outcome: { kind: 'kept', runs, checks: draft.checks, repairs, mutation: `${draft.mutation.find} → ${draft.mutation.replace}`, watch: [...brittleChecks(draft.code), ...(weak ? [weak] : [])] } };
+}

package/dist/engine/quality/write.js

@@ -0,0 +1,144 @@
+// Phase 7: the model's side of `tewip write`. It drafts one test for one plan scenario, in the
+// project's own framework and style, using only elements seen on the real page, and names one
+// deliberate break of its main check (the mutation) so Tewip can prove the test is able to fail.
+// When a draft fails, the model repairs it from the real error - or says the app looks wrong,
+// because weakening a test until it passes would hide exactly the bug it exists to catch.
+import { readFileSync } from 'node:fs';
+export const WRITE_PROMPT_VERSION = 'write-6';
+const RULES = `Rules:
+- Write in the given framework and language, in the style of the example test: the same imports, helpers, and way of opening pages (if it uses a base URL with relative paths, do the same).
+- Find elements only by what the accessibility snapshots show: role and accessible name, label, placeholder or visible text. Never invent ids, classes or test ids that no snapshot shows.
+- Check the scenario's expected result specifically. A check that is always true (a page merely loads, a list merely exists) is not a test.
+- Wait for conditions, never for fixed times.
+- Do not hard-code values that change as the app's content grows (counts, totals, dates, ids) unless the scenario is about that exact value: check their shape instead (a number of books, a date), so the test does not break the day a record is added.
+- One test file, complete, runnable as is, no placeholders. It must actually run here: Tewip runs it against the app now, and a skipped test proves nothing. A guard like the example's (skip where the app lacks something) is fine, as long as it does not skip here.
+- mutation.find must be an exact substring of your code that appears once, inside the main assertion (the value or condition it expects); mutation.replace is what to put there instead so that the test must fail if it is a real check. Keep the code valid: change an expected value, not the syntax.`;
+const DRAFT_SYSTEM = `You are Tewip's test writer, a senior SDET. Write one automated test for the scenario given.\n\n${RULES}`;
+const REPAIR_SYSTEM = `You are Tewip's test writer, a senior SDET. Your test failed when run against the working app. Read the real error and decide:
+- fault "test": the test is wrong (a locator, a wait, a wrong assumption about the page). Return the corrected full test.
+- fault "app": the test is right and the app does not do what the scenario expects. Return the test unchanged, say in reason what the app did instead, and put in observed an exact quote (copied character for character) from the real error or the page snapshot that shows it. Without such a quote, it is not an app fault. For fault "test", observed is empty.
+If the code documents the behaviour the app showed (a comment or doc saying so), the scenario's expectation was wrong: the fault is "test"; correct the check to the documented behaviour and say so in reason.
+Never weaken the check of the scenario's expected result to make it pass: that would hide the bug the test exists to catch. If the only way to pass is to stop checking the expected result, the fault is "app".\n\n${RULES}`;
+const DRAFT_PROPS = {
+    code: { type: 'string' },
+    checks: { type: 'string' },
+    mutation: { type: 'object', additionalProperties: false, required: ['find', 'replace'], properties: { find: { type: 'string' }, replace: { type: 'string' } } },
+};
+const DRAFT_SCHEMA = { type: 'object', additionalProperties: false, required: ['code', 'checks', 'mutation'], properties: DRAFT_PROPS };
+const REPAIR_SCHEMA = {
+    type: 'object',
+    additionalProperties: false,
+    required: ['fault', 'reason', 'observed', 'code', 'checks', 'mutation'],
+    properties: { fault: { type: 'string', enum: ['test', 'app'] }, reason: { type: 'string' }, observed: { type: 'string' }, ...DRAFT_PROPS },
+};
+function contextFor(ctx, scenario) {
+    const example = ctx.framework.example ? readFileSync(ctx.framework.example, 'utf8').slice(0, 6000) : 'none: write in the framework\'s usual style';
+    return {
+        framework: ctx.framework.label,
+        fileName: ctx.fileName,
+        note: /\.(java|kt|cs)$/.test(ctx.fileName) ? `the class must be named ${ctx.fileName.replace(/\.\w+$/, '')}` : undefined,
+        feature: ctx.feature,
+        scenario: { id: scenario.id, title: scenario.title, given: scenario.preconditions, steps: scenario.steps, expected: scenario.expected },
+        exampleTest: example,
+        pages: ctx.pages.map((p) => ({ readAt: p.url, title: p.title, accessibilitySnapshot: p.aria })),
+        codeUnderTest: ctx.code?.length ? ctx.code : undefined,
+    };
+}
+export async function draftTest(model, ctx, scenario) {
+    const raw = await model.complete({ system: DRAFT_SYSTEM, user: JSON.stringify(contextFor(ctx, scenario)), schemaName: 'tewip_test_draft', schema: DRAFT_SCHEMA });
+    return JSON.parse(raw);
+}
+/** The first lines of a failure: which check failed and how, without the page or the call log. */
+export function failureHead(error) {
+    const start = error.search(/\b(?:\w*Error|FAILED|AssertionError|Exception)\b/);
+    return error.slice(Math.max(0, start)).split(/\n\s*(?:Call log:|at |The page when)/)[0].trim().slice(0, 800);
+}
+export async function repairTest(model, ctx, scenario, draft, error, earlier = []) {
+    // A check that keeps failing the same way after repairs is what the app consistently does: say so,
+    // so the model weighs "the app is wrong" instead of reshaping the test around the app.
+    const repeated = earlier.length > 0 ? { earlierFailures: earlier, note: 'Your earlier repairs failed too. If the same check keeps failing because the page does something other than the scenario expects, the fault is "app".' } : {};
+    const user = JSON.stringify({ ...contextFor(ctx, scenario), yourTest: draft.code, realError: error.slice(-10000), ...repeated });
+    const raw = await model.complete({ system: REPAIR_SYSTEM, user, schemaName: 'tewip_test_repair', schema: REPAIR_SCHEMA });
+    return JSON.parse(raw);
+}
+/**
+ * The broken-on-purpose version of a test, or why none could be made. The piece to change must
+ * occur exactly once, sit on a line that checks something, and actually change it.
+ */
+export function mutate(draft) {
+    const { find, replace } = draft.mutation;
+    if (!find || find === replace)
+        return { problem: 'the mutation changes nothing' };
+    const lineAt = (at) => {
+        const end = draft.code.indexOf('\n', at);
+        return draft.code.slice(draft.code.lastIndexOf('\n', at) + 1, end < 0 ? undefined : end);
+    };
+    const places = [];
+    for (let at = draft.code.indexOf(find); at >= 0; at = draft.code.indexOf(find, at + 1))
+        places.push(at);
+    if (places.length === 0)
+        return { problem: `the piece to change ("${find.slice(0, 60)}") is not in the test` };
+    // Several places: fine when exactly one of them is in a check (the same value is often both the
+    // input and the expected result).
+    // A check may span lines (a multi-line expect, or `out:` above the expected value), so the two
+    // lines before the piece count too.
+    const withContext = (at) => {
+        const start = draft.code.lastIndexOf('\n', at);
+        const before = draft.code.slice(0, Math.max(0, start)).split('\n').slice(-2).join('\n');
+        return `${before}\n${lineAt(at)}`;
+    };
+    const inChecks = places.filter((at) => CHECK_LINE.test(withContext(at)));
+    if (inChecks.length !== 1) {
+        return { problem: inChecks.length === 0 ? `the piece to change is not in a check: ${lineAt(places[0]).trim().slice(0, 100)}` : `the piece to change ("${find.slice(0, 60)}") is in ${inChecks.length} checks; pick one exact place` };
+    }
+    const at = inChecks[0];
+    return { code: draft.code.slice(0, at) + replace + draft.code.slice(at + find.length) };
+}
+/** A line that checks a result: an assertion, or the expected column of a table-driven test. */
+const CHECK_LINE = /expect|assert|should|verify|Assert|toHave|toBe|toEqual|toContain|toMatch|contains|Equal|raises|Raises|Throws|toThrow|[=!]==?|\b(?:out|output|expected|want|result)\s*[:=]/;
+const MUTATION_SCHEMA = {
+    type: 'object',
+    additionalProperties: false,
+    required: ['mutation'],
+    properties: { mutation: DRAFT_PROPS.mutation },
+};
+/** One more try at a usable deliberate break, when the first could not be applied. */
+export async function remutate(model, draft, problem) {
+    const user = JSON.stringify({ test: draft.code, yourMutation: draft.mutation, problem });
+    const system = 'You chose a deliberate break for this test so Tewip can prove it can fail, but it could not be applied. Choose again: mutation.find must be an exact substring of the test, inside the main check (an assertion, or the expected value in a table of cases), appearing exactly once inside a check; mutation.replace must make a correct test fail while keeping the code valid.';
+    const raw = await model.complete({ system, user, schemaName: 'tewip_mutation', schema: MUTATION_SCHEMA });
+    return { ...draft, mutation: JSON.parse(raw).mutation };
+}
+/**
+ * Checks that may break when the app's data changes rather than when it breaks: a number of three
+ * or more digits written into an expected text or value. Shown to the reviewer, not a reason to drop
+ * the test, since some scenarios are about an exact figure.
+ */
+export function brittleChecks(code) {
+    return code
+        .split('\n')
+        .filter((line) => /expect|assert|should|Assert/.test(line) && /(['"`])[^'"`\n]*\b\d{3,}\b[^'"`\n]*\1/.test(line) && !/timeout|status|toBeLessThan|toBeGreaterThan/i.test(line))
+        .map((line) => line.trim().slice(0, 140));
+}
+/** Lines that check something: an assertion, or an expected value in a table of cases. */
+const checkLines = (code) => code.split('\n').filter((line) => /expect|assert|should|Assert|verify/.test(line) && !/^\s*(?:\/\/|\*|#)/.test(line));
+/** A check that can only say something is absent: it passes as long as the thing never appears. */
+const NEGATIVE = /\.not[.\s(]|not\.to|toBeNull|toBeUndefined|toBeFalsy|assertFalse|assertNull|toHaveLength\(0\)|isNull|isFalse|\bnot_/;
+/** A check that is true of almost any value: it proves the code ran, not that it is right. */
+const SHALLOW = /toBeDefined\(\)|toBeTruthy\(\)|not\.toBeNull\(\)|toBeInstanceOf\(|typeof\s|assertIsNotNone|is not None|toBeGreaterThanOrEqual\(0\)/;
+/**
+ * Checks worth a second look, though the test passes and fails when broken: a test whose every
+ * check only says something is absent, or only that a value exists, can keep passing while the
+ * behaviour it is named for disappears. The deliberate break cannot see this, because it changes
+ * one check and that check does fail.
+ */
+export function weakChecks(code) {
+    const lines = checkLines(code);
+    if (lines.length === 0)
+        return null;
+    if (lines.every((l) => NEGATIVE.test(l)))
+        return `every check only says something is absent (${lines.length} check(s)), so it passes while the feature is missing`;
+    if (lines.every((l) => SHALLOW.test(l)))
+        return `every check only says a value exists (${lines.length} check(s)), not that it is right`;
+    return null;
+}

package/dist/engine/rules.js

@@ -0,0 +1,174 @@
+import { deviceEnvSignal } from './device-errors.js';
+import { isElementNotFound, networkSignal, valueMismatch } from './error-parse.js';
+import { locatorDrift, locatorDriftStrict } from './drift-rules.js';
+import { timingFlaky, timingIntercepted, timingIntermittent, timingKnownFlaky, timingRetryPass, timingStaleElement } from './timing-rules.js';
+import { errorScenarioOf } from './llm/context.js';
+import { errorShape, truncate } from './signature.js';
+export const MASS_MIN_TESTS = 3;
+export const MASS_MIN_SHARE = 0.3;
+export { FLAKY_MAX_RATE, FLAKY_MIN_RATE, TIMING_MIN_RUNS } from './timing-rules.js';
+export { DRIFT_MIN_SIMILARITY } from './drift-rules.js';
+function declined(source, message, data) {
+    return { category: null, confidence: 0, evidence: [{ source, message, data }] };
+}
+/**
+ * Many tests in one run failing with the same error shape → one shared cause outside the
+ * tests. Many failures with *different* errors (e.g. a bad deploy) are not enough, and
+ * neither are many "element not found" timeouts: those share a shape only because the call
+ * log (with each test's own locator) is left out, and they point to a UI change or a broken
+ * page rather than to the environment (found in the drift lab, rules-3).
+ */
+const envMassFailure = {
+    id: 'env-mass-failure',
+    evaluate({ record, run }) {
+        if (isElementNotFound(record.errorMessage)) {
+            const alike = run.failures.filter((f) => isElementNotFound(f.errorMessage)).length;
+            if (alike >= MASS_MIN_TESTS)
+                return declined(this.id, `${alike} tests in this run could not find their elements: a UI change or a broken page, not by itself an environment signal`);
+            return null;
+        }
+        const shape = errorShape(record.errorMessage);
+        const sameShape = new Set(run.failures.filter((f) => !isElementNotFound(f.errorMessage) && errorShape(f.errorMessage) === shape).map((f) => f.testId));
+        const failing = new Set(run.failures.map((f) => f.testId));
+        const attempted = Math.max(run.testsAttempted, failing.size, 1);
+        const share = sameShape.size / attempted;
+        if (sameShape.size >= MASS_MIN_TESTS && share >= MASS_MIN_SHARE) {
+            return {
+                category: 'ENV',
+                confidence: Math.min(0.95, 0.6 + 0.35 * share),
+                evidence: [
+                    {
+                        source: this.id,
+                        message: `${sameShape.size} of ${attempted} tests in this run failed with the same error: "${truncate(shape, 80)}"`,
+                        data: { tests: [...sameShape], share },
+                    },
+                ],
+            };
+        }
+        if (failing.size >= MASS_MIN_TESTS && failing.size / attempted >= MASS_MIN_SHARE) {
+            const shapes = new Set(run.failures.map((f) => errorShape(f.errorMessage))).size;
+            return declined(this.id, `${failing.size} of ${attempted} tests failed in this run, but with ${shapes} different errors, not one shared cause`);
+        }
+        return null;
+    },
+};
+/**
+ * A mobile run that failed because of the device, not the app: no emulator or simulator, the app
+ * never installed or crashed on launch, the automation server unreachable, the test sandbox not
+ * buildable. The same kind of answer as a refused connection, and as safe to act on: re-run it
+ * somewhere healthy. Detox reports a crash and a test timeout with the same error, so the reader
+ * declines the timeout case and this rule never sees it.
+ */
+const envDevice = {
+    id: 'env-device',
+    evaluate({ record }) {
+        const signal = deviceEnvSignal(record.errorMessage);
+        if (!signal)
+            return null;
+        const what = { 'no-device': 'no device or simulator was available', 'app-crashed': 'the app under test was not running', 'install-failed': 'the app or test runner could not be installed', 'driver-unreachable': 'the automation server could not be reached', sandbox: 'the test sandbox could not be built' }[signal.kind];
+        return {
+            category: 'ENV',
+            confidence: signal.confidence,
+            evidence: [{ source: this.id, message: `${what}: "${truncate(signal.match, 80)}"`, data: { ...signal } }],
+        };
+    },
+};
+const envNetwork = {
+    id: 'env-network',
+    evaluate({ record }) {
+        const signal = networkSignal(record.errorMessage);
+        if (!signal)
+            return null;
+        return {
+            category: 'ENV',
+            confidence: signal.confidence,
+            evidence: [{ source: this.id, message: `error shows a ${signal.kind} problem: "${truncate(signal.match, 60)}"`, data: { ...signal } }],
+        };
+    },
+};
+const DATA_COLLISION = /duplicate key value violates unique constraint|UNIQUE constraint failed|already exists/i;
+/**
+ * Tests colliding on shared data (two tests creating the same record, leftovers from another
+ * test). Checked before the retry rule: a retry passing only means the collision did not
+ * happen again. Found on Supabase CI (rules-4).
+ */
+const testDataCollision = {
+    id: 'test-data-collision',
+    evaluate({ record }) {
+        const match = record.errorMessage.match(DATA_COLLISION)?.[0];
+        if (!match)
+            return null;
+        return {
+            category: 'TEST_DATA',
+            confidence: 0.8,
+            evidence: [{ source: this.id, message: `the test's data collided with data that already existed: "${truncate(match, 60)}"`, data: { match } }],
+        };
+    },
+};
+/**
+ * The app's backend answered a server error (5xx) during the test. Checked before the retry
+ * rule, which used to call these races. Declines when the test provokes errors on purpose, and
+ * when the error is specific to this commit (possibly a backend bug the change introduced).
+ */
+const envServerError = {
+    id: 'env-server-error',
+    evaluate({ record, history, loadFailedRequests }) {
+        const failure = (loadFailedRequests?.() ?? []).find((r) => r.status !== null && r.status >= 500);
+        if (!failure)
+            return null;
+        const what = `${failure.url} returned ${failure.status} during the test`;
+        if (errorScenarioOf(record.testId))
+            return declined(this.id, `${what}, but the test's name says it exercises error handling`);
+        if (history.sameCommitAttempts > 0 && history.sameCommitFailures === history.sameCommitAttempts && history.passesOnOtherCommits > 0) {
+            return declined(this.id, `${what}, but only on this commit while other commits pass: possibly a backend bug the change introduced`);
+        }
+        // A test that has never passed anywhere (often a new test for new code) gives no baseline: its
+        // server error is as likely a bug in the new backend code as an environment problem. Found as
+        // the one real bug the merge gate would have let through on the labelled data (rules-8).
+        if (history.runs > 0 && history.failures >= history.runs) {
+            return declined(this.id, `${what}, but this test has never passed (${history.failures} failures in ${history.runs} attempts), so the error may be a bug in new code`);
+        }
+        return { category: 'ENV', confidence: 0.7, evidence: [{ source: this.id, message: what, data: { ...failure } }] };
+    },
+};
+/**
+ * A value mismatch on a test that was reliable until it started failing the same way.
+ * Declines when the server failed during the test: the wrong value may come from the
+ * backend (ENV) or from a test that provokes errors on purpose, which only the LLM layer,
+ * seeing the whole trace, can tell apart (rules-3).
+ */
+const realBugAssertion = {
+    id: 'real-bug-assertion',
+    evaluate({ record, history, loadFailedRequests }) {
+        const mismatch = valueMismatch(record.errorMessage);
+        if (!mismatch)
+            return null;
+        const { runs, failingStreak, streakDistinctErrors, failuresBeforeStreak, passesBeforeStreak } = history;
+        const values = `expected ${truncate(mismatch.expected, 30)}, received ${truncate(mismatch.received, 30)}`;
+        if (runs < 2)
+            return declined(this.id, `value mismatch (${values}) on the test's first recorded run; no history to call it stable`);
+        if (failingStreak === 0 || failuresBeforeStreak > 0) {
+            return declined(this.id, `value mismatch (${values}), but the test has passed and failed on and off, so its history isn't stable`);
+        }
+        if (streakDistinctErrors > 1) {
+            return declined(this.id, `value mismatch, but the error changed across the last ${failingStreak} failing runs (${streakDistinctErrors} variants)`);
+        }
+        const serverFailure = (loadFailedRequests?.() ?? []).find((r) => r.status !== null && r.status >= 500);
+        if (serverFailure) {
+            return declined(this.id, `value mismatch (${values}), but ${serverFailure.url} returned ${serverFailure.status} during the test: the backend may explain the wrong value`);
+        }
+        const message = passesBeforeStreak > 0
+            ? `passed ${passesBeforeStreak} time(s), then failed the last ${failingStreak} run(s) with the same mismatch: ${values}`
+            : `failed all ${failingStreak} recorded runs with the same mismatch: ${values}`;
+        return {
+            category: 'REAL_BUG',
+            confidence: Math.min(0.9, 0.65 + 0.05 * failingStreak + (passesBeforeStreak >= 3 ? 0.05 : 0)),
+            evidence: [{ source: this.id, message, data: { ...mismatch, failingStreak, passesBeforeStreak } }],
+        };
+    },
+};
+/**
+ * Priority order: the first matching rule decides. Signals from outside the test (ENV)
+ * come first, so a 5xx that also fails an assertion is ENV, not REAL_BUG.
+ */
+export const RULES = [envMassFailure, envNetwork, envDevice, testDataCollision, envServerError, locatorDriftStrict, timingIntercepted, timingStaleElement, timingRetryPass, locatorDrift, timingFlaky, timingIntermittent, timingKnownFlaky, realBugAssertion];

package/dist/engine/signature.js

@@ -0,0 +1,23 @@
+// Error-text normalisation shared by the history table and the rule engine.
+/**
+ * Error identity used for "same error every time?". Playwright appends a call log whose
+ * details (e.g. "9 × locator resolved to ...") change between identical failures, so
+ * only the text before "Call log:" is compared.
+ */
+export function errorSignature(message) {
+    return message.split(/\n\s*Call log:/)[0].replace(/\s+/g, ' ').trim();
+}
+/**
+ * Coarser identity used to spot one cause hitting many tests: numbers, URL paths and
+ * long ids are blanked, so "ECONNREFUSED at http://host/cart" and ".../checkout" compare
+ * equal. Quoted strings are kept, so unrelated assertion failures stay apart.
+ */
+export function errorShape(message) {
+    return errorSignature(message)
+        .replace(/\b(https?:\/\/[^/\s'"]+)[^\s'"]*/g, '$1')
+        .replace(/\b[0-9a-f]{8,}\b/gi, '<id>')
+        .replace(/\d+(?:\.\d+)?/g, '<n>');
+}
+export function truncate(text, max) {
+    return text.length <= max ? text : `${text.slice(0, max - 1)}…`;
+}