tewip-ai 0.8.0 → 0.9.0

package/README.md

@@ -166,6 +166,24 @@ that lost its markup) and **5 of 5 harmless changes ignored** (capitals, extra s
 implemented as a link, reordered sections). No designer has judged the output yet, so the roadmap's
 own metric is still open.
 
+## Which flaky tests to quarantine, and what they cost (new)
+
+```bash
+tewip flaky --days 30        # the dossier: quarantine, fix, or watch - with the evidence
+tewip flaky --apply          # writes the quarantine line into the test file
+```
+
+Other tools count retries. Tewip already decided *why* each failure happened, so it can say: this
+test failed in 161 of 647 runs **across 109 different branches** - that is the test, not your
+change - and quarantining it gives the team back 148 CI minutes a week. It refuses to hide a test
+that mostly fails (that one is broken, not flaky) and **never** quarantines a test whose failures
+looked like a real bug.
+
+Measured on three real projects' CI history (Gutenberg twice, Supabase): 76 tests proposed for
+quarantine, **zero of them containing a single real-bug failure**, giving back roughly 34, 148 and
+1,178 CI minutes a week. It also tells you when a quarantined test has stopped failing and should
+come back - the half nobody ever does.
+
 ## Bugs filed back, and test cases where your team keeps them (Phase 10, completed)
 
 ```bash

package/TEAM-GUIDE.md

@@ -127,6 +127,7 @@ collects environment failures in one issue, and (optionally) opens a fix pull re
 | `tewip spec "<feature>"` | An epic, its user stories and acceptance criteria, grounded in your code; files them in Jira with `--project PROJ --create` | Before the work is broken down |
 | `tewip report report.xml --gate` | The CI command for GitLab, Jenkins, CircleCI, Azure, Bitbucket and the rest: diagnoses the run, comments on the GitLab merge request, and fails the build only on failures that need a person | Your CI is not GitHub Actions |
 | `tewip watch --cmd "npm test"` | Runs your tests when you save a file and explains each failure straight away | While you are writing code |
+| `tewip flaky --days 30` | Which tests to quarantine, which to fix, what they cost in CI minutes - and when a quarantined one can come back | Monday morning |
 | `tewip note --days 7` | The weekly quality note for a lead, from runs already diagnosed (no model calls, free) | Monday morning |
 | `tewip fix --code report.xml` | Opt-in: proposes the smallest change to application code for a real bug, and keeps it only if the failing test passes and the whole suite still passes | "It is a real bug and I want a starting patch" |
 | `tewip design <figma link> --url <page>` | Compares a Figma frame against the running page: the words the design specifies and the controls its layer names imply | "Did we build what was designed?" |

package/dist/cli/tewip.js

@@ -71,6 +71,11 @@ Usage:
                                     Beside your editor: runs your tests when a file changes (or
                                     watches for new reports) and explains each failure as it
                                     happens, without CI.
+  tewip flaky [--days 30] [--apply] [--name <project>] [--out flaky.md]
+                                    Which tests are wasting the team's time, how much CI time they
+                                    cost, and what to do with each: quarantine, fix, or watch. Read
+                                    from runs already diagnosed, so it costs nothing. --apply puts
+                                    the quarantine line in the test file with the evidence above it.
   tewip note [--days 7] [--name <project>] [--out note.md] [--post]
                                     The weekly quality note for a lead, written from the runs Tewip
                                     already diagnosed. No model calls, so it costs nothing.
@@ -254,6 +259,17 @@ async function main() {
             });
             return;
         }
+        case 'flaky': {
+            const { runFlaky } = await import('../engine/flaky/flaky-command.js');
+            await runFlaky({
+                root: process.cwd(),
+                name: flag(args, 'name'),
+                days: Math.max(1, Number(flag(args, 'days')) || 30),
+                apply: args.includes('--apply'),
+                out: flag(args, 'out'),
+            });
+            return;
+        }
         case 'note': {
             const { runNote } = await import('../engine/notify/note-command.js');
             await runNote({

package/dist/engine/flaky/flaky-command.js

@@ -0,0 +1,142 @@
+// `tewip flaky`: the Monday morning answer. Which tests are wasting the team's time, how much, and
+// what to do about each one - read from the runs Tewip already diagnosed, so it costs nothing and
+// cannot re-judge an old failure with today's rules.
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { datasetFiles, readRunSummaries, readTriageRecords } from '../../reporters/store.js';
+import { flakyTests, readyToRelease, weeklyCost } from './flaky.js';
+import { applyQuarantine, frameworkOf, quarantineFor } from './quarantine.js';
+const DAY_MS = 24 * 60 * 60 * 1000;
+const minutes = (ms) => `${(ms / 60_000).toFixed(1)} CI min`;
+function projects(root, only) {
+    const dir = join(root, '.tewip', 'history');
+    if (only)
+        return [only];
+    return existsSync(dir) ? readdirSync(dir).filter((name) => existsSync(join(dir, name, 'triage.jsonl'))) : [];
+}
+/** The test file on disk, when the report's path still exists in this project. */
+function testFile(root, test) {
+    const direct = resolve(root, test.file);
+    return test.file && existsSync(direct) ? direct : null;
+}
+export async function runFlaky(cmd) {
+    const names = projects(cmd.root, cmd.name);
+    if (names.length === 0) {
+        throw new Error(`no history in ${join(cmd.root, '.tewip', 'history')}. Run \`tewip explain\` or \`tewip report\` on a few runs first: this is read from what those kept.`);
+    }
+    const since = new Date(Date.now() - cmd.days * DAY_MS).toISOString();
+    const all = [];
+    const lines = ['# Flaky tests', '', `The last ${cmd.days} days, from runs Tewip already diagnosed. No tests were run and no model was called.`, ''];
+    for (const name of names) {
+        const history = datasetFiles(join(cmd.root, '.tewip', 'history', name));
+        const records = readTriageRecords(history.triage);
+        const runs = readRunSummaries(history.runs);
+        const tests = flakyTests(records, runs, { since });
+        if (tests.length === 0)
+            continue;
+        all.push(...tests);
+        const cost = weeklyCost(tests, cmd.days);
+        lines.push(`## ${name}`, '', `${tests.length} test(s) failed at least once. About ${cost.minutes} CI minute(s) and ${cost.attempts} failed attempt(s) a week.`, '');
+        const group = (verdict, title, explain) => {
+            const chosen = tests.filter((t) => t.verdict === verdict);
+            if (chosen.length === 0)
+                return;
+            lines.push(`### ${title} (${chosen.length})`, '', explain, '');
+            for (const test of chosen.slice(0, 15)) {
+                lines.push(`- **${test.title}**`);
+                lines.push(`  - ${test.failedRuns} of ${test.totalRuns} run(s), ${test.failedAttempts} failed attempt(s), ${minutes(test.wastedMs)} lost${test.recoveredOnRetry > 0 ? `, passed on a retry ${test.recoveredOnRetry} time(s)` : ''}`);
+                lines.push(`  - Tewip read them as: ${test.categories.map((c) => `${c.category} ×${c.count}`).join(', ')}`);
+                lines.push(`  - Why: ${test.why}`);
+                if (test.file)
+                    lines.push(`  - Where: ${test.file}`);
+            }
+            lines.push('');
+        };
+        group('quarantine', 'Quarantine these', 'Unreliable rather than wrong. Stop them blocking everyone, then fix them on purpose.');
+        group('fix-now', 'Fix these, do not hide them', 'Hiding one of these would hide something real.');
+        group('watch', 'Watch', 'Not enough evidence yet to act.');
+        // Coming back out is the half nobody ever does.
+        const quarantined = readQuarantined(cmd.root);
+        const back = readyToRelease(quarantined, records, runs, since);
+        if (back.length > 0) {
+            lines.push('### Ready to come out of quarantine', '', 'These ran in this period and did not fail once.', '');
+            for (const testId of back)
+                lines.push(`- ${testId.split(' > ').at(-1)} — take its quarantine line out and let it block again.`);
+            lines.push('');
+        }
+    }
+    if (all.length === 0) {
+        console.log(`Nothing failed in the last ${cmd.days} days. Try --days 30.`);
+        return [];
+    }
+    const toQuarantine = all.filter((t) => t.verdict === 'quarantine');
+    if (cmd.apply && toQuarantine.length > 0)
+        lines.push(...applyAll(cmd.root, toQuarantine));
+    else if (toQuarantine.length > 0)
+        lines.push(...suggest(cmd.root, toQuarantine));
+    const report = lines.join('\n');
+    console.log(report);
+    if (cmd.out) {
+        writeFileSync(resolve(cmd.out), `${report}\n`, 'utf8');
+        console.log(`\nWritten to ${cmd.out}`);
+    }
+    return all;
+}
+/** Tests already carrying Tewip's quarantine line, found by reading the test files it knows about. */
+function readQuarantined(root) {
+    const file = join(root, '.tewip', 'quarantined.txt');
+    return existsSync(file) ? readFileSync(file, 'utf8').split('\n').map((l) => l.trim()).filter(Boolean) : [];
+}
+function suggest(root, tests) {
+    const lines = ['### The line to add', ''];
+    for (const test of tests.slice(0, 5)) {
+        const path = testFile(root, test);
+        const framework = path ? frameworkOf(path, readFileSync(path, 'utf8')) : null;
+        if (!path || !framework) {
+            lines.push(`- ${test.title}: ${test.file || 'file not found in this project'} - quarantine it the way this project does.`);
+            continue;
+        }
+        lines.push(`- ${test.title} (${test.file}):`, '```', quarantineFor(test, framework).line, '```');
+    }
+    lines.push('', 'Run `tewip flaky --apply` to put these lines in, each with the evidence above it. Only the named tests are touched.', '');
+    return lines;
+}
+/** Writes the quarantine into the test files, and remembers which tests were quarantined. */
+function applyAll(root, tests) {
+    const lines = ['### Applied', ''];
+    const done = [];
+    for (const test of tests) {
+        const path = testFile(root, test);
+        if (!path) {
+            lines.push(`- ${test.title}: not applied, ${test.file || 'no file'} is not in this project`);
+            continue;
+        }
+        const source = readFileSync(path, 'utf8');
+        const framework = frameworkOf(path, source);
+        if (!framework) {
+            lines.push(`- ${test.title}: not applied, the framework of ${test.file} was not recognised`);
+            continue;
+        }
+        const changed = applyQuarantine(source, test, framework);
+        if (!changed) {
+            lines.push(`- ${test.title}: not applied, the test was not found in ${test.file} (was it renamed?)`);
+            continue;
+        }
+        if (changed === source) {
+            lines.push(`- ${test.title}: already quarantined`);
+            done.push(test.testId);
+            continue;
+        }
+        writeFileSync(path, changed, 'utf8');
+        done.push(test.testId);
+        lines.push(`- ${test.title}: quarantined in ${test.file}`);
+    }
+    if (done.length > 0) {
+        const file = join(root, '.tewip', 'quarantined.txt');
+        const before = readQuarantined(root);
+        writeFileSync(file, [...new Set([...before, ...done])].join('\n') + '\n', 'utf8');
+        lines.push('', `Remembered in ${join('.tewip', 'quarantined.txt')}, so Tewip can tell you when each one is safe to bring back.`);
+    }
+    lines.push('', 'Nothing else in those files was changed. Review the diff before committing: a quarantined test is a decision, not a fix.', '');
+    return lines;
+}

package/dist/engine/flaky/flaky.js

@@ -0,0 +1,104 @@
+const titleOf = (testId) => testId.split(' > ').at(-1) ?? testId;
+const fileOf = (testId, record) => record?.file ?? testId.split(' > ').find((p) => /\.(spec|test)\./.test(p)) ?? '';
+/**
+ * A test's story across the kept runs. `records` are the failures Tewip diagnosed; `runs` say how
+ * often each test ran at all, which is what turns a count into a rate.
+ */
+export function flakyTests(records, runs, options = {}) {
+    const { since = '', minRuns = 2, alwaysFailsAbove = 0.8 } = options;
+    const inWindow = runs.filter((r) => (r.startedAt ?? '') >= since);
+    const runIds = new Set(inWindow.map((r) => r.runId));
+    const branchOf = new Map(inWindow.map((r) => [r.runId, r.branch ?? '']));
+    const failures = records.filter((r) => runIds.has(r.runId) || (!r.runId && !since));
+    const ranIn = new Map();
+    for (const run of inWindow)
+        for (const testId of Object.keys(run.attempts ?? {}))
+            ranIn.set(testId, (ranIn.get(testId) ?? 0) + 1);
+    const byTest = new Map();
+    for (const record of failures)
+        byTest.set(record.testId, [...(byTest.get(record.testId) ?? []), record]);
+    const out = [];
+    for (const [testId, all] of byTest) {
+        const runsFailed = new Set(all.map((r) => r.runId));
+        // A retry that passed: the same test, same run, a later attempt that is not in the failures.
+        const attemptsByRun = new Map();
+        for (const r of all)
+            attemptsByRun.set(r.runId, [...(attemptsByRun.get(r.runId) ?? []), r]);
+        let recovered = 0;
+        for (const [runId, attempts] of attemptsByRun) {
+            const ran = inWindow.find((run) => run.runId === runId)?.attempts?.[testId] ?? attempts.length;
+            if (ran > attempts.length)
+                recovered++;
+        }
+        const counts = new Map();
+        for (const r of all)
+            counts.set(r.diagnosis?.category ?? 'UNKNOWN', (counts.get(r.diagnosis?.category ?? 'UNKNOWN') ?? 0) + 1);
+        const categories = [...counts].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count);
+        const times = all.map((r) => r.timestamp).sort();
+        const branches = [...new Set([...runsFailed].map((id) => branchOf.get(id) ?? '').filter(Boolean))];
+        const totalRuns = ranIn.get(testId) ?? runsFailed.size;
+        const test = {
+            testId,
+            title: titleOf(testId),
+            file: fileOf(testId, all[0]),
+            failedRuns: runsFailed.size,
+            totalRuns,
+            failedAttempts: all.length,
+            wastedMs: all.reduce((sum, r) => sum + (r.duration ?? 0), 0),
+            recoveredOnRetry: recovered,
+            branches,
+            categories,
+            firstSeen: times[0] ?? '',
+            lastSeen: times.at(-1) ?? '',
+            verdict: 'watch',
+            why: '',
+        };
+        Object.assign(test, decide(test, { minRuns, alwaysFailsAbove }));
+        out.push(test);
+    }
+    return out.sort((a, b) => b.wastedMs - a.wastedMs || b.failedRuns - a.failedRuns);
+}
+const share = (test) => (test.totalRuns > 0 ? test.failedRuns / test.totalRuns : 1);
+const flakyShare = (test) => {
+    const flaky = test.categories.find((c) => c.category === 'TIMING')?.count ?? 0;
+    return test.failedAttempts > 0 ? flaky / test.failedAttempts : 0;
+};
+/**
+ * The decision, and why. Three rules, all conservative:
+ *   a test that mostly fails is broken, not flaky - hiding it hides a real problem;
+ *   a test whose failures Tewip read as the application being wrong is never quarantined;
+ *   quarantine needs the test to have failed in several runs, and to be unreliable rather than
+ *   consistently wrong.
+ */
+function decide(test, { minRuns, alwaysFailsAbove }) {
+    const rate = share(test);
+    const percent = `${Math.round(rate * 100)}% of the ${test.totalRuns} run(s) it was in`;
+    const realBugs = test.categories.find((c) => c.category === 'REAL_BUG')?.count ?? 0;
+    if (realBugs > 0 && realBugs >= test.failedAttempts / 2) {
+        return { verdict: 'fix-now', why: `${realBugs} of its ${test.failedAttempts} failures looked like the application being wrong, not the test. Quarantining it would hide a bug.` };
+    }
+    if (rate > alwaysFailsAbove && test.totalRuns >= minRuns) {
+        return { verdict: 'fix-now', why: `it fails in ${percent}. That is not flakiness, it is a test that does not work - hiding it would hide whatever it is telling you.` };
+    }
+    if (test.failedRuns >= minRuns && flakyShare(test) >= 0.5) {
+        const spread = test.branches.length > 1 ? ` It failed on ${test.branches.length} different branches, so it is the test rather than one person's change.` : '';
+        return { verdict: 'quarantine', why: `it failed in ${percent}, and Tewip read most of those as the test being unreliable.${spread}` };
+    }
+    return { verdict: 'watch', why: `it failed in ${percent}: not often enough, or not consistently enough one thing, to act on yet.` };
+}
+/** What a week of this costs, so the number means something to whoever pays for CI. */
+export function weeklyCost(tests, days) {
+    const wasted = tests.reduce((sum, t) => sum + t.wastedMs, 0);
+    const attempts = tests.reduce((sum, t) => sum + t.failedAttempts, 0);
+    const weeks = Math.max(days / 7, 0.1);
+    return { minutes: Math.round(wasted / 60_000 / weeks), attempts: Math.round(attempts / weeks) };
+}
+/**
+ * Tests that were quarantined and have stopped failing. Nobody ever takes a test out of quarantine,
+ * so this looks for the ones that have earned it: no failure in the window, and they ran.
+ */
+export function readyToRelease(quarantined, records, runs, since) {
+    const failed = new Set(records.filter((r) => r.timestamp >= since).map((r) => r.testId));
+    const ran = new Set(runs.filter((r) => (r.startedAt ?? '') >= since).flatMap((r) => Object.keys(r.attempts ?? {})));
+    return quarantined.filter((testId) => ran.has(testId) && !failed.has(testId));
+}

package/dist/engine/flaky/quarantine.js

@@ -0,0 +1,77 @@
+/** Which framework a test file belongs to, from its name and its contents. */
+export function frameworkOf(file, source) {
+    if (/\.py$/.test(file))
+        return 'pytest';
+    if (/\.java$|\.kt$/.test(file))
+        return 'junit';
+    if (/\.cs$/.test(file))
+        return 'dotnet';
+    if (/_test\.go$/.test(file))
+        return 'go';
+    if (/@playwright\/test|from ['"]playwright/.test(source))
+        return 'playwright';
+    if (/from ['"]vitest['"]|\bvi\./.test(source))
+        return 'vitest';
+    if (/@jest\/globals|jest\./.test(source))
+        return 'jest';
+    if (/\brequire\(['"]mocha|describe\(/.test(source))
+        return 'mocha';
+    return /\.[cm]?[jt]sx?$/.test(file) ? 'jest' : null;
+}
+const MARK = {
+    playwright: (title) => `test.fixme(true, ${JSON.stringify(`quarantined by Tewip: ${title}`)});`,
+    jest: () => 'test.skip',
+    vitest: () => 'test.skip',
+    mocha: () => 'it.skip',
+    pytest: (title) => `@pytest.mark.skip(reason=${JSON.stringify(`quarantined by Tewip: ${title}`)})`,
+    junit: (title) => `@Disabled(${JSON.stringify(`quarantined by Tewip: ${title}`)})`,
+    dotnet: (title) => `[Fact(Skip = ${JSON.stringify(`quarantined by Tewip: ${title}`)})]`,
+    go: () => 't.Skip("quarantined by Tewip")',
+};
+/** What to write, and the note that goes above it. */
+export function quarantineFor(test, framework) {
+    const rate = test.totalRuns > 0 ? `${test.failedRuns} of ${test.totalRuns} runs` : `${test.failedRuns} run(s)`;
+    return {
+        line: MARK[framework](test.title),
+        comment: [
+            `Quarantined by Tewip on ${new Date().toISOString().slice(0, 10)}: failed in ${rate}, ${Math.round(test.wastedMs / 60_000)} CI minute(s) lost.`,
+            test.branches.length > 1 ? `It failed on ${test.branches.length} branches, so it is the test and not one change.` : '',
+            'Take this line out when the cause is fixed. `tewip flaky` will say when it has stopped failing.',
+        ].filter(Boolean),
+    };
+}
+/**
+ * The file with the test quarantined, or null when the test could not be found in it. Only the one
+ * test named is touched; everything else in the file stays byte for byte as it was.
+ */
+export function applyQuarantine(source, test, framework) {
+    const nl = source.includes('\r\n') ? '\r\n' : '\n';
+    const lines = source.split(nl);
+    const quoted = test.title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const declaration = new RegExp(`^(\\s*)((?:async\\s+)?(?:export\\s+)?(?:test|it|def test_|public\\s+void|func\\s+Test)\\b[^\\n]*${quoted})`, 'i');
+    const index = lines.findIndex((line) => declaration.test(line) || line.includes(`"${test.title}"`) || line.includes(`'${test.title}'`));
+    if (index < 0)
+        return null;
+    if (lines.slice(Math.max(0, index - 4), index + 1).some((l) => /quarantined by tewip/i.test(l)))
+        return source; // already done
+    const indent = (lines[index].match(/^\s*/) ?? [''])[0];
+    const { line, comment } = quarantineFor(test, framework);
+    const commentMark = framework === 'pytest' ? '#' : '//';
+    const note = comment.map((c) => `${indent}${commentMark} ${c}`);
+    // Where the mark goes differs: an annotation sits above the test, a skip replaces the call.
+    if (framework === 'jest' || framework === 'vitest' || framework === 'mocha') {
+        const replaced = lines[index].replace(/\b(test|it)(\.\w+)?\(/, `${line}(`);
+        if (replaced === lines[index])
+            return null;
+        return [...lines.slice(0, index), ...note, replaced, ...lines.slice(index + 1)].join(nl);
+    }
+    if (framework === 'playwright') {
+        // Playwright's fixme goes inside the test body, so the test still reports itself as skipped.
+        const body = lines[index].includes('=>') ? index : -1;
+        if (body < 0)
+            return null;
+        return [...lines.slice(0, index), ...note, lines[index], `${indent}  ${line}`, ...lines.slice(index + 1)].join(nl);
+    }
+    // pytest, JUnit, .NET: the annotation goes on its own line above the test.
+    return [...lines.slice(0, index), ...note, `${indent}${line}`, ...lines.slice(index)].join(nl);
+}

package/dist/engine/llm/client.js

@@ -14,14 +14,41 @@ const MAX_ATTEMPTS = 3;
 /** TEWIP_LLM_CACHE moves the cache, e.g. into the project the command line runs in. */
 const CACHE_FILE = process.env.TEWIP_LLM_CACHE || join(DATA_DIR, 'llm-cache.jsonl');
 let envLoaded = false;
-/** Loads <project>/.env once. Variables already set in the environment win. */
+/**
+ * Loads a `.env` once, from the project being worked on.
+ *
+ * This used to read only Tewip's own folder, which is right when running from a checkout and wrong
+ * for everybody who installs it: their key sits in their project's .env, and Tewip silently found
+ * no model and answered with rules alone. The working directory is now looked at first, then its
+ * parent (a monorepo package), then Tewip's own folder for development.
+ *
+ * Variables already set in the environment always win: CI sets real secrets, a file must not
+ * quietly replace them.
+ */
 export function loadDotEnv() {
     if (envLoaded)
         return;
     envLoaded = true;
-    const file = join(PROJECT_ROOT, '.env');
-    if (existsSync(file))
-        process.loadEnvFile(file);
+    const places = [process.cwd(), join(process.cwd(), '..'), PROJECT_ROOT];
+    for (const dir of places) {
+        const file = join(dir, '.env');
+        if (!existsSync(file))
+            continue;
+        const before = new Set(Object.keys(process.env));
+        const kept = {};
+        for (const name of before)
+            kept[name] = process.env[name];
+        try {
+            process.loadEnvFile(file);
+        }
+        catch {
+            continue; // an unreadable or malformed file is not a reason to stop
+        }
+        for (const [name, value] of Object.entries(kept))
+            if (value !== undefined)
+                process.env[name] = value;
+        return;
+    }
 }
 export function openAiClient({ apiKey, model, reasoningEffort, baseUrl = 'https://api.openai.com/v1' }) {
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [
     "test-failures",